Public: anyone can read this; the crew can change it.

The model door never counts a served reply at $0: usage never goes down, a last event is read, a reply with no usage is estimated #13763

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13762-a-run-on-included-credit-costs-money-tha → main 4 commits fork

  • Merged. torque/13762-a-run-on-included-credit-costs-money-tha was merged into main as acf1eb3.

Review

Sign in to review this change.
3 files changed +239 −16
docs/features/child-credit.md +8
@@ -39,6 +39,14 @@ crew calls the model through the parent; the child never holds the credential.
3939 Haiku 1/5; cache write 1.25x, cache read 0.1x); an unknown model is priced as Opus.
4040 - A call under way when the cap is reached finishes and is counted. At most 4 calls per child run at
4141 once (a fifth is answered 429 at once), so the spend can pass the cap by no more than 4 calls cost.
42 - A served reply is never counted at $0 (#13762).
43 - Usage figures only rise across a stream's events. A gateway that repeats a field as 0 takes nothing back.
44 - A last event with no newline after it is still read.
45 - A reply that reports no usage is priced from the bytes that crossed: 3 bytes a token, at the
46 request's model, or at the dearest price when it names none.
47 - That estimate overcharges on purpose. It counts the whole request body as input at full price, so
48 a large prompt the provider served from its cache is billed as if it were not cached.
49 - A provider's error answer (any status but 2xx) is passed on and charged nothing, not estimated.
4250 - A request body is cut by the bytes that arrive, never by Content-Length; the bearer is checked
4351 before a byte is read (413 past 16 MB). A heartbeat must carry its signature and is read to 1 KB
4452 only. Bun holds one body limit per listener, so the listener's own limit is the model route's 16 MB.
src/spawn/model_door.ts +79 −16
@@ -50,26 +50,62 @@ function bearerOf(req: Request): string {
5050
5151 const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0);
5252
53 /**
54 * Usage is cumulative across a stream's events, and a gateway may repeat a field as 0 in a later
55 * one; a figure never goes down, so what was seen is kept (the larger of the two).
56 */
5357 function usageOf(u: unknown, into: Usage): Usage {
5458 if (typeof u !== "object" || u === null) return into;
5559 const r = u as Record<string, unknown>;
5660 return {
57 inputTokens: "input_tokens" in r ? num(r["input_tokens"]) : into.inputTokens,
58 outputTokens: "output_tokens" in r ? num(r["output_tokens"]) : into.outputTokens,
59 cacheWriteTokens: "cache_creation_input_tokens" in r ? num(r["cache_creation_input_tokens"]) : into.cacheWriteTokens,
60 cacheReadTokens: "cache_read_input_tokens" in r ? num(r["cache_read_input_tokens"]) : into.cacheReadTokens,
61 inputTokens: Math.max(into.inputTokens, num(r["input_tokens"])),
62 outputTokens: Math.max(into.outputTokens, num(r["output_tokens"])),
63 cacheWriteTokens: Math.max(into.cacheWriteTokens, num(r["cache_creation_input_tokens"])),
64 cacheReadTokens: Math.max(into.cacheReadTokens, num(r["cache_read_input_tokens"])),
6165 };
6266 }
6367
64 /** Reads usage out of a server-sent stream as it passes: `message_start` carries the input, `message_delta` the output so far. */
68 /**
69 * Bytes per token when a reply carries no usage: low, so the estimate is never under what was
70 * spent.
71 */
72 const BYTES_PER_TOKEN = 3;
73 const isNone = (u: Usage): boolean => u.inputTokens + u.outputTokens + u.cacheWriteTokens + u.cacheReadTokens === 0;
74
75 /**
76 * What a call is owed when the provider's reply said no usage at all: the bytes it carried, at list
77 * price, never $0.
78 */
79 export function estimatedUsage(askedBytes: number, answeredBytes: number): Usage {
80 return { ...NO_USAGE, inputTokens: Math.ceil(askedBytes / BYTES_PER_TOKEN), outputTokens: Math.ceil(answeredBytes / BYTES_PER_TOKEN) };
81 }
82
83 /** What the call asked of the model: its name and the bytes it sent. */
84 interface Asked {
85 readonly model: string;
86 readonly bytes: number;
87 }
88
89 /**
90 * Reads usage out of a server-sent stream as it passes: `message_start` carries the input,
91 * `message_delta` the output so far.
92 */
6593 class Tally {
6694 usage: Usage = NO_USAGE;
6795 model = "";
6896 private buf = "";
97 bytes = 0;
6998 private readonly dec = new TextDecoder();
7099
71 feed(chunk: Uint8Array): void {
72 this.buf += this.dec.decode(chunk, { stream: true });
100 /** The last line, when the stream ended without a newline after it. */
101 end(): void {
102 this.feed(new Uint8Array(0), true);
103 }
104
105 feed(chunk: Uint8Array, last = false): void {
106 this.bytes += chunk.length;
107 this.buf += this.dec.decode(chunk, { stream: !last });
108 if (last) this.buf += "\n";
73109 let at: number;
74110 while ((at = this.buf.indexOf("\n")) >= 0) {
75111 const line = this.buf.slice(0, at).trimEnd();
@@ -89,8 +125,18 @@ class Tally {
89125 }
90126 }
91127
128 /** The model a request names, or "" (priced as the dearest). */
129 function modelOf(raw: Uint8Array): string {
130 try {
131 const m = (JSON.parse(new TextDecoder().decode(raw)) as { model?: unknown }).model;
132 return typeof m === "string" ? m : "";
133 } catch {
134 return "";
135 }
136 }
137
92138 /** Counts a reply as it passes and gives the call's place back when it ends, however it ends. */
93 function meter(app: App, credit: Credit, fetched: Response, release: () => void): Response {
139 function meter(app: App, credit: Credit, fetched: Response, asked: Asked, release: () => void): Response {
94140 const type = fetched.headers.get("content-type") ?? "";
95141 const headers = new Headers({ "content-type": type || "application/json" });
96142 // A refusal or an error from the provider is not a call that was served: nothing is charged.
@@ -103,18 +149,32 @@ function meter(app: App, credit: Credit, fetched: Response, release: () => void)
103149 const tally = new Tally();
104150 const dec = new TextDecoder();
105151 let text = "";
152 let textBytes = 0;
106153 let done = false;
107154 const finish = (): void => {
108155 if (done) return;
109156 done = true;
110157 release();
111 if (streamed) return void addSpend(app, credit.name, listPriceUsd(tally.model, tally.usage));
112 try {
113 const doc = JSON.parse(text) as { model?: unknown; usage?: unknown };
114 addSpend(app, credit.name, listPriceUsd(typeof doc.model === "string" ? doc.model : "", usageOf(doc.usage, NO_USAGE)));
115 } catch {
116 // an unreadable answer is not counted; the provider's own ledger is the check
158 let model = asked.model;
159 let usage = NO_USAGE;
160 let answered = textBytes;
161 if (streamed) {
162 tally.end();
163 if (tally.model !== "") model = tally.model;
164 usage = tally.usage;
165 answered = tally.bytes;
166 } else {
167 try {
168 const doc = JSON.parse(text) as { model?: unknown; usage?: unknown };
169 if (typeof doc.model === "string" && doc.model !== "") model = doc.model;
170 usage = usageOf(doc.usage, NO_USAGE);
171 } catch {
172 // an answer that is not JSON says no usage: it is estimated below, not skipped
173 }
117174 }
175 // A served reply with no usage in it is not free: it is priced from the bytes that crossed.
176 if (isNone(usage) && answered > 0) usage = estimatedUsage(asked.bytes, answered);
177 addSpend(app, credit.name, listPriceUsd(model, usage));
118178 };
119179 return new Response(
120180 new ReadableStream<Uint8Array>({
@@ -127,7 +187,10 @@ function meter(app: App, credit: Credit, fetched: Response, release: () => void)
127187 return;
128188 }
129189 if (streamed) tally.feed(value);
130 else if (text.length < MODEL_BODY_MAX) text += dec.decode(value, { stream: true });
190 else {
191 textBytes += value.length;
192 if (text.length < MODEL_BODY_MAX) text += dec.decode(value, { stream: true });
193 }
131194 c.enqueue(value);
132195 } catch (e) {
133196 finish();
@@ -175,7 +238,7 @@ export async function lendModel(app: App, req: Request, deps: ModelDeps = {}): P
175238 return refuse(502, "api_error", "the parent could not reach its model provider");
176239 }
177240 handed = true;
178 return meter(app, credit, fetched, release);
241 return meter(app, credit, fetched, { model: modelOf(raw), bytes: raw.length }, release);
179242 } finally {
180243 if (!handed) release();
181244 }
test/door-counts-every-run.test.ts added +152
@@ -0,0 +1,152 @@
1 /**
2 * A run on included credit moves the host's counter by its list-price cost (#13762): driven
3 * through the door the way a crew run reaches it (`fundedEnv`'s variables), then read from the
4 * counter the host's `/api/v1/credit` and the plan line both report.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { openBeatDoor, parentLocal, type BeatDoor } from "../src/spawn/beat-door.ts";
12 import { readCredit, setCap } from "../src/spawn/credit.ts";
13 import { fundedEnv, keepFunding } from "../src/spawn/funding.ts";
14
15 let dir = "";
16 let childDir = "";
17 let parent: App;
18 let door: BeatDoor;
19 let reply: () => Response = () => new Response("");
20 const saved = process.env["FORGE_PARENT_LOCAL"];
21
22 beforeAll(async () => {
23 dir = await mkdtemp(join(tmpdir(), "door-counts-"));
24 childDir = join(dir, "child");
25 await mkdir(childDir, { recursive: true });
26 await writeFile(join(childDir, "parent.json"), "{}");
27 parent = createApp({ dataDir: join(dir, "parent"), url: "https://forge.test", port: 0 });
28 setCap(parent, "maria", 20, false, (token, credit) => keepFunding(childDir, { token, capUsd: credit.capUsd, spentUsd: credit.spentUsd }));
29 door = openBeatDoor(parent, { model: { upstream: () => ({ base: "https://provider.test", headers: { "x-api-key": "k" } }), fetch: async () => reply() } });
30 process.env["FORGE_PARENT_LOCAL"] = parentLocal(door.port);
31 });
32 afterAll(async () => {
33 if (saved === undefined) delete process.env["FORGE_PARENT_LOCAL"];
34 else process.env["FORGE_PARENT_LOCAL"] = saved;
35 door.close();
36 await rm(dir, { recursive: true, force: true });
37 });
38
39 const sse = (...events: object[]): Response =>
40 new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(""), { headers: { "content-type": "text/event-stream" } });
41 const spent = (): number => readCredit(parent, "maria")!.spentUsd;
42 const BODY = { model: "claude-sonnet-4-5", max_tokens: 8, stream: true, messages: [{ role: "user", content: "hi" }] };
43
44 /** A run's model call, from the environment the child hands its crew. */
45 async function run(body: object = BODY): Promise<string> {
46 const env = fundedEnv(childDir)!;
47 const res = await fetch(`${env["ANTHROPIC_BASE_URL"]}/v1/messages`, {
48 method: "POST",
49 headers: { "x-api-key": env["ANTHROPIC_API_KEY"]!, "content-type": "application/json" },
50 body: JSON.stringify(body),
51 });
52 expect(res.status).toBe(200);
53 return res.text();
54 }
55
56 test("a streamed run is counted at list price: 128k in, 3k out on sonnet is $0.429", async () => {
57 reply = () =>
58 sse(
59 { type: "message_start", message: { model: "claude-sonnet-4-5", usage: { input_tokens: 128_000, output_tokens: 1 } } },
60 { type: "message_delta", usage: { output_tokens: 3_000 } },
61 );
62 await run();
63 expect(spent()).toBeCloseTo((128_000 * 3 + 3_000 * 15) / 1e6, 6);
64 });
65
66 test("a later event that repeats a field as 0 does not take back what was counted", async () => {
67 const before = spent();
68 reply = () =>
69 sse(
70 { type: "message_start", message: { model: "claude-sonnet-4-5", usage: { input_tokens: 1_000_000, output_tokens: 1 } } },
71 { type: "message_delta", usage: { input_tokens: 0, output_tokens: 0 } },
72 );
73 await run();
74 expect(spent() - before).toBeCloseTo(3 + 15 / 1e6, 5);
75 });
76
77 test("a stream whose last event has no newline is still counted", async () => {
78 const before = spent();
79 const last = JSON.stringify({ type: "message_delta", usage: { output_tokens: 100_000 } });
80 reply = () =>
81 new Response(`data: ${JSON.stringify({ type: "message_start", message: { model: "claude-haiku-4-5", usage: { input_tokens: 0 } } })}\n\ndata: ${last}`, {
82 headers: { "content-type": "text/event-stream" },
83 });
84 await run();
85 expect(spent() - before).toBeCloseTo((100_000 * 5) / 1e6, 6);
86 });
87
88 /**
89 * A request of exactly `bytes` bytes naming `model` (or none), so the estimate has a hand figure.
90 */
91 const sized = (bytes: number, model?: string): object => {
92 const bare = JSON.stringify({ ...(model === undefined ? {} : { model }), pad: "" }).length;
93 return { ...(model === undefined ? {} : { model }), pad: "x".repeat(bytes - bare) };
94 };
95 const pad = (bytes: number): string => `data: ${JSON.stringify({ type: "ping", pad: "x".repeat(bytes - 32) })}\n\n`;
96
97 test("a streamed reply that says no usage is priced from its bytes: 3000 asked + 6000 answered on sonnet is $0.033", async () => {
98 const before = spent();
99 const answer = pad(6000);
100 expect(answer.length).toBe(6000);
101 reply = () => new Response(answer, { headers: { "content-type": "text/event-stream" } });
102 await run(sized(3000, "claude-sonnet-4-5"));
103 // 3 bytes a token: 1000 in at $3/M + 2000 out at $15/M
104 expect(spent() - before).toBeCloseTo(0.033, 6);
105 });
106
107 test("a JSON reply with no usage is priced from its bytes too: 3000 asked + 6000 answered on haiku is $0.011", async () => {
108 const before = spent();
109 const answer = JSON.stringify({ type: "message", pad: "x".repeat(6000 - 27) });
110 expect(answer.length).toBe(6000);
111 reply = () => new Response(answer, { headers: { "content-type": "application/json" } });
112 await run(sized(3000, "claude-haiku-4-5"));
113 // 1000 in at $1/M + 2000 out at $5/M
114 expect(spent() - before).toBeCloseTo(0.011, 6);
115 });
116
117 test("a request that names no model is priced as the dearest: 3000 asked + 6000 answered is $0.165", async () => {
118 const before = spent();
119 reply = () => new Response(pad(6000), { headers: { "content-type": "text/event-stream" } });
120 await run(sized(3000));
121 // opus: 1000 in at $15/M + 2000 out at $75/M
122 expect(spent() - before).toBeCloseTo(0.165, 6);
123 });
124
125 test("cache write and cache read figures never go down across a stream's events either", async () => {
126 const before = spent();
127 reply = () =>
128 sse(
129 {
130 type: "message_start",
131 message: { model: "claude-sonnet-4-5", usage: { input_tokens: 0, cache_creation_input_tokens: 1_000_000, cache_read_input_tokens: 1_000_000 } },
132 },
133 { type: "message_delta", usage: { cache_creation_input_tokens: 0, cache_read_input_tokens: 0, output_tokens: 0 } },
134 );
135 await run();
136 // sonnet: 1M cache write at 1.25 x $3 + 1M cache read at 0.1 x $3
137 expect(spent() - before).toBeCloseTo(3.75 + 0.3, 6);
138 });
139
140 test("a provider's error answer is passed on and charged nothing, not even an estimate from its bytes", async () => {
141 const before = spent();
142 const env = fundedEnv(childDir)!;
143 reply = () => Response.json({ type: "error", error: { type: "overloaded_error", message: "x".repeat(30_000) } }, { status: 529 });
144 const res = await fetch(`${env["ANTHROPIC_BASE_URL"]}/v1/messages`, {
145 method: "POST",
146 headers: { "x-api-key": env["ANTHROPIC_API_KEY"]!, "content-type": "application/json" },
147 body: JSON.stringify(sized(3000, "claude-sonnet-4-5")),
148 });
149 expect(res.status).toBe(529);
150 await res.text();
151 expect(spent() - before).toBe(0);
152 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs