Public: anyone can read this; the crew can change it.
The model door never counts a served reply at $0: usage never goes down, a last event is read, a reply with no usage is estimated #13763
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13762-a-run-on-included-credit-costs-money-tha → main 4 commits fork
3 files
changed +239
−16
docs/features/child-credit.md +8
| @@ -39,6 +39,14 @@ crew calls the model through the parent; the child never holds the credential. | ||
| 39 | 39 | Haiku 1/5; cache write 1.25x, cache read 0.1x); an unknown model is priced as Opus. |
| 40 | 40 | - A call under way when the cap is reached finishes and is counted. At most 4 calls per child run at |
| 41 | 41 | once (a fifth is answered 429 at once), so the spend can pass the cap by no more than 4 calls cost. |
| 42 | - A served reply is never counted at $0 (#13762). | |
| 43 | - Usage figures only rise across a stream's events. A gateway that repeats a field as 0 takes nothing back. | |
| 44 | - A last event with no newline after it is still read. | |
| 45 | - A reply that reports no usage is priced from the bytes that crossed: 3 bytes a token, at the | |
| 46 | request's model, or at the dearest price when it names none. | |
| 47 | - That estimate overcharges on purpose. It counts the whole request body as input at full price, so | |
| 48 | a large prompt the provider served from its cache is billed as if it were not cached. | |
| 49 | - A provider's error answer (any status but 2xx) is passed on and charged nothing, not estimated. | |
| 42 | 50 | - A request body is cut by the bytes that arrive, never by Content-Length; the bearer is checked |
| 43 | 51 | before a byte is read (413 past 16 MB). A heartbeat must carry its signature and is read to 1 KB |
| 44 | 52 | only. Bun holds one body limit per listener, so the listener's own limit is the model route's 16 MB. |
src/spawn/model_door.ts +79 −16
| @@ -50,26 +50,62 @@ function bearerOf(req: Request): string { | ||
| 50 | 50 | |
| 51 | 51 | const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0); |
| 52 | 52 | |
| 53 | /** | |
| 54 | * Usage is cumulative across a stream's events, and a gateway may repeat a field as 0 in a later | |
| 55 | * one; a figure never goes down, so what was seen is kept (the larger of the two). | |
| 56 | */ | |
| 53 | 57 | function usageOf(u: unknown, into: Usage): Usage { |
| 54 | 58 | if (typeof u !== "object" || u === null) return into; |
| 55 | 59 | const r = u as Record<string, unknown>; |
| 56 | 60 | return { |
| 57 | inputTokens: "input_tokens" in r ? num(r["input_tokens"]) : into.inputTokens, | |
| 58 | outputTokens: "output_tokens" in r ? num(r["output_tokens"]) : into.outputTokens, | |
| 59 | cacheWriteTokens: "cache_creation_input_tokens" in r ? num(r["cache_creation_input_tokens"]) : into.cacheWriteTokens, | |
| 60 | cacheReadTokens: "cache_read_input_tokens" in r ? num(r["cache_read_input_tokens"]) : into.cacheReadTokens, | |
| 61 | inputTokens: Math.max(into.inputTokens, num(r["input_tokens"])), | |
| 62 | outputTokens: Math.max(into.outputTokens, num(r["output_tokens"])), | |
| 63 | cacheWriteTokens: Math.max(into.cacheWriteTokens, num(r["cache_creation_input_tokens"])), | |
| 64 | cacheReadTokens: Math.max(into.cacheReadTokens, num(r["cache_read_input_tokens"])), | |
| 61 | 65 | }; |
| 62 | 66 | } |
| 63 | 67 | |
| 64 | /** Reads usage out of a server-sent stream as it passes: `message_start` carries the input, `message_delta` the output so far. */ | |
| 68 | /** | |
| 69 | * Bytes per token when a reply carries no usage: low, so the estimate is never under what was | |
| 70 | * spent. | |
| 71 | */ | |
| 72 | const BYTES_PER_TOKEN = 3; | |
| 73 | const isNone = (u: Usage): boolean => u.inputTokens + u.outputTokens + u.cacheWriteTokens + u.cacheReadTokens === 0; | |
| 74 | ||
| 75 | /** | |
| 76 | * What a call is owed when the provider's reply said no usage at all: the bytes it carried, at list | |
| 77 | * price, never $0. | |
| 78 | */ | |
| 79 | export function estimatedUsage(askedBytes: number, answeredBytes: number): Usage { | |
| 80 | return { ...NO_USAGE, inputTokens: Math.ceil(askedBytes / BYTES_PER_TOKEN), outputTokens: Math.ceil(answeredBytes / BYTES_PER_TOKEN) }; | |
| 81 | } | |
| 82 | ||
| 83 | /** What the call asked of the model: its name and the bytes it sent. */ | |
| 84 | interface Asked { | |
| 85 | readonly model: string; | |
| 86 | readonly bytes: number; | |
| 87 | } | |
| 88 | ||
| 89 | /** | |
| 90 | * Reads usage out of a server-sent stream as it passes: `message_start` carries the input, | |
| 91 | * `message_delta` the output so far. | |
| 92 | */ | |
| 65 | 93 | class Tally { |
| 66 | 94 | usage: Usage = NO_USAGE; |
| 67 | 95 | model = ""; |
| 68 | 96 | private buf = ""; |
| 97 | bytes = 0; | |
| 69 | 98 | private readonly dec = new TextDecoder(); |
| 70 | 99 | |
| 71 | feed(chunk: Uint8Array): void { | |
| 72 | this.buf += this.dec.decode(chunk, { stream: true }); | |
| 100 | /** The last line, when the stream ended without a newline after it. */ | |
| 101 | end(): void { | |
| 102 | this.feed(new Uint8Array(0), true); | |
| 103 | } | |
| 104 | ||
| 105 | feed(chunk: Uint8Array, last = false): void { | |
| 106 | this.bytes += chunk.length; | |
| 107 | this.buf += this.dec.decode(chunk, { stream: !last }); | |
| 108 | if (last) this.buf += "\n"; | |
| 73 | 109 | let at: number; |
| 74 | 110 | while ((at = this.buf.indexOf("\n")) >= 0) { |
| 75 | 111 | const line = this.buf.slice(0, at).trimEnd(); |
| @@ -89,8 +125,18 @@ class Tally { | ||
| 89 | 125 | } |
| 90 | 126 | } |
| 91 | 127 | |
| 128 | /** The model a request names, or "" (priced as the dearest). */ | |
| 129 | function modelOf(raw: Uint8Array): string { | |
| 130 | try { | |
| 131 | const m = (JSON.parse(new TextDecoder().decode(raw)) as { model?: unknown }).model; | |
| 132 | return typeof m === "string" ? m : ""; | |
| 133 | } catch { | |
| 134 | return ""; | |
| 135 | } | |
| 136 | } | |
| 137 | ||
| 92 | 138 | /** Counts a reply as it passes and gives the call's place back when it ends, however it ends. */ |
| 93 | function meter(app: App, credit: Credit, fetched: Response, release: () => void): Response { | |
| 139 | function meter(app: App, credit: Credit, fetched: Response, asked: Asked, release: () => void): Response { | |
| 94 | 140 | const type = fetched.headers.get("content-type") ?? ""; |
| 95 | 141 | const headers = new Headers({ "content-type": type || "application/json" }); |
| 96 | 142 | // A refusal or an error from the provider is not a call that was served: nothing is charged. |
| @@ -103,18 +149,32 @@ function meter(app: App, credit: Credit, fetched: Response, release: () => void) | ||
| 103 | 149 | const tally = new Tally(); |
| 104 | 150 | const dec = new TextDecoder(); |
| 105 | 151 | let text = ""; |
| 152 | let textBytes = 0; | |
| 106 | 153 | let done = false; |
| 107 | 154 | const finish = (): void => { |
| 108 | 155 | if (done) return; |
| 109 | 156 | done = true; |
| 110 | 157 | release(); |
| 111 | if (streamed) return void addSpend(app, credit.name, listPriceUsd(tally.model, tally.usage)); | |
| 112 | try { | |
| 113 | const doc = JSON.parse(text) as { model?: unknown; usage?: unknown }; | |
| 114 | addSpend(app, credit.name, listPriceUsd(typeof doc.model === "string" ? doc.model : "", usageOf(doc.usage, NO_USAGE))); | |
| 115 | } catch { | |
| 116 | // an unreadable answer is not counted; the provider's own ledger is the check | |
| 158 | let model = asked.model; | |
| 159 | let usage = NO_USAGE; | |
| 160 | let answered = textBytes; | |
| 161 | if (streamed) { | |
| 162 | tally.end(); | |
| 163 | if (tally.model !== "") model = tally.model; | |
| 164 | usage = tally.usage; | |
| 165 | answered = tally.bytes; | |
| 166 | } else { | |
| 167 | try { | |
| 168 | const doc = JSON.parse(text) as { model?: unknown; usage?: unknown }; | |
| 169 | if (typeof doc.model === "string" && doc.model !== "") model = doc.model; | |
| 170 | usage = usageOf(doc.usage, NO_USAGE); | |
| 171 | } catch { | |
| 172 | // an answer that is not JSON says no usage: it is estimated below, not skipped | |
| 173 | } | |
| 117 | 174 | } |
| 175 | // A served reply with no usage in it is not free: it is priced from the bytes that crossed. | |
| 176 | if (isNone(usage) && answered > 0) usage = estimatedUsage(asked.bytes, answered); | |
| 177 | addSpend(app, credit.name, listPriceUsd(model, usage)); | |
| 118 | 178 | }; |
| 119 | 179 | return new Response( |
| 120 | 180 | new ReadableStream<Uint8Array>({ |
| @@ -127,7 +187,10 @@ function meter(app: App, credit: Credit, fetched: Response, release: () => void) | ||
| 127 | 187 | return; |
| 128 | 188 | } |
| 129 | 189 | if (streamed) tally.feed(value); |
| 130 | else if (text.length < MODEL_BODY_MAX) text += dec.decode(value, { stream: true }); | |
| 190 | else { | |
| 191 | textBytes += value.length; | |
| 192 | if (text.length < MODEL_BODY_MAX) text += dec.decode(value, { stream: true }); | |
| 193 | } | |
| 131 | 194 | c.enqueue(value); |
| 132 | 195 | } catch (e) { |
| 133 | 196 | finish(); |
| @@ -175,7 +238,7 @@ export async function lendModel(app: App, req: Request, deps: ModelDeps = {}): P | ||
| 175 | 238 | return refuse(502, "api_error", "the parent could not reach its model provider"); |
| 176 | 239 | } |
| 177 | 240 | handed = true; |
| 178 | return meter(app, credit, fetched, release); | |
| 241 | return meter(app, credit, fetched, { model: modelOf(raw), bytes: raw.length }, release); | |
| 179 | 242 | } finally { |
| 180 | 243 | if (!handed) release(); |
| 181 | 244 | } |
test/door-counts-every-run.test.ts added +152
| @@ -0,0 +1,152 @@ | ||
| 1 | /** | |
| 2 | * A run on included credit moves the host's counter by its list-price cost (#13762): driven | |
| 3 | * through the door the way a crew run reaches it (`fundedEnv`'s variables), then read from the | |
| 4 | * counter the host's `/api/v1/credit` and the plan line both report. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { openBeatDoor, parentLocal, type BeatDoor } from "../src/spawn/beat-door.ts"; | |
| 12 | import { readCredit, setCap } from "../src/spawn/credit.ts"; | |
| 13 | import { fundedEnv, keepFunding } from "../src/spawn/funding.ts"; | |
| 14 | ||
| 15 | let dir = ""; | |
| 16 | let childDir = ""; | |
| 17 | let parent: App; | |
| 18 | let door: BeatDoor; | |
| 19 | let reply: () => Response = () => new Response(""); | |
| 20 | const saved = process.env["FORGE_PARENT_LOCAL"]; | |
| 21 | ||
| 22 | beforeAll(async () => { | |
| 23 | dir = await mkdtemp(join(tmpdir(), "door-counts-")); | |
| 24 | childDir = join(dir, "child"); | |
| 25 | await mkdir(childDir, { recursive: true }); | |
| 26 | await writeFile(join(childDir, "parent.json"), "{}"); | |
| 27 | parent = createApp({ dataDir: join(dir, "parent"), url: "https://forge.test", port: 0 }); | |
| 28 | setCap(parent, "maria", 20, false, (token, credit) => keepFunding(childDir, { token, capUsd: credit.capUsd, spentUsd: credit.spentUsd })); | |
| 29 | door = openBeatDoor(parent, { model: { upstream: () => ({ base: "https://provider.test", headers: { "x-api-key": "k" } }), fetch: async () => reply() } }); | |
| 30 | process.env["FORGE_PARENT_LOCAL"] = parentLocal(door.port); | |
| 31 | }); | |
| 32 | afterAll(async () => { | |
| 33 | if (saved === undefined) delete process.env["FORGE_PARENT_LOCAL"]; | |
| 34 | else process.env["FORGE_PARENT_LOCAL"] = saved; | |
| 35 | door.close(); | |
| 36 | await rm(dir, { recursive: true, force: true }); | |
| 37 | }); | |
| 38 | ||
| 39 | const sse = (...events: object[]): Response => | |
| 40 | new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(""), { headers: { "content-type": "text/event-stream" } }); | |
| 41 | const spent = (): number => readCredit(parent, "maria")!.spentUsd; | |
| 42 | const BODY = { model: "claude-sonnet-4-5", max_tokens: 8, stream: true, messages: [{ role: "user", content: "hi" }] }; | |
| 43 | ||
| 44 | /** A run's model call, from the environment the child hands its crew. */ | |
| 45 | async function run(body: object = BODY): Promise<string> { | |
| 46 | const env = fundedEnv(childDir)!; | |
| 47 | const res = await fetch(`${env["ANTHROPIC_BASE_URL"]}/v1/messages`, { | |
| 48 | method: "POST", | |
| 49 | headers: { "x-api-key": env["ANTHROPIC_API_KEY"]!, "content-type": "application/json" }, | |
| 50 | body: JSON.stringify(body), | |
| 51 | }); | |
| 52 | expect(res.status).toBe(200); | |
| 53 | return res.text(); | |
| 54 | } | |
| 55 | ||
| 56 | test("a streamed run is counted at list price: 128k in, 3k out on sonnet is $0.429", async () => { | |
| 57 | reply = () => | |
| 58 | sse( | |
| 59 | { type: "message_start", message: { model: "claude-sonnet-4-5", usage: { input_tokens: 128_000, output_tokens: 1 } } }, | |
| 60 | { type: "message_delta", usage: { output_tokens: 3_000 } }, | |
| 61 | ); | |
| 62 | await run(); | |
| 63 | expect(spent()).toBeCloseTo((128_000 * 3 + 3_000 * 15) / 1e6, 6); | |
| 64 | }); | |
| 65 | ||
| 66 | test("a later event that repeats a field as 0 does not take back what was counted", async () => { | |
| 67 | const before = spent(); | |
| 68 | reply = () => | |
| 69 | sse( | |
| 70 | { type: "message_start", message: { model: "claude-sonnet-4-5", usage: { input_tokens: 1_000_000, output_tokens: 1 } } }, | |
| 71 | { type: "message_delta", usage: { input_tokens: 0, output_tokens: 0 } }, | |
| 72 | ); | |
| 73 | await run(); | |
| 74 | expect(spent() - before).toBeCloseTo(3 + 15 / 1e6, 5); | |
| 75 | }); | |
| 76 | ||
| 77 | test("a stream whose last event has no newline is still counted", async () => { | |
| 78 | const before = spent(); | |
| 79 | const last = JSON.stringify({ type: "message_delta", usage: { output_tokens: 100_000 } }); | |
| 80 | reply = () => | |
| 81 | new Response(`data: ${JSON.stringify({ type: "message_start", message: { model: "claude-haiku-4-5", usage: { input_tokens: 0 } } })}\n\ndata: ${last}`, { | |
| 82 | headers: { "content-type": "text/event-stream" }, | |
| 83 | }); | |
| 84 | await run(); | |
| 85 | expect(spent() - before).toBeCloseTo((100_000 * 5) / 1e6, 6); | |
| 86 | }); | |
| 87 | ||
| 88 | /** | |
| 89 | * A request of exactly `bytes` bytes naming `model` (or none), so the estimate has a hand figure. | |
| 90 | */ | |
| 91 | const sized = (bytes: number, model?: string): object => { | |
| 92 | const bare = JSON.stringify({ ...(model === undefined ? {} : { model }), pad: "" }).length; | |
| 93 | return { ...(model === undefined ? {} : { model }), pad: "x".repeat(bytes - bare) }; | |
| 94 | }; | |
| 95 | const pad = (bytes: number): string => `data: ${JSON.stringify({ type: "ping", pad: "x".repeat(bytes - 32) })}\n\n`; | |
| 96 | ||
| 97 | test("a streamed reply that says no usage is priced from its bytes: 3000 asked + 6000 answered on sonnet is $0.033", async () => { | |
| 98 | const before = spent(); | |
| 99 | const answer = pad(6000); | |
| 100 | expect(answer.length).toBe(6000); | |
| 101 | reply = () => new Response(answer, { headers: { "content-type": "text/event-stream" } }); | |
| 102 | await run(sized(3000, "claude-sonnet-4-5")); | |
| 103 | // 3 bytes a token: 1000 in at $3/M + 2000 out at $15/M | |
| 104 | expect(spent() - before).toBeCloseTo(0.033, 6); | |
| 105 | }); | |
| 106 | ||
| 107 | test("a JSON reply with no usage is priced from its bytes too: 3000 asked + 6000 answered on haiku is $0.011", async () => { | |
| 108 | const before = spent(); | |
| 109 | const answer = JSON.stringify({ type: "message", pad: "x".repeat(6000 - 27) }); | |
| 110 | expect(answer.length).toBe(6000); | |
| 111 | reply = () => new Response(answer, { headers: { "content-type": "application/json" } }); | |
| 112 | await run(sized(3000, "claude-haiku-4-5")); | |
| 113 | // 1000 in at $1/M + 2000 out at $5/M | |
| 114 | expect(spent() - before).toBeCloseTo(0.011, 6); | |
| 115 | }); | |
| 116 | ||
| 117 | test("a request that names no model is priced as the dearest: 3000 asked + 6000 answered is $0.165", async () => { | |
| 118 | const before = spent(); | |
| 119 | reply = () => new Response(pad(6000), { headers: { "content-type": "text/event-stream" } }); | |
| 120 | await run(sized(3000)); | |
| 121 | // opus: 1000 in at $15/M + 2000 out at $75/M | |
| 122 | expect(spent() - before).toBeCloseTo(0.165, 6); | |
| 123 | }); | |
| 124 | ||
| 125 | test("cache write and cache read figures never go down across a stream's events either", async () => { | |
| 126 | const before = spent(); | |
| 127 | reply = () => | |
| 128 | sse( | |
| 129 | { | |
| 130 | type: "message_start", | |
| 131 | message: { model: "claude-sonnet-4-5", usage: { input_tokens: 0, cache_creation_input_tokens: 1_000_000, cache_read_input_tokens: 1_000_000 } }, | |
| 132 | }, | |
| 133 | { type: "message_delta", usage: { cache_creation_input_tokens: 0, cache_read_input_tokens: 0, output_tokens: 0 } }, | |
| 134 | ); | |
| 135 | await run(); | |
| 136 | // sonnet: 1M cache write at 1.25 x $3 + 1M cache read at 0.1 x $3 | |
| 137 | expect(spent() - before).toBeCloseTo(3.75 + 0.3, 6); | |
| 138 | }); | |
| 139 | ||
| 140 | test("a provider's error answer is passed on and charged nothing, not even an estimate from its bytes", async () => { | |
| 141 | const before = spent(); | |
| 142 | const env = fundedEnv(childDir)!; | |
| 143 | reply = () => Response.json({ type: "error", error: { type: "overloaded_error", message: "x".repeat(30_000) } }, { status: 529 }); | |
| 144 | const res = await fetch(`${env["ANTHROPIC_BASE_URL"]}/v1/messages`, { | |
| 145 | method: "POST", | |
| 146 | headers: { "x-api-key": env["ANTHROPIC_API_KEY"]!, "content-type": "application/json" }, | |
| 147 | body: JSON.stringify(sized(3000, "claude-sonnet-4-5")), | |
| 148 | }); | |
| 149 | expect(res.status).toBe(529); | |
| 150 | await res.text(); | |
| 151 | expect(spent() - before).toBe(0); | |
| 152 | }); |
new above
offline