Public: anyone can read this; the crew can change it.
The OpenAI runner honours the picked subscription and the spend caps, and reports what a run cost #15199
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/15198-the-openai-runner-honours-the-picked-sub → main 2 commits fork
9 files
changed +224
−26
docs/features/openai-runner.md +15 −2
| @@ -33,7 +33,20 @@ bytes, Grep skips files over 1 MB, reads 1 000 characters of a line and refuses | ||
| 33 | 33 | group that can backtrack without bound. A run ends at 60 turns. |
| 34 | 34 | |
| 35 | 35 | Not carried over from the Claude runner: resuming a session (each run starts fresh), steering a run |
| 36 | while it works, images, and cost: an endpoint reports tokens, not dollars, so cost is 0 and the turn | |
| 37 | limit holds the budget. | |
| 36 | while it works, and images. | |
| 37 | ||
| 38 | **A picked subscription is what the run spends** (#15198). When the forge picks an account for the run, | |
| 39 | its token goes to that account's gateway (`<gateway>/v1/chat/completions`, through the forge's door for | |
| 40 | a crew run), never to `OPENAI_BASE_URL` with the operator's key. An account with no gateway (a bare | |
| 41 | Anthropic token) cannot drive this runner, so the run ends as an error saying so rather than spending the | |
| 42 | operator's key on it. | |
| 43 | ||
| 44 | **Cost and the cap.** An endpoint reports tokens, not dollars, so the forge prices them from the table | |
| 45 | in `src/crew/openai_price.ts`: `cost = tokens in × in/1e6 + tokens out × out/1e6`, at the providers' list | |
| 46 | prices in USD per million tokens. It errs high: cached input is charged as fresh, a model pays the | |
| 47 | price of the longest name it starts with, and a model the table does not know pays its dearest row, | |
| 48 | $15/$75. The running cost reaches the crew through `onUsage` and the outcome's `costUsd`, and a turn | |
| 49 | that brings it to the run's budget ends the run with `max_budget_reached`, as a Claude run does; that | |
| 50 | turn's tool calls are not run. The run log's first line names the price used. | |
| 38 | 51 | |
| 39 | 52 | Pinned by `test/openai-runner.test.ts` and `test/openai-runner-escape.test.ts`; the code is `src/crew/openai_runner.ts`. |
src/crew/agent_runner.ts +13
| @@ -23,6 +23,13 @@ export interface AgentRun { | ||
| 23 | 23 | readonly appendSystemPrompt: string; |
| 24 | 24 | readonly model?: string | undefined; |
| 25 | 25 | readonly budgetUsd: number; |
| 26 | /** | |
| 27 | * The subscription picked for this run, for a runner that does not read | |
| 28 | * Claude Code's variables from `env` (#15198): its token and, for an | |
| 29 | * account with a gateway, the base its calls go to (a crew run's is the | |
| 30 | * forge's door). Absent: no account was picked and the run is the forge's own. | |
| 31 | */ | |
| 32 | readonly account?: RunAccount | undefined; | |
| 26 | 33 | /** A session id to continue. */ |
| 27 | 34 | readonly resume?: string | undefined; |
| 28 | 35 | readonly env: Record<string, string>; |
| @@ -78,6 +85,12 @@ export interface AgentRun { | ||
| 78 | 85 | } |
| 79 | 86 | |
| 80 | 87 | |
| 88 | export interface RunAccount { | |
| 89 | readonly token: string; | |
| 90 | /** The gateway's base as the run reaches it; absent for an account on its provider itself. */ | |
| 91 | readonly base?: string | undefined; | |
| 92 | } | |
| 93 | ||
| 81 | 94 | export interface Spent { |
| 82 | 95 | readonly tokensIn: number; |
| 83 | 96 | readonly tokensOut: number; |
src/crew/conversation.ts +2
| @@ -41,6 +41,7 @@ import { running } from "../kernel/version.ts"; | ||
| 41 | 41 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 42 | 42 | import { appendRunLog } from "../runs/log.ts"; |
| 43 | 43 | import { runAgent, type AgentOutcome, type RunnerLimit } from "./agent_runner.ts"; |
| 44 | import { runAccount } from "./runner_env.ts"; | |
| 44 | 45 | import { CONVERSATION_BRIEF } from "./conversation_brief.ts"; |
| 45 | 46 | import { isRateLimited, takeLimit, whenText } from "./limits.ts"; |
| 46 | 47 | import { turnBudget } from "./model_access.ts"; |
| @@ -587,6 +588,7 @@ function begin( | ||
| 587 | 588 | appendSystemPrompt: [CONVERSATION_BRIEF, memoryBrief(app.db, userId)].filter((p) => p !== "").join(" "), |
| 588 | 589 | model: options.model, |
| 589 | 590 | budgetUsd: budget, |
| 591 | account: runAccount(picked, modelEnv), | |
| 590 | 592 | resume: session, |
| 591 | 593 | env: { |
| 592 | 594 | ...modelEnv, |
src/crew/one_shot.ts +2
| @@ -13,6 +13,7 @@ import { id } from "../kernel/id.ts"; | ||
| 13 | 13 | import { redactValues, releaseValues } from "../kernel/log.ts"; |
| 14 | 14 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 15 | 15 | import { runAgent } from "./agent_runner.ts"; |
| 16 | import { runAccount } from "./runner_env.ts"; | |
| 16 | 17 | import { conversationOptions, sessionsDir, turnDir } from "./conversation.ts"; |
| 17 | 18 | import { isRateLimited } from "./limits.ts"; |
| 18 | 19 | import { turnBudget } from "./model_access.ts"; |
| @@ -55,6 +56,7 @@ export async function oneShot(app: App, userId: string, system: string, prompt: | ||
| 55 | 56 | appendSystemPrompt: system, |
| 56 | 57 | model: model ?? options.model, |
| 57 | 58 | budgetUsd: budget, |
| 59 | account: runAccount(picked, modelEnv), | |
| 58 | 60 | env: { ...modelEnv, CLAUDE_CONFIG_DIR: sessionsDir(app.config.dataDir, options.configDir) }, |
| 59 | 61 | sandbox: sandboxPolicy({ dataDir: app.config.dataDir, agentName: "one-shot", urls: [], enforced: false, runId }), |
| 60 | 62 | label: "one-shot", |
src/crew/openai_price.ts added +59
| @@ -0,0 +1,59 @@ | ||
| 1 | /** | |
| 2 | * What a run on the OpenAI-compatible runner cost (#15198): an endpoint | |
| 3 | * reports tokens, never dollars, so the forge prices them itself. | |
| 4 | * | |
| 5 | * costUsd = tokensIn × in/1e6 + tokensOut × out/1e6 | |
| 6 | * | |
| 7 | * with `in` and `out` in USD per million tokens, from `PRICES`: the providers' | |
| 8 | * list prices for standard (not batch, not priority) requests, as published by | |
| 9 | * OpenAI (openai.com/api/pricing) and Anthropic (anthropic.com/pricing). The | |
| 10 | * figures err high on purpose: a cached input token is charged as a fresh one, | |
| 11 | * a model is matched to the longest name it starts with (so `o3-mini` pays | |
| 12 | * `o3`'s price), and a model the table does not know pays `UNKNOWN_PRICE`, the | |
| 13 | * dearest row. Over-counting stops a run early; under-counting lets a cap | |
| 14 | * be passed. | |
| 15 | */ | |
| 16 | ||
| 17 | export interface Price { | |
| 18 | /** USD per million input (prompt) tokens. */ | |
| 19 | readonly inPerM: number; | |
| 20 | /** USD per million output (completion, reasoning included) tokens. */ | |
| 21 | readonly outPerM: number; | |
| 22 | } | |
| 23 | ||
| 24 | /** | |
| 25 | * Model name prefix → price. Lower-case; a vendor prefix (`openai/`) is dropped before matching. | |
| 26 | */ | |
| 27 | export const PRICES: Readonly<Record<string, Price>> = { | |
| 28 | "gpt-5": { inPerM: 1.25, outPerM: 10 }, | |
| 29 | "gpt-5-mini": { inPerM: 0.25, outPerM: 2 }, | |
| 30 | "gpt-5-nano": { inPerM: 0.05, outPerM: 0.4 }, | |
| 31 | "gpt-4.1": { inPerM: 2, outPerM: 8 }, | |
| 32 | "gpt-4.1-mini": { inPerM: 0.4, outPerM: 1.6 }, | |
| 33 | "gpt-4.1-nano": { inPerM: 0.1, outPerM: 0.4 }, | |
| 34 | "gpt-4o": { inPerM: 2.5, outPerM: 10 }, | |
| 35 | "gpt-4o-mini": { inPerM: 0.15, outPerM: 0.6 }, | |
| 36 | "o3": { inPerM: 2, outPerM: 8 }, | |
| 37 | "o4-mini": { inPerM: 1.1, outPerM: 4.4 }, | |
| 38 | "claude-opus": { inPerM: 15, outPerM: 75 }, | |
| 39 | "claude-sonnet": { inPerM: 3, outPerM: 15 }, | |
| 40 | "claude-haiku": { inPerM: 1, outPerM: 5 }, | |
| 41 | }; | |
| 42 | ||
| 43 | /** | |
| 44 | * What a model the table does not name pays: the dearest row, so an unknown model is never cheap. | |
| 45 | */ | |
| 46 | export const UNKNOWN_PRICE: Price = { inPerM: 15, outPerM: 75 }; | |
| 47 | ||
| 48 | /** `model`'s price and whether the table knew it. */ | |
| 49 | export function priceOf(model: string): { readonly price: Price; readonly known: boolean } { | |
| 50 | const name = model.trim().toLowerCase().replace(/^.*\//, ""); | |
| 51 | let best = ""; | |
| 52 | for (const prefix of Object.keys(PRICES)) if (name.startsWith(prefix) && prefix.length > best.length) best = prefix; | |
| 53 | return best === "" ? { price: UNKNOWN_PRICE, known: false } : { price: PRICES[best]!, known: true }; | |
| 54 | } | |
| 55 | ||
| 56 | /** The cost in USD of `tokensIn` and `tokensOut` at `price` (the formula above). */ | |
| 57 | export function costOf(price: Price, tokensIn: number, tokensOut: number): number { | |
| 58 | return (tokensIn * price.inPerM + tokensOut * price.outPerM) / 1_000_000; | |
| 59 | } |
src/crew/openai_runner.ts +54 −17
| @@ -7,8 +7,11 @@ | ||
| 7 | 7 | * |
| 8 | 8 | * Configuration, from the forge's own environment alone, never the run's (a |
| 9 | 9 | * granted secret is in the run's): `OPENAI_BASE_URL`, `OPENAI_API_KEY`, and |
| 10 | * the model name (`AgentRun.model`, else `OPENAI_MODEL`). The endpoint's | |
| 11 | * redirects are refused, so the key goes to the configured host or nowhere. | |
| 10 | * the model name (`AgentRun.model`, else `OPENAI_MODEL`). A run on a picked | |
| 11 | * subscription (`AgentRun.account`, #15198) spends that account instead: its | |
| 12 | * token, sent to its gateway's `/v1`; an account with no gateway is refused, | |
| 13 | * never quietly paid for with the operator's key. The endpoint's redirects | |
| 14 | * are refused, so the key goes to the configured host or nowhere. | |
| 12 | 15 | * |
| 13 | 16 | * Every file tool resolves its path against the root, follows symlinks, and |
| 14 | 17 | * refuses anything that lands outside it or passes through a link to nothing; |
| @@ -17,8 +20,10 @@ | ||
| 17 | 20 | * (`crew/openai_jail.ts`: its denied paths, its writable ones, its hosts |
| 18 | 21 | * through the forge's egress proxy), with the env the Claude runner's shell |
| 19 | 22 | * gets (`agentEnv`), and is refused when the sandbox policy is not enforced. |
| 20 | * An endpoint reports tokens, not dollars, so `costUsd` is 0 and the budget | |
| 21 | * is held by a turn limit (`MAX_TURNS`) instead. | |
| 23 | * An endpoint reports tokens, not dollars: the forge prices them | |
| 24 | * (`crew/openai_price.ts`, erring high), reports the running cost through | |
| 25 | * `onUsage` and `costUsd`, and stops a run whose cost reaches its budget with | |
| 26 | * `BUDGET_REACHED`, the reason a Claude run gives. `MAX_TURNS` still holds too. | |
| 22 | 27 | * |
| 23 | 28 | * A model names tools its own way (`glob`, `ls`, `git`, `read_file`) and |
| 24 | 29 | * leaves a path empty: a name is read case-insensitively and through |
| @@ -28,7 +33,8 @@ | ||
| 28 | 33 | |
| 29 | 34 | import { closeSync, constants, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, statSync, writeSync } from "node:fs"; |
| 30 | 35 | import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; |
| 31 | import type { AgentDetection, AgentOutcome, AgentRun, AgentRunner } from "./agent_runner.ts"; | |
| 36 | import { BUDGET_REACHED, type AgentDetection, type AgentOutcome, type AgentRun, type AgentRunner } from "./agent_runner.ts"; | |
| 37 | import { costOf, priceOf } from "./openai_price.ts"; | |
| 32 | 38 | import { stopGroup, type GroupStop } from "./stop_group.ts"; |
| 33 | 39 | import { childEnv } from "../apps/process.ts"; |
| 34 | 40 | import { isModelCredential } from "./runner_env.ts"; |
| @@ -357,31 +363,44 @@ function modelOf(model: string | undefined, get: (k: string) => string): string | ||
| 357 | 363 | return /^(opus|sonnet|haiku)$/i.test(model) && configured !== "" ? configured : model; |
| 358 | 364 | } |
| 359 | 365 | |
| 366 | /** Why a picked account without a gateway does not run here. */ | |
| 367 | export const NO_GATEWAY = "the picked subscription has no gateway, and this runner reaches a model only through an OpenAI-compatible one; it does not spend the forge's own key on another account's run"; | |
| 368 | ||
| 360 | 369 | /** |
| 361 | * The endpoint, its key and the model, from `config`: the forge's own | |
| 370 | * The endpoint, its key and the model. The model always, and the endpoint | |
| 371 | * when no account was picked, come from `config`: the forge's own | |
| 362 | 372 | * environment, never the run's. A run's env carries the secrets a brief or |
| 363 | * an issue granted, so a repository could otherwise name the URL the | |
| 364 | * operator's key is sent to. | |
| 373 | * an issue granted, so a repository could otherwise name the URL a key is | |
| 374 | * sent to. A picked account's base is the forge's (`AgentRun.account`: the | |
| 375 | * door it opened, or the gateway the account names), never a granted one. | |
| 365 | 376 | */ |
| 366 | function settings(run: AgentRun, config: Record<string, string | undefined>): { base: string; key: string; model: string } | undefined { | |
| 377 | export function settings(run: AgentRun, config: Record<string, string | undefined>): { base: string; key: string; model: string } | { refused: string } { | |
| 367 | 378 | const get = (k: string) => config[k] ?? ""; |
| 368 | const base = baseUrl(get); | |
| 369 | 379 | const model = modelOf(run.model, get); |
| 370 | if (base === "" || model === "") return undefined; | |
| 380 | const account = run.account; | |
| 381 | if (account !== undefined) { | |
| 382 | const base = baseUrl(() => account.base ?? ""); | |
| 383 | if (base === "") return { refused: NO_GATEWAY }; | |
| 384 | if (model === "") return { refused: "a model name is required" }; | |
| 385 | return { base: `${base}/v1`, key: account.token, model }; | |
| 386 | } | |
| 387 | const base = baseUrl(get); | |
| 388 | if (base === "" || model === "") return { refused: "OPENAI_BASE_URL (http/https) and a model name are required" }; | |
| 371 | 389 | return { base, key: get("OPENAI_API_KEY"), model }; |
| 372 | 390 | } |
| 373 | 391 | |
| 374 | 392 | export async function runOpenAI(run: AgentRun, config: Record<string, string | undefined> = process.env): Promise<AgentOutcome> { |
| 375 | 393 | const sessionId = run.resume ?? crypto.randomUUID(); |
| 376 | let tokensIn = 0, tokensOut = 0, turns = 0, contextTokens = 0, result = ""; | |
| 394 | let tokensIn = 0, tokensOut = 0, turns = 0, contextTokens = 0, costUsd = 0, result = ""; | |
| 377 | 395 | const done = (ok: boolean, reason: string): AgentOutcome => ({ |
| 378 | ok, sessionId, result, costUsd: 0, turns, tokensIn, tokensOut, contextTokens, contextWindow: 0, reason, | |
| 396 | ok, sessionId, result, costUsd, turns, tokensIn, tokensOut, contextTokens, contextWindow: 0, reason, | |
| 379 | 397 | }); |
| 380 | 398 | const cfg = settings(run, config); |
| 381 | if (!cfg) { | |
| 382 | result = "OPENAI_BASE_URL (http/https) and a model name are required"; | |
| 399 | if ("refused" in cfg) { | |
| 400 | result = cfg.refused; | |
| 383 | 401 | return done(false, "error"); |
| 384 | 402 | } |
| 403 | const { price } = priceOf(cfg.model); | |
| 385 | 404 | run.onSession?.(sessionId); |
| 386 | 405 | const signal = AbortSignal.any([run.signal, AbortSignal.timeout(run.timeoutMs)]); |
| 387 | 406 | const messages: Message[] = [ |
| @@ -419,7 +438,8 @@ export async function runOpenAI(run: AgentRun, config: Record<string, string | u | ||
| 419 | 438 | contextTokens = body.usage?.prompt_tokens ?? 0; |
| 420 | 439 | tokensIn += contextTokens; |
| 421 | 440 | tokensOut += body.usage?.completion_tokens ?? 0; |
| 422 | run.onUsage?.({ tokensIn, tokensOut, costUsd: 0 }); | |
| 441 | costUsd = costOf(price, tokensIn, tokensOut); | |
| 442 | run.onUsage?.({ tokensIn, tokensOut, costUsd }); | |
| 423 | 443 | messages.push({ role: "assistant", content: message.content ?? null, ...(message.tool_calls?.length ? { tool_calls: message.tool_calls } : {}) }); |
| 424 | 444 | if (message.content) { |
| 425 | 445 | result = message.content; |
| @@ -427,6 +447,13 @@ export async function runOpenAI(run: AgentRun, config: Record<string, string | u | ||
| 427 | 447 | run.onSaid?.({ kind: "text", text: message.content }); |
| 428 | 448 | } |
| 429 | 449 | if (!message.tool_calls?.length) return done(true, "completed"); |
| 450 | // Checked once the turn is paid for, as Claude Code checks its own: the | |
| 451 | // turn that crossed the cap is spent, its tool calls are not run. | |
| 452 | if (costUsd >= run.budgetUsd) { | |
| 453 | result ||= `stopped at $${costUsd.toFixed(4)}, past this run's budget of $${run.budgetUsd.toFixed(2)}`; | |
| 454 | run.log(`! budget reached: $${costUsd.toFixed(4)} of $${run.budgetUsd.toFixed(2)}`); | |
| 455 | return done(false, BUDGET_REACHED); | |
| 456 | } | |
| 430 | 457 | for (const call of message.tool_calls) { |
| 431 | 458 | let args: Record<string, unknown> = {}; |
| 432 | 459 | let bad = ""; |
| @@ -455,12 +482,22 @@ export async function runOpenAI(run: AgentRun, config: Record<string, string | u | ||
| 455 | 482 | } |
| 456 | 483 | } |
| 457 | 484 | |
| 485 | /** | |
| 486 | * The run log's first line: the model, the price its tokens are counted at, and what holds the run. | |
| 487 | */ | |
| 488 | export function headingOf(model: string): string { | |
| 489 | if (model === "") return "openai-compatible endpoint, model (none set)"; | |
| 490 | const { price, known } = priceOf(model); | |
| 491 | const rate = `$${price.inPerM}/$${price.outPerM} per million tokens in/out${known ? "" : " (a model the price table does not know: its dearest row)"}`; | |
| 492 | return `openai-compatible endpoint, model ${model}, priced at ${rate}: held to its budget, ${MAX_TURNS} turns and its deadline`; | |
| 493 | } | |
| 494 | ||
| 458 | 495 | export const openaiRunner: AgentRunner = { |
| 459 | 496 | run: (run) => runOpenAI(run), |
| 460 | 497 | // The run's env is not read: the model is the forge's to configure (`settings`). |
| 461 | 498 | heading(model) { |
| 462 | 499 | const get = (k: string) => process.env[k] ?? ""; |
| 463 | return `openai-compatible endpoint, model ${modelOf(model, get) || "(none set)"}: it reports tokens and no price, so no dollar budget applies; the run is held to ${MAX_TURNS} turns and its deadline`; | |
| 500 | return headingOf(modelOf(model, get)); | |
| 464 | 501 | }, |
| 465 | 502 | async detect(): Promise<AgentDetection> { |
| 466 | 503 | return baseUrl((k) => process.env[k] ?? "") !== "" ? { ok: true, version: "openai-compatible" } : { ok: false, error: "OPENAI_BASE_URL is not set" }; |
src/crew/runner.ts +2 −1
| @@ -211,7 +211,7 @@ import { | ||
| 211 | 211 | } from "./guard.ts"; |
| 212 | 212 | import { doorForRun, type GatewayDoor } from "./gateway_door.ts"; |
| 213 | 213 | import { passportEnv } from "./passport_env.ts"; |
| 214 | import { isGatewayAddress, layerRunEnv, ownEnv, sandboxTmpEnv } from "./runner_env.ts"; | |
| 214 | import { isGatewayAddress, layerRunEnv, ownEnv, runAccount, sandboxTmpEnv } from "./runner_env.ts"; | |
| 215 | 215 | import { cappedByLent, fundedEnv, fundingOf, lentCost, lentCreditLeft, lentSpent, refreshFundingOnce, runEnded } from "../spawn/funding.ts"; |
| 216 | 216 | |
| 217 | 217 | const log = createLog("crew"); |
| @@ -1671,6 +1671,7 @@ async function work( | ||
| 1671 | 1671 | appendSystemPrompt: systemPrompt(brief), |
| 1672 | 1672 | model: chosen.model || undefined, |
| 1673 | 1673 | budgetUsd, |
| 1674 | account: runAccount(picked, env), | |
| 1674 | 1675 | resume, |
| 1675 | 1676 | env, |
| 1676 | 1677 | sandbox: shellPolicy(), |
src/crew/runner_env.ts +13 −1
| @@ -4,7 +4,8 @@ | ||
| 4 | 4 | * `CLAUDE_CODE_*`; no other module spells those names. Plain data in, plain |
| 5 | 5 | * data out: nothing here reads the process environment on its own account. |
| 6 | 6 | */ |
| 7 | import type { SubscriptionKind } from "../core/subscriptions.ts"; | |
| 7 | import type { Pick, SubscriptionKind } from "../core/subscriptions.ts"; | |
| 8 | import type { RunAccount } from "./agent_runner.ts"; | |
| 8 | 9 | |
| 9 | 10 | type Env = Record<string, string | undefined>; |
| 10 | 11 | |
| @@ -166,3 +167,14 @@ export function layerRunEnv(layers: { | ||
| 166 | 167 | ...(layers.configDir === undefined ? {} : { CLAUDE_CONFIG_DIR: layers.configDir }), |
| 167 | 168 | }; |
| 168 | 169 | } |
| 170 | ||
| 171 | /** | |
| 172 | * The picked account as `AgentRun.account` carries it: its token, and the | |
| 173 | * gateway base from `env` (where a crew run's door has already replaced the | |
| 174 | * account's own). Undefined when nothing was picked: the forge's own credential. | |
| 175 | */ | |
| 176 | export function runAccount(picked: Pick, env: Readonly<Record<string, string>>): RunAccount | undefined { | |
| 177 | if (picked.kind !== "ready") return undefined; | |
| 178 | const base = picked.apiBase !== null && picked.apiBase !== "" ? env[GATEWAY_VAR] : undefined; | |
| 179 | return { token: picked.token, ...(base === undefined || base === "" ? {} : { base }) }; | |
| 180 | } |
test/openai-runner.test.ts +64 −5
| @@ -2,8 +2,11 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; | ||
| 2 | 2 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, symlinkSync } from "node:fs"; |
| 3 | 3 | import { tmpdir } from "node:os"; |
| 4 | 4 | import { join } from "node:path"; |
| 5 | import { currentRunner, type AgentRun } from "../src/crew/agent_runner.ts"; | |
| 6 | import { openaiRunner, runOpenAI, callTool, bashArgv, runShell, shellEnv, BODY_CAP, MAX_TOOL_CALLS, MAX_TURNS } from "../src/crew/openai_runner.ts"; | |
| 5 | import { BUDGET_REACHED, currentRunner, type AgentRun } from "../src/crew/agent_runner.ts"; | |
| 6 | import { costOf, priceOf, UNKNOWN_PRICE } from "../src/crew/openai_price.ts"; | |
| 7 | import { runAccount } from "../src/crew/runner_env.ts"; | |
| 8 | import type { Pick, Subscription } from "../src/core/subscriptions.ts"; | |
| 9 | import { openaiRunner, runOpenAI, callTool, bashArgv, runShell, shellEnv, BODY_CAP, MAX_TOOL_CALLS, MAX_TURNS, NO_GATEWAY } from "../src/crew/openai_runner.ts"; | |
| 7 | 10 | import { runGitAuth } from "../src/crew/git_auth.ts"; |
| 8 | 11 | |
| 9 | 12 | let server: ReturnType<typeof Bun.serve>; |
| @@ -51,7 +54,9 @@ test("a Write runs, Bash is refused without an enforced sandbox, and the run end | ||
| 51 | 54 | expect(readFileSync(join(cwd, "out/hello.txt"), "utf8")).toBe("hi"); |
| 52 | 55 | expect(existsSync(join(cwd, "copy.txt"))).toBe(false); |
| 53 | 56 | expect(seen.bodies[2].messages.at(-1).content).toContain("refused: Bash needs an enforced sandbox"); |
| 54 | expect(out).toMatchObject({ ok: true, result: "all done", reason: "completed", turns: 3, tokensIn: 30, tokensOut: 15, costUsd: 0 }); | |
| 57 | expect(out).toMatchObject({ ok: true, result: "all done", reason: "completed", turns: 3, tokensIn: 30, tokensOut: 15 }); | |
| 58 | // stub-model is not in the table: 30 x 15/1e6 + 15 x 75/1e6 = 0.00045 + 0.001125. | |
| 59 | expect(out.costUsd).toBeCloseTo(0.001575, 9); | |
| 55 | 60 | expect(usage).toEqual([10, 20, 30]); |
| 56 | 61 | expect(seen.auth).toBe("Bearer k-test"); |
| 57 | 62 | expect(seen.bodies[0].model).toBe("stub-model"); |
| @@ -241,12 +246,14 @@ test("a command that fails silently says so", async () => { | ||
| 241 | 246 | expect(out).toBe("[exit 3, no output on stdout or stderr]"); |
| 242 | 247 | }); |
| 243 | 248 | |
| 244 | test("the run's heading names the endpoint's model and no dollar budget; a tier word is the configured model", () => { | |
| 249 | test("the run's heading names the endpoint's model and its price; a tier word is the configured model", () => { | |
| 245 | 250 | process.env["OPENAI_MODEL"] = "gpt-oss:120b"; |
| 246 | 251 | try { |
| 247 | 252 | expect(openaiRunner.heading?.("sonnet", {})).toContain("model gpt-oss:120b"); |
| 248 | 253 | expect(openaiRunner.heading?.("local-coder", {})).toContain("model local-coder"); |
| 249 | expect(openaiRunner.heading?.(undefined, {})).toContain("no dollar budget"); | |
| 254 | expect(openaiRunner.heading?.(undefined, {})).toContain("held to its budget"); | |
| 255 | expect(openaiRunner.heading?.("gpt-4o", {})).toContain("$2.5/$10 per million tokens in/out: held to its budget"); | |
| 256 | expect(openaiRunner.heading?.("local-coder", {})).toContain("$15/$75 per million tokens in/out (a model the price table does not know"); | |
| 250 | 257 | // The run's env names no model: a granted secret is not the forge's configuration. |
| 251 | 258 | expect(openaiRunner.heading?.("sonnet", { OPENAI_MODEL: "granted" })).toContain("model gpt-oss:120b"); |
| 252 | 259 | } finally { delete process.env["OPENAI_MODEL"]; } |
| @@ -262,3 +269,55 @@ test("a shell past its deadline is killed, and its output is capped", async () = | ||
| 262 | 269 | expect(loud).toEndWith("[truncated]"); |
| 263 | 270 | expect(loud.length).toBeLessThan(31_000); |
| 264 | 271 | }); |
| 272 | ||
| 273 | test("a cap of $0.01 stops a run whose tokens cost more, with the reason a Claude run gives (#15198)", async () => { | |
| 274 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 275 | const spent: (number | undefined)[] = []; | |
| 276 | const out = await withEndpoint( | |
| 277 | () => Response.json({ choices: [{ message: { role: "assistant", content: null, tool_calls: [call("w", "Write", { file_path: "late.txt", content: "x" })] } }], usage: { prompt_tokens: 1000, completion_tokens: 1000 } }), | |
| 278 | (env) => runOpenAI(runIn(cwd, { budgetUsd: 0.01, onUsage: (s) => spent.push(s.costUsd) }), { ...env, OPENAI_API_KEY: "k" }), | |
| 279 | ); | |
| 280 | // Unknown model, $15/$75 per million: 1000 x 15/1e6 + 1000 x 75/1e6 = 0.015 + 0.075. | |
| 281 | expect(out).toMatchObject({ ok: false, reason: BUDGET_REACHED, turns: 1, tokensIn: 1000, tokensOut: 1000 }); | |
| 282 | expect(out.costUsd).toBeCloseTo(0.09, 9); | |
| 283 | expect(spent).toEqual([out.costUsd]); | |
| 284 | expect(out.result).toContain("past this run's budget of $0.01"); | |
| 285 | // The turn that crossed the cap is paid for; its tool calls are not run. | |
| 286 | expect(existsSync(join(cwd, "late.txt"))).toBe(false); | |
| 287 | }); | |
| 288 | ||
| 289 | test("a run on a picked account sends its token to its gateway, never the operator's key", async () => { | |
| 290 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 291 | let auth = null as string | null, path = ""; | |
| 292 | const srv = Bun.serve({ port: 0, fetch(req) { auth = req.headers.get("authorization"); path = new URL(req.url).pathname; return say({ role: "assistant", content: "done" }); } }); | |
| 293 | try { | |
| 294 | const account = { token: "sk-picked", base: `http://127.0.0.1:${srv.port}/door/` }; | |
| 295 | const out = await runOpenAI(runIn(cwd, { account }), { OPENAI_BASE_URL: "http://127.0.0.1:1/v1", OPENAI_API_KEY: "k-operator" }); | |
| 296 | expect(out).toMatchObject({ ok: true, reason: "completed" }); | |
| 297 | expect(auth).toBe("Bearer sk-picked"); | |
| 298 | expect(path).toBe("/door/v1/chat/completions"); | |
| 299 | } finally { srv.stop(true); } | |
| 300 | // An account with no gateway is refused, not paid for with the operator's key. | |
| 301 | const bare = await runOpenAI(runIn(cwd, { account: { token: "sk-ant-x" } }), endpoint()); | |
| 302 | expect(bare).toMatchObject({ ok: false, reason: "error", result: NO_GATEWAY, turns: 0, costUsd: 0 }); | |
| 303 | }); | |
| 304 | ||
| 305 | test("runAccount carries a picked account's token and its gateway as the run's env has it", () => { | |
| 306 | const subscription: Subscription = { | |
| 307 | id: "s1", name: "work", kind: "api_key", apiBase: "https://gw.example", | |
| 308 | restingUntil: null, restingSaid: "", lastUsedAt: null, createdBy: null, createdAt: "2026-11-21T00:00:00Z", | |
| 309 | }; | |
| 310 | const ready = (apiBase: string | null): Pick => ({ kind: "ready", subscription: { ...subscription, apiBase }, envName: "ANTHROPIC_API_KEY", token: "t", apiBase }); | |
| 311 | expect(runAccount(ready("https://gw.example"), { ANTHROPIC_BASE_URL: "http://127.0.0.1:9/s" })).toEqual({ token: "t", base: "http://127.0.0.1:9/s" }); | |
| 312 | expect(runAccount(ready(null), { ANTHROPIC_BASE_URL: "http://operator" })).toEqual({ token: "t" }); | |
| 313 | expect(runAccount({ kind: "env" }, { ANTHROPIC_BASE_URL: "http://operator" })).toBeUndefined(); | |
| 314 | }); | |
| 315 | ||
| 316 | test("the price table matches a hand calculation, the longest prefix wins, and an unknown model pays the dearest row", () => { | |
| 317 | expect(priceOf("gpt-4o-mini-2024-07-18")).toEqual({ price: { inPerM: 0.15, outPerM: 0.6 }, known: true }); | |
| 318 | expect(priceOf("openai/gpt-4o")).toEqual({ price: { inPerM: 2.5, outPerM: 10 }, known: true }); | |
| 319 | expect(priceOf("anthropic/claude-sonnet-4.5").price).toEqual({ inPerM: 3, outPerM: 15 }); | |
| 320 | expect(priceOf("gpt-oss:120b")).toEqual({ price: UNKNOWN_PRICE, known: false }); | |
| 321 | // 1,000,000 in x $2.50 + 200,000 out x $10 / 1e6 = 2.50 + 2.00. | |
| 322 | expect(costOf(priceOf("gpt-4o").price, 1_000_000, 200_000)).toBeCloseTo(4.5, 9); | |
| 323 | }); |
new above
offline