Public: anyone can read this; the crew can change it.
The crew-activity rows on an issue show tokens in, tokens out and the price per million beside each run's cost #14180
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14178-the-crew-activity-rows-on-an-issue-show → main 9 commits fork 3 threads · 3 unresolved
6 files
changed +179
−8
docs/features/README.md +1
| @@ -171,6 +171,7 @@ Read with the guide page [Tell the crew](../guide-tell-the-crew.md). | ||
| 171 | 171 | - [](desk-runs.md) |
| 172 | 172 | - [](crew-week-card.md) |
| 173 | 173 | - [](crew-week-returned.md) |
| 174 | - [](crew-row-price.md) | |
| 174 | 175 | <!-- ssh --> |
| 175 | 176 | - [](ssh-first-ask.md) |
| 176 | 177 | <!-- /ssh --> |
docs/features/crew-row-price.md added +12
| @@ -0,0 +1,12 @@ | ||
| 1 | # Tokens and price per million on each crew-activity row | |
| 2 | ||
| 3 | Each agent run listed under an issue or pull shows what it cost and how: tokens | |
| 4 | in, tokens out, the price per million and the cost, so the line can be checked | |
| 5 | by hand. A model with no declared price says "price per million unknown"; a run | |
| 6 | the ledger has not counted yet shows its cost alone. | |
| 7 | ||
| 8 | - **Web:** the "Crew activity" card on an issue or pull (`crewActivityCard`, `src/web/pages/runs.ts`), one muted line under the row title (a `Cost` fact, so it wraps instead of shifting the row on a phone): the price line, or for an unpriced model `N in · M out · price per million unknown`. Tokens print once. | |
| 9 | - **Terminal:** the same line in the issue screen's crew list only (`RunList` with `prices: true`, `src/tui/screens/runlist.ts`; other run lists are unchanged), read once when the list loads, not per frame, the line from `priceWords` (`src/core/run_price.ts`). | |
| 10 | - The cost is the owner's fact: a reader who cannot see a run's cost sees no tokens either, on both doors. | |
| 11 | - Try it: open an issue a crew member has worked on; set `FORGE_MODEL_PRICES=sonnet=3/15` to see the price. | |
| 12 | - `test/cost-tokens.test.ts` renders the card for a priced run, an unpriced run, an uncounted run and a reader without the cost, and the terminal's row. |
src/tui/screens/issue.ts +1 −1
| @@ -188,7 +188,7 @@ export function issueScreen( | ||
| 188 | 188 | /** Which issues this one waits for and holds up; a line when any. */ |
| 189 | 189 | let waits: Waits = { blockedBy: [], blocks: [] }; |
| 190 | 190 | /** What the crew has done here, newest first; a card when there is any. */ |
| 191 | const crew = new RunList(f); | |
| 191 | const crew = new RunList(f, { prices: true }); | |
| 192 | 192 | /** Which pane `j`/`k`/`enter` belong to; `tab` moves between them. */ |
| 193 | 193 | let focus: "thread" | "crew" = "thread"; |
| 194 | 194 | /** The private repository made for this sealed report, shown until we leave. */ |
src/tui/screens/runlist.ts +41 −3
| @@ -18,6 +18,8 @@ import { getIssuesByIds, type Issue } from "../../core/issues.ts"; | ||
| 18 | 18 | import { canWrite, type Repo } from "../../core/repos.ts"; |
| 19 | 19 | import type { Run, RunState } from "../../core/runs.ts"; |
| 20 | 20 | import { getUsersByIds } from "../../core/users.ts"; |
| 21 | import { getUsage } from "../../core/usage.ts"; | |
| 22 | import { priceWords } from "../../core/run_price.ts"; | |
| 21 | 23 | import { stillEnding } from "../../runs/ending.ts"; |
| 22 | 24 | import type { Ctx, Screen } from "../app/app.ts"; |
| 23 | 25 | import type { Rect, Screen as Surface } from "../term/screen.ts"; |
| @@ -85,6 +87,7 @@ export function runRows( | ||
| 85 | 87 | set: IconSet = "plain", |
| 86 | 88 | issues?: ReadonlyMap<string, Issue>, |
| 87 | 89 | ending: (runId: string) => boolean = () => false, |
| 90 | counted: (run: Run) => string = () => "", | |
| 88 | 91 | ): ListRow[] { |
| 89 | 92 | return runs.map((run) => { |
| 90 | 93 | const tone = runTone(run.state); |
| @@ -107,6 +110,7 @@ export function runRows( | ||
| 107 | 110 | meta: metaOf( |
| 108 | 111 | run.commitOid === "" ? "" : run.commitOid.slice(0, 7), |
| 109 | 112 | costText(run.costUsd), |
| 113 | counted(run), | |
| 110 | 114 | placementWords(run), |
| 111 | 115 | relTime(runWhen(run)), |
| 112 | 116 | ), |
| @@ -135,14 +139,17 @@ export class RunList { | ||
| 135 | 139 | runs: Run[] = []; |
| 136 | 140 | private names = new Map<string, { username: string }>(); |
| 137 | 141 | private issues: Map<string, Issue> | undefined; |
| 142 | /** The price line of each counted run, read once in `set`, not per frame. */ | |
| 143 | private priced = new Map<string, string>(); | |
| 138 | 144 | |
| 139 | 145 | /** |
| 140 | 146 | * `nameIssues` reads the conversation behind every run — one grouped |
| 141 | * query per list — for a list that spans a repository. | |
| 147 | * query per list — for a list that spans a repository. `prices` adds the | |
| 148 | * tokens × price line to each agent run's row: the issue's crew list only. | |
| 142 | 149 | */ |
| 143 | 150 | constructor( |
| 144 | 151 | private readonly f: Session, |
| 145 | private readonly o: { nameIssues?: boolean } = {}, | |
| 152 | private readonly o: { nameIssues?: boolean; prices?: boolean } = {}, | |
| 146 | 153 | ) {} |
| 147 | 154 | |
| 148 | 155 | set(runs: Run[]): void { |
| @@ -155,6 +162,12 @@ export class RunList { | ||
| 155 | 162 | runs.flatMap((run) => (run.issueId === null ? [] : [run.issueId])), |
| 156 | 163 | ) |
| 157 | 164 | : undefined; |
| 165 | this.priced = new Map( | |
| 166 | this.o.prices === true ? runs.flatMap((run) => { | |
| 167 | const words = this.tokenWords(run); | |
| 168 | return words === "" ? [] : [[run.id, words] as const]; | |
| 169 | }) : [], | |
| 170 | ); | |
| 158 | 171 | this.state.clamp(runs.length); |
| 159 | 172 | } |
| 160 | 173 | |
| @@ -162,6 +175,31 @@ export class RunList { | ||
| 162 | 175 | return this.runs[this.state.index]; |
| 163 | 176 | } |
| 164 | 177 | |
| 178 | /** | |
| 179 | * The run's tokens × price line, as the web's crew card shows it | |
| 180 | * (`runTokens`, `web/pages/runs.ts`): only beside a cost the viewer may | |
| 181 | * see, and nothing until the ledger has counted the run. | |
| 182 | */ | |
| 183 | private tokenWords(run: Run): string { | |
| 184 | if (run.kind !== "agent" || run.costUsd <= 0) return ""; | |
| 185 | const used = getUsage(this.f.db, run.id); | |
| 186 | return used === undefined || used.tokensIn + used.tokensOut === 0 | |
| 187 | ? "" | |
| 188 | : priceWords(used, run.model); | |
| 189 | } | |
| 190 | ||
| 191 | /** The rows as drawn: each run's line, with its price line read in `set`. */ | |
| 192 | rows(icons: IconSet): ListRow[] { | |
| 193 | return runRows( | |
| 194 | this.runs, | |
| 195 | this.names, | |
| 196 | icons, | |
| 197 | this.issues, | |
| 198 | (id) => stillEnding(this.f.db, id), | |
| 199 | (run) => this.priced.get(run.id) ?? "", | |
| 200 | ); | |
| 201 | } | |
| 202 | ||
| 165 | 203 | render( |
| 166 | 204 | s: Surface, |
| 167 | 205 | r: Rect, |
| @@ -175,7 +213,7 @@ export class RunList { | ||
| 175 | 213 | r, |
| 176 | 214 | c, |
| 177 | 215 | this.state, |
| 178 | runRows(this.runs, this.names, ctx.icons, this.issues, (id) => stillEnding(this.f.db, id)), | |
| 216 | this.rows(ctx.icons), | |
| 179 | 217 | focused, |
| 180 | 218 | ); |
| 181 | 219 | hitList(ctx, r, this.state, this.runs.length, { |
src/web/pages/runs.ts +23 −2
| @@ -173,12 +173,13 @@ export function runPriceLine( | ||
| 173 | 173 | model: string, |
| 174 | 174 | used: { tokensIn: number; tokensOut: number; costUsd: number }, |
| 175 | 175 | prices: Prices = pricesFromEnv(), |
| 176 | ids = true, | |
| 176 | 177 | ): Html { |
| 177 | 178 | const priced = runPrice(used, model, prices); |
| 178 | 179 | if (!priced.known) |
| 179 | return html`<span class="num muted" id="run-price">price per million unknown</span>`; | |
| 180 | return html`<span class="num muted"${ids ? html` id="run-price"` : null}>price per million unknown</span>`; | |
| 180 | 181 | const { price, productUsd, ledgerUsd, differs } = priced; |
| 181 | return html`<span class="num" id="run-price" | |
| 182 | return html`<span class="num"${ids ? html` id="run-price"` : null} | |
| 182 | 183 | >${tokenCount(used.tokensIn, "counted")} × $${String(price.inPerM)}/M + |
| 183 | 184 | ${tokenCount(used.tokensOut, "counted")} × $${String(price.outPerM)}/M = |
| 184 | 185 | $${productUsd.toFixed(4)}${differs |
| @@ -315,6 +316,23 @@ export function runRow( | ||
| 315 | 316 | }); |
| 316 | 317 | } |
| 317 | 318 | |
| 319 | /** | |
| 320 | * The one line a crew row carries under its title: the price line when the | |
| 321 | * model's price is declared, else the counts and "price per million unknown". | |
| 322 | * Nothing before the ledger has the run, and nothing beside a cost the viewer | |
| 323 | * may not see (`costUsd` 0), so tokens never outrun the cost's gate. | |
| 324 | */ | |
| 325 | function runPriceFact(db: Database, run: Run): Html | undefined { | |
| 326 | if (run.costUsd <= 0) return undefined; | |
| 327 | const used = getUsage(db, run.id); | |
| 328 | if (used === undefined || used.tokensIn + used.tokensOut === 0) return undefined; | |
| 329 | if (runPrice(used, run.model, pricesFromEnv()).known) return runPriceLine(run.model, used, pricesFromEnv(), false); | |
| 330 | return html`<span class="num" | |
| 331 | >${tokenCount(used.tokensIn, "counted")} in · ${tokenCount(used.tokensOut, "counted")} out · | |
| 332 | price per million unknown</span | |
| 333 | >`; | |
| 334 | } | |
| 335 | ||
| 318 | 336 | /** |
| 319 | 337 | * The agent runs about one issue or pull — the crew's side of the story, |
| 320 | 338 | * beside the conversation. Empty when nobody on the crew has touched it. |
| @@ -328,11 +346,13 @@ export function crewActivityCard(ctx: Ctx, repo: Repo, issue: Issue): Html { | ||
| 328 | 346 | if (!runs.ok || runs.value.length === 0) return html``; |
| 329 | 347 | const base = repoBase(repo); |
| 330 | 348 | const actors = actorsFor(ctx, runs.value); |
| 349 | const db = ctx.app.db; | |
| 331 | 350 | return listCard( |
| 332 | 351 | runs.value.map((run) => { |
| 333 | 352 | const who = |
| 334 | 353 | (run.actorId === null ? undefined : actors.get(run.actorId)) |
| 335 | 354 | ?.username ?? "an agent"; |
| 355 | const fact = runPriceFact(db, run); | |
| 336 | 356 | return row({ |
| 337 | 357 | href: `${base}/runs/${run.id}`, |
| 338 | 358 | lead: crewFace(who, run.state === "running" ? "working" : "still", 16, true) ?? icon("bot"), |
outdated
Principle 1/3: tokens appear twice in this span (in/out, then the × price line) and the whole thing is crammed into the trail beside the badge. One line, under the title; see the review body. |
||
| @@ -341,6 +361,7 @@ export function crewActivityCard(ctx: Ctx, repo: Repo, issue: Issue): Html { | ||
| 341 | 361 | meta: html`<span title="${run.summary}" |
| 342 | 362 | >${run.summary === "" ? "no summary yet" : run.summary}</span |
| 343 | 363 | >`, |
| 364 | ...(fact === undefined ? {} : { facts: [{ label: "Cost", value: fact }] }), | |
| 344 | 365 | trail: html`${runStateBadge(run.state, "agent")}${when( |
| 345 | 366 | run.costUsd > 0, |
| 346 | 367 | () => |
test/cost-tokens.test.ts +101 −2
| @@ -6,10 +6,12 @@ import { join } from "node:path"; | ||
| 6 | 6 | import { createApp } from "../src/app.ts"; |
| 7 | 7 | import { createRepo } from "../src/actions/repos.ts"; |
| 8 | 8 | import { createUser } from "../src/core/users.ts"; |
| 9 | import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts"; | |
| 9 | import { claimRun, createRun, getRun, finishRun, noteRunProgress } from "../src/core/runs.ts"; | |
| 10 | 10 | import { parsePrices, pricesFromEnv, priceWords, runPrice } from "../src/core/run_price.ts"; |
| 11 | 11 | import { getUsage, recordUsage } from "../src/core/usage.ts"; |
| 12 | import { runPriceLine } from "../src/web/pages/runs.ts"; | |
| 12 | import { createIssue } from "../src/core/issues.ts"; | |
| 13 | import type { Ctx } from "../src/http/context.ts"; | |
| 14 | import { crewActivityCard, runPriceLine } from "../src/web/pages/runs.ts"; | |
| 13 | 15 | |
| 14 | 16 | const prices = parsePrices("opus=15/75,sonnet=3/15,bad=x/1,also=1/2/3"); |
| 15 | 17 | |
| @@ -90,3 +92,100 @@ test("pricesFromEnv parses once per text, not per call", () => { | ||
| 90 | 92 | else process.env.FORGE_MODEL_PRICES = was; |
| 91 | 93 | } |
| 92 | 94 | }); |
| 95 | ||
| 96 | async function cardFor(model: string, unseen: boolean, counted = true, priceEnv = "sonnet=3/15") { | |
| 97 | const was = process.env.FORGE_MODEL_PRICES; | |
| 98 | process.env.FORGE_MODEL_PRICES = priceEnv; | |
| 99 | const app = createApp({ dataDir: mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "card-")), port: 0 }); | |
| 100 | try { | |
| 101 | return await cardWith(app, model, unseen, counted); | |
| 102 | } finally { | |
| 103 | app.close(); | |
| 104 | if (was === undefined) delete process.env.FORGE_MODEL_PRICES; | |
| 105 | else process.env.FORGE_MODEL_PRICES = was; | |
| 106 | } | |
| 107 | } | |
| 108 | ||
| 109 | async function cardWith(app: ReturnType<typeof createApp>, model: string, unseen: boolean, counted: boolean) { | |
| 110 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 111 | if (!u.ok) throw new Error(u.error); | |
| 112 | const made = await createRepo(app, u.value, { name: "r" }); | |
| 113 | if (!made.ok) throw new Error(String(made.error)); | |
| 114 | const issue = createIssue(app.db, { repoId: made.value.id, title: "t", authorId: u.value.id }); | |
| 115 | if (!issue.ok) throw new Error(issue.error); | |
| 116 | const run = createRun(app.db, { repoId: made.value.id, kind: "agent", issueId: issue.value.id }); | |
| 117 | claimRun(app.db, run.id); | |
| 118 | noteRunProgress(app.db, run.id, { model }); | |
| 119 | finishRun(app.db, run.id, { state: "ok", summary: "done", costUsd: 0.75 }); | |
| 120 | if (counted) | |
| 121 | recordUsage(app.db, { | |
| 122 | runId: run.id, repoId: made.value.id, issueId: issue.value.id, agentId: null, subscriptionId: null, | |
| 123 | tokensIn: 200_000, tokensOut: 10_000, costUsd: 0.75, wallMs: 1, outcome: "ok", at: new Date().toISOString(), | |
| 124 | }); | |
| 125 | const ctx = { app, auth: { user: unseen ? undefined : u.value } } as unknown as Ctx; | |
| 126 | return String(crewActivityCard(ctx, made.value, issue.value)); | |
| 127 | } | |
| 128 | ||
| 129 | test("the crew-activity row shows tokens, price per million and cost for a priced run", async () => { | |
| 130 | const card = await cardFor("sonnet", false); | |
| 131 | expect(card.split("200,000").length).toBe(2); // the counts print once | |
| 132 | expect(card).not.toContain(" in ·"); | |
| 133 | expect(card).toContain("row-facts"); | |
| 134 | expect(card).toContain("$3/M"); | |
| 135 | expect(card).toContain("$15/M"); | |
| 136 | expect(card).toContain("$0.7500"); | |
| 137 | expect(card).not.toContain('id="run-'); | |
| 138 | }); | |
| 139 | ||
| 140 | test("the crew-activity row says price per million unknown for an unpriced run", async () => { | |
| 141 | const card = await cardFor("mystery", false); | |
| 142 | expect(card.replace(/<[^>]+>/g, "").replace(/\s+/g, " ")).toContain("200k tok in · 10k tok out · price per million unknown"); | |
| 143 | expect(card.split("200,000").length).toBe(2); | |
| 144 | }); | |
| 145 | ||
| 146 | test("a run the ledger has not counted shows its cost alone", async () => { | |
| 147 | const card = await cardFor("sonnet", false, false); | |
| 148 | expect(card).toContain("$0.75"); | |
| 149 | expect(card).not.toContain(" in ·"); | |
| 150 | expect(card).not.toContain("/M"); | |
| 151 | }); | |
| 152 | ||
| 153 | test("a reader who cannot see the cost sees no tokens either", async () => { | |
| 154 | const card = await cardFor("sonnet", true); | |
| 155 | expect(card).not.toContain("200,000"); | |
| 156 | expect(card).not.toContain("/M"); | |
| 157 | }); | |
| 158 | ||
| 159 | test("the terminal's run row carries the same tokens × price line", async () => { | |
This exercises runRows only, with the words supplied by the test. Cover RunList itself (prices:true, set(), the costUsd<=0 gate, and the default without prices) so the terminal door's owner-gate is actually tested. |
||
| 160 | const { runRows } = await import("../src/tui/screens/runlist.ts"); | |
| 161 | const { db, id } = await seed("sonnet"); | |
| 162 | const run = getRun(db, id)!; | |
| 163 | const words = priceWords({ tokensIn: 200_000, tokensOut: 10_000, costUsd: 0.75 }, "sonnet", prices); | |
| 164 | const rows = runRows([run], new Map(), "plain", undefined, () => false, () => words); | |
| 165 | expect(rows[0]!.meta).toContain("200,000 in × $3/M + 10,000 out × $15/M = $0.7500"); | |
| 166 | expect(runRows([run], new Map())[0]!.meta).not.toContain("/M"); | |
| 167 | }); | |
| 168 | ||
| 169 | test("the issue's RunList reads the price line in set(); a hidden cost or a list without prices carries none", async () => { | |
| 170 | const { RunList } = await import("../src/tui/screens/runlist.ts"); | |
| 171 | const was = process.env.FORGE_MODEL_PRICES; | |
| 172 | process.env.FORGE_MODEL_PRICES = "sonnet=3/15"; | |
| 173 | try { | |
| 174 | const { db, id } = await seed("sonnet"); | |
| 175 | const run = getRun(db, id)!; | |
| 176 | const f = { db } as unknown as ConstructorParameters<typeof RunList>[0]; | |
| 177 | const priced = new RunList(f, { prices: true }); | |
| 178 | priced.set([run]); | |
| 179 | expect(priced.rows("plain")[0]!.meta).toContain("200,000 in × $3/M + 10,000 out × $15/M = $0.7500"); | |
| 180 | // A reader who cannot see the cost gets the run with costUsd 0: no tokens either. | |
| 181 | priced.set([{ ...run, costUsd: 0 }]); | |
| 182 | expect(priced.rows("plain")[0]!.meta).not.toContain("/M"); | |
| 183 | expect(priced.rows("plain")[0]!.meta).not.toContain("200,000"); | |
| 184 | const plain = new RunList(f); | |
| 185 | plain.set([run]); | |
| 186 | expect(plain.rows("plain")[0]!.meta).not.toContain("/M"); | |
| 187 | } finally { | |
| 188 | if (was === undefined) delete process.env.FORGE_MODEL_PRICES; | |
| 189 | else process.env.FORGE_MODEL_PRICES = was; | |
| 190 | } | |
| 191 | }); |
new above
offline
A DB read per run per render frame. Load usage once with the runs and look it up from a map.