Public: anyone can read this; the crew can change it.
Each crew run records the model and prompt version it ran on, and the scorecard splits by them #2686
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2682-each-crew-run-records-the-model-and-prom → main 2 commits fork 1 thread · 1 unresolved
10 files
changed +392
−2
README.md +9
| @@ -787,6 +787,15 @@ off: no autonomy tick, no dispatch, no spend; checks still run and everything | ||
| 787 | 787 | can still be read. The boot log and the studio's crew card say so. Set |
| 788 | 788 | `FORGE_CREW_IN_PREVIEW=1` to run it anyway. |
| 789 | 789 | |
| 790 | **Per model, per prompt version.** Every agent run records the model it ran on | |
| 791 | and a prompt version: a short hash of the agent's brief, its crew role and the | |
| 792 | standing system-prompt text (not the issue or branch, so two runs on one brief | |
| 793 | share a version and editing the brief starts a new one). Settings → Crew shows | |
| 794 | the scorecard cut both ways ("Per model", "Per prompt version": runs, landed, | |
| 795 | failed, spent, per run), and the terminal's `Crew scorecard` command shows the | |
| 796 | same to an admin. Runs from before this was recorded show as *default* / | |
| 797 | *not recorded*; they are not backfilled. | |
| 798 | ||
| 790 | 799 | **Tell the crew from any page.** In a repository you may direct, the web palette |
| 791 | 800 | (`:` in the browser) has a *Tell the crew* row. With the script it opens the thread's own |
| 792 | 801 | comment box over the page you are on, without leaving it: `enter` sends (shift+ |
src/core/migrations/20261019130000-run-model-prompt.ts added +7
| @@ -0,0 +1,7 @@ | ||
| 1 | // Each agent run records the model it ran on and a version of the prompt it | |
| 2 | // was given (a hash of the brief and the system prompt), so the crew | |
| 3 | // scorecard can read per model and per prompt version (#2682). | |
| 4 | export default ` | |
| 5 | ALTER TABLE run ADD COLUMN model TEXT NOT NULL DEFAULT ''; | |
| 6 | ALTER TABLE run ADD COLUMN prompt_version TEXT NOT NULL DEFAULT ''; | |
| 7 | `; |
src/core/runs.ts +44 −1
| @@ -372,6 +372,9 @@ export function noteRunProgress( | ||
| 372 | 372 | /** Where a check ran, and the node when a cluster Job did. */ |
| 373 | 373 | placement?: string | undefined; |
| 374 | 374 | node?: string | undefined; |
| 375 | /** The model an agent run ran on (empty: the runner's default), and its prompt version. */ | |
| 376 | model?: string | undefined; | |
| 377 | promptVersion?: string | undefined; | |
| 375 | 378 | }, |
| 376 | 379 | ): void { |
| 377 | 380 | db.query( |
| @@ -380,7 +383,9 @@ export function noteRunProgress( | ||
| 380 | 383 | sandbox_enforced = COALESCE(?6, sandbox_enforced), |
| 381 | 384 | sandbox_note = COALESCE(?7, sandbox_note), |
| 382 | 385 | placement = COALESCE(?8, placement), |
| 383 | node = COALESCE(?9, node) | |
| 386 | node = COALESCE(?9, node), | |
| 387 | model = COALESCE(?10, model), | |
| 388 | prompt_version = COALESCE(?11, prompt_version) | |
| 384 | 389 | WHERE id = ?1`, |
| 385 | 390 | ).run( |
| 386 | 391 | runId, |
| @@ -392,6 +397,8 @@ export function noteRunProgress( | ||
| 392 | 397 | progress.sandbox?.note.slice(0, 300) ?? null, |
| 393 | 398 | progress.placement?.slice(0, 20) ?? null, |
| 394 | 399 | progress.node?.slice(0, 253) ?? null, |
| 400 | progress.model?.slice(0, 100) ?? null, | |
| 401 | progress.promptVersion?.slice(0, 40) ?? null, | |
| 395 | 402 | ); |
| 396 | 403 | } |
| 397 | 404 | |
| @@ -937,6 +944,42 @@ export function agentTotals(db: Database): Map<string, AgentTotals> { | ||
| 937 | 944 | ); |
| 938 | 945 | } |
| 939 | 946 | |
| 947 | /** Agent runs of one model or one prompt version, in the scorecard's terms. */ | |
| 948 | export interface SplitTotals { | |
| 949 | /** The model or the prompt version; empty for runs from before it was recorded. */ | |
| 950 | readonly key: string; | |
| 951 | readonly runs: number; | |
| 952 | readonly ok: number; | |
| 953 | readonly failed: number; | |
| 954 | readonly spentUsd: number; | |
| 955 | } | |
| 956 | ||
| 957 | /** Agent runs grouped by what they ran on: `model` or `prompt_version`. */ | |
| 958 | export function agentSplits( | |
| 959 | db: Database, | |
| 960 | by: "model" | "prompt_version", | |
| 961 | ): SplitTotals[] { | |
| 962 | // `by` is one of two literals, never user text. | |
| 963 | return db | |
| 964 | .query< | |
| 965 | { key: string; runs: number; ok: number; failed: number; spent: number }, | |
| 966 | [] | |
| 967 | >( | |
| 968 | `SELECT ${by} AS key, COUNT(*) AS runs, SUM(state = 'ok') AS ok, | |
| 969 | SUM(state = 'failed') AS failed, COALESCE(SUM(cost_usd), 0) AS spent | |
| 970 | FROM run WHERE kind = 'agent' AND actor_id IS NOT NULL | |
| 971 | GROUP BY ${by} ORDER BY runs DESC, key`, | |
| 972 | ) | |
| 973 | .all() | |
| 974 | .map((r) => ({ | |
| 975 | key: r.key, | |
| 976 | runs: r.runs, | |
| 977 | ok: r.ok, | |
| 978 | failed: r.failed, | |
| 979 | spentUsd: r.spent, | |
| 980 | })); | |
| 981 | } | |
| 982 | ||
| 940 | 983 | /** One repository's agent runs over a window: what "this week" is made of. */ |
| 941 | 984 | export interface AgentRunTotals { |
| 942 | 985 | readonly runs: number; |
src/crew/brief.ts +14
| @@ -69,6 +69,20 @@ export const SANDBOX_LINES = [ | ||
| 69 | 69 | "It cannot: reach any host but the forge and the package registry, write outside the clone and `$TMPDIR`, read the forge's data or the machine's, bind a unix socket, or signal a process started in an earlier command. Never leave the sandbox: it cannot be asked to. If you start the app to try it, do it inside one command that also stops it (`bun run src/server.ts & pid=$!; sleep 2; curl …; kill $pid`). Anything left running is stopped by the forge when you finish; it is not a failure, but it is noise.", |
| 70 | 70 | ]; |
| 71 | 71 | |
| 72 | /** | |
| 73 | * A short name for the prompt an agent ran on: a hash of its brief, its | |
| 74 | * role and the standing system-prompt text. Per-run facts (issue, branch) | |
| 75 | * are left out, so two runs on one brief share a version and an edited | |
| 76 | * brief starts a new one. | |
| 77 | */ | |
| 78 | export function promptVersion(agent: Agent, role?: CrewRole): string { | |
| 79 | const hasher = new Bun.CryptoHasher("sha256"); | |
| 80 | hasher.update( | |
| 81 | JSON.stringify([agent.brief.trim(), role ?? "", SANDBOX_LINES]), | |
| 82 | ); | |
| 83 | return hasher.digest("hex").slice(0, 10); | |
| 84 | } | |
| 85 | ||
| 72 | 86 | export function systemPrompt(input: BriefInput): string { |
| 73 | 87 | const { agent, repo } = input; |
| 74 | 88 | const lines = [ |
src/crew/runner.ts +3
| @@ -85,6 +85,7 @@ import { appendRunLog, readRunLog } from "../runs/log.ts"; | ||
| 85 | 85 | import { |
| 86 | 86 | branchFor, |
| 87 | 87 | issueOfBranch, |
| 88 | promptVersion, | |
| 88 | 89 | systemPrompt, |
| 89 | 90 | taskPrompt, |
| 90 | 91 | wrapUpPrompt, |
| @@ -1186,6 +1187,8 @@ async function work( | ||
| 1186 | 1187 | `claude: budget $${budget.toFixed(2)}, $${reserve.toFixed(2)} of it kept for the wrap-up${brief.resuming ? `, resuming ${previous?.sessionId}` : ""}`, |
| 1187 | 1188 | ); |
| 1188 | 1189 | noteRunProgress(app.db, run.id, { |
| 1190 | model: agent.model, | |
| 1191 | promptVersion: promptVersion(agent, brief.role), | |
| 1189 | 1192 | sandbox: { |
| 1190 | 1193 | enforced: sandbox.enforced, |
| 1191 | 1194 | note: options.sandboxNote ?? "", |
src/tui/forge.ts +9
| @@ -46,6 +46,7 @@ import { mayManageFlags } from "../core/flags.ts"; | ||
| 46 | 46 | import { accessScreen } from "./screens/access.ts"; |
| 47 | 47 | import { flagsScreen } from "./screens/flags.ts"; |
| 48 | 48 | import { cloudScreen } from "./screens/cloud.ts"; |
| 49 | import { crewSplitsScreen } from "./screens/crewsplits.ts"; | |
| 49 | 50 | import { usageScreen } from "./screens/usage.ts"; |
| 50 | 51 | import { canScreen } from "./screens/can.ts"; |
| 51 | 52 | import { peopleScreen } from "./screens/people.ts"; |
| @@ -366,6 +367,14 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] { | ||
| 366 | 367 | hint: "every route, unused ones first", |
| 367 | 368 | run: () => ctx.push(usageScreen(session, ctx)), |
| 368 | 369 | }); |
| 370 | // Settings → Crew's scorecard by model and prompt version: an admin (#2682). | |
| 371 | if (session.user?.isAdmin === true) | |
| 372 | commands.push({ | |
| 373 | id: "crew-scorecard", | |
| 374 | title: "Crew scorecard", | |
| 375 | hint: "agent runs per model and per prompt version", | |
| 376 | run: () => ctx.push(crewSplitsScreen(session, ctx)), | |
| 377 | }); | |
| 369 | 378 | // Proposal tokens for a repository you write, as the API and MCP have them (#2148). |
| 370 | 379 | if (session.user !== undefined) |
| 371 | 380 | commands.push({ |
src/tui/screens/crewsplits.ts added +54
| @@ -0,0 +1,54 @@ | ||
| 1 | /** | |
| 2 | * The crew scorecard in the terminal, a forge admin's (#2682): agent runs | |
| 3 | * cut by the model they ran on and by prompt version, as Settings → Crew | |
| 4 | * has them (`agentSplits`). The gate is the page's: an admin. | |
| 5 | */ | |
| 6 | ||
| 7 | import { agentSplits, type SplitTotals } from "../../core/runs.ts"; | |
| 8 | import type { Ctx, Screen } from "../app/app.ts"; | |
| 9 | import { binding } from "../app/keys.ts"; | |
| 10 | import { style } from "../term/style.ts"; | |
| 11 | import { truncate } from "../term/width.ts"; | |
| 12 | import type { Session } from "../session.ts"; | |
| 13 | import { fill, len, split } from "../ui/layout.ts"; | |
| 14 | import { ListState, drawList, type ListRow } from "./common.ts"; | |
| 15 | ||
| 16 | export function splitRows( | |
| 17 | label: string, | |
| 18 | blank: string, | |
| 19 | splits: readonly SplitTotals[], | |
| 20 | ): ListRow[] { | |
| 21 | return splits.map((r) => ({ | |
| 22 | title: `${label}: ${r.key === "" ? blank : r.key}`, | |
| 23 | secondary: `${r.runs} runs`, | |
| 24 | meta: `${r.ok} landed ${r.failed} failed $${r.spentUsd.toFixed(2)} spent $${(r.spentUsd / r.runs).toFixed(2)} per run`, | |
| 25 | })); | |
| 26 | } | |
| 27 | ||
| 28 | export function crewSplitsScreen(f: Session, ctx: Ctx): Screen { | |
| 29 | const state = new ListState(); | |
| 30 | const allowed = f.user?.isAdmin === true; | |
| 31 | // Read once when the screen opens, not per frame. | |
| 32 | const rows: ListRow[] = allowed | |
| 33 | ? [ | |
| 34 | ...splitRows("model", "default", agentSplits(f.db, "model")), | |
| 35 | ...splitRows("prompt", "not recorded", agentSplits(f.db, "prompt_version")), | |
| 36 | ] | |
| 37 | : []; | |
| 38 | return { | |
| 39 | title: "Crew scorecard", | |
| 40 | keys: [binding(["j", "k"], "move cursor")], | |
| 41 | render(area, s) { | |
| 42 | const c = ctx.theme.c; | |
| 43 | const [head, list] = split(area, "v", [len(1), fill(1)]); | |
| 44 | const text = !allowed | |
| 45 | ? "The crew scorecard is read by a forge admin." | |
| 46 | : rows.length === 0 | |
| 47 | ? "No agent runs yet." | |
| 48 | : "Agent runs per model, then per prompt version."; | |
| 49 | s.text(head!.x, head!.y, truncate(text, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w); | |
| 50 | drawList(s, list!, c, state, rows); | |
| 51 | }, | |
| 52 | key: (e) => state.key(e, rows.length), | |
| 53 | }; | |
| 54 | } |
src/web/pages/crew.ts +57 −1
| @@ -18,7 +18,12 @@ import { | ||
| 18 | 18 | MODEL_MAX, |
| 19 | 19 | type Agent, |
| 20 | 20 | } from "../../core/agents.ts"; |
| 21 | import { agentTotals, type AgentTotals } from "../../core/runs.ts"; | |
| 21 | import { | |
| 22 | agentSplits, | |
| 23 | agentTotals, | |
| 24 | type AgentTotals, | |
| 25 | type SplitTotals, | |
| 26 | } from "../../core/runs.ts"; | |
| 22 | 27 | import { crewFor } from "../../crew/dispatch.ts"; |
| 23 | 28 | import { MODELS } from "../../crew/models.ts"; |
| 24 | 29 | import type { Ctx } from "../../http/context.ts"; |
| @@ -376,6 +381,18 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html { | ||
| 376 | 381 | </div>`, |
| 377 | 382 | { pad: false }, |
| 378 | 383 | )} |
| 384 | ${splitsCard( | |
| 385 | "Per model", | |
| 386 | "Model", | |
| 387 | agentSplits(ctx.app.db, "model"), | |
| 388 | () => html`<span class="muted">default</span>`, | |
| 389 | )} | |
| 390 | ${splitsCard( | |
| 391 | "Per prompt version", | |
| 392 | "Prompt version", | |
| 393 | agentSplits(ctx.app.db, "prompt_version"), | |
| 394 | () => html`<span class="muted">not recorded</span>`, | |
| 395 | )} | |
| 379 | 396 | ${card( |
| 380 | 397 | html`<form method="post" action="/settings/crew"> |
| 381 | 398 | ${uiField({ |
| @@ -405,6 +422,45 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html { | ||
| 405 | 422 | )}`; |
| 406 | 423 | } |
| 407 | 424 | |
| 425 | /** The scorecard cut by one thing the runs ran on; nothing before a first run. */ | |
| 426 | function splitsCard( | |
| 427 | title: string, | |
| 428 | column: string, | |
| 429 | rows: readonly SplitTotals[], | |
| 430 | blank: (key: string) => Html, | |
| 431 | ): Html { | |
| 432 | if (rows.length === 0) return html``; | |
| 433 | return card( | |
| 434 | html`<div class="table-scroll"> | |
| 435 | <table class="table"> | |
| 436 | <thead> | |
| 437 | <tr> | |
| 438 | <th>${column}</th> | |
| 439 | <th>Runs</th> | |
| 440 | <th>Landed</th> | |
| 441 | <th>Failed</th> | |
| 442 | <th>Spent</th> | |
| 443 | <th>Per run</th> | |
| 444 | </tr> | |
| 445 | </thead> | |
| 446 | <tbody> | |
| 447 | ${rows.map( | |
| 448 | (r) => html`<tr> | |
| 449 | <td class="mono">${r.key === "" ? blank(r.key) : r.key}</td> | |
| 450 | <td class="num">${r.runs}</td> | |
| 451 | <td class="num">${r.ok}</td> | |
| 452 | <td class="num">${r.failed}</td> | |
| 453 | <td class="num">$${r.spentUsd.toFixed(2)}</td> | |
| 454 | <td class="num">$${(r.spentUsd / r.runs).toFixed(2)}</td> | |
| 455 | </tr>`, | |
| 456 | )} | |
| 457 | </tbody> | |
| 458 | </table> | |
| 459 | </div>`, | |
| 460 | { pad: false, head: html`<h3>${title}</h3>` }, | |
| 461 | ); | |
| 462 | } | |
| 463 | ||
| 408 | 464 | const EMPTY: AgentValues = { |
| 409 | 465 | username: "", |
| 410 | 466 | displayName: "", |
test/crew-splits.test.ts added +108
| @@ -0,0 +1,108 @@ | ||
| 1 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp, type App } from "../src/app.ts"; | |
| 6 | import { | |
| 7 | agentSplits, | |
| 8 | claimRun, | |
| 9 | createRun, | |
| 10 | finishRun, | |
| 11 | noteRunProgress, | |
| 12 | } from "../src/core/runs.ts"; | |
| 13 | import { promptVersion } from "../src/crew/brief.ts"; | |
| 14 | import { getUserByName } from "../src/core/users.ts"; | |
| 15 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 16 | import { adminCookie } from "./helpers/admin.ts"; | |
| 17 | ||
| 18 | describe("the scorecard splits by model and prompt version", () => { | |
| 19 | let dataDir: string; | |
| 20 | let app: App; | |
| 21 | let servers: Servers; | |
| 22 | let cookie = ""; | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | dataDir = await mkdtemp(join(tmpdir(), "forge-splits-")); | |
| 26 | app = createApp({ dataDir, port: 0 }); | |
| 27 | servers = await serve(app); | |
| 28 | cookie = await adminCookie(app, { | |
| 29 | username: "ada", | |
| 30 | email: "[email protected]", | |
| 31 | password: "analytical-engine", | |
| 32 | }); | |
| 33 | const headers = { cookie, "content-type": "application/json" }; | |
| 34 | const repo = (await fetch(`${servers.url}/api/v1/repos`, { | |
| 35 | method: "POST", | |
| 36 | headers, | |
| 37 | body: JSON.stringify({ name: "hello", visibility: "public" }), | |
| 38 | }).then((r) => r.json())) as { id: string }; | |
| 39 | await fetch(`${servers.url}/api/v1/agents`, { | |
| 40 | method: "POST", | |
| 41 | headers, | |
| 42 | body: JSON.stringify({ username: "smith" }), | |
| 43 | }); | |
| 44 | const agentId = getUserByName(app.db, "smith")!.id; | |
| 45 | const agent = (brief: string) => | |
| 46 | ({ brief, model: "" }) as Parameters<typeof promptVersion>[0]; | |
| 47 | for (const [brief, model] of [ | |
| 48 | ["Be terse.", "claude-sonnet-5-5"], | |
| 49 | ["Be terse.", "claude-sonnet-5-5"], | |
| 50 | ["Be thorough.", "claude-opus-5-5"], | |
| 51 | ] as const) { | |
| 52 | const run = createRun(app.db, { | |
| 53 | repoId: repo.id, | |
| 54 | kind: "agent", | |
| 55 | actorId: agentId, | |
| 56 | }); | |
| 57 | claimRun(app.db, run.id); | |
| 58 | noteRunProgress(app.db, run.id, { | |
| 59 | model, | |
| 60 | promptVersion: promptVersion(agent(brief)), | |
| 61 | }); | |
| 62 | finishRun(app.db, run.id, { state: "ok", summary: "", costUsd: 0.1 }); | |
| 63 | } | |
| 64 | // A run from before the recording: both keys empty. | |
| 65 | const old = createRun(app.db, { | |
| 66 | repoId: repo.id, | |
| 67 | kind: "agent", | |
| 68 | actorId: agentId, | |
| 69 | }); | |
| 70 | claimRun(app.db, old.id); | |
| 71 | finishRun(app.db, old.id, { state: "ok", summary: "", costUsd: 0.1 }); | |
| 72 | }); | |
| 73 | ||
| 74 | afterAll(async () => { | |
| 75 | servers.stop(); | |
| 76 | app.close(); | |
| 77 | await rm(dataDir, { recursive: true, force: true }); | |
| 78 | }); | |
| 79 | ||
| 80 | test("two briefs are two prompt versions; one brief is one", () => { | |
| 81 | const a = { brief: "x", model: "" } as Parameters<typeof promptVersion>[0]; | |
| 82 | expect(promptVersion(a)).toBe(promptVersion({ ...a })); | |
| 83 | expect(promptVersion(a)).not.toBe(promptVersion({ ...a, brief: "y" })); | |
| 84 | const versions = agentSplits(app.db, "prompt_version"); | |
| 85 | expect(versions.map((v) => v.runs)).toEqual([2, 1, 1]); | |
| 86 | }); | |
| 87 | ||
| 88 | test("runs split by model", () => { | |
| 89 | const models = agentSplits(app.db, "model"); | |
| 90 | expect(models.map((m) => [m.key, m.runs])).toEqual([ | |
| 91 | ["claude-sonnet-5-5", 2], | |
| 92 | ["", 1], | |
| 93 | ["claude-opus-5-5", 1], | |
| 94 | ]); | |
| 95 | }); | |
| 96 | ||
| 97 | test("the crew page shows both cuts", async () => { | |
| 98 | const page = await fetch(`${servers.url}/settings/crew`, { | |
| 99 | headers: { cookie }, | |
| 100 | }).then((r) => r.text()); | |
| 101 | expect(page).toContain("Per model"); | |
| 102 | expect(page).toContain("Per prompt version"); | |
| 103 | expect(page).toContain("claude-opus-5-5"); | |
| 104 | // The shipped state of every existing run: the empty key. | |
| 105 | expect(page).toContain(`<td class="mono"><span class="muted">default</span></td>`); | |
| 106 | expect(page).toContain(`<td class="mono"><span class="muted">not recorded</span></td>`); | |
| 107 | }); | |
| 108 | }); |
test/tui-crew-splits.test.ts added +87
| @@ -0,0 +1,87 @@ | ||
| 1 | /** The crew scorecard in the terminal (#2682): an admin sees the per-model and per-prompt cuts; others have no command. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App as Forge } from "../src/app.ts"; | |
| 7 | import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts"; | |
| 8 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 9 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 10 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 11 | import { quietLogs } from "./helpers/globals.ts"; | |
| 12 | import { settleApp } from "./helpers/tui.ts"; | |
| 13 | ||
| 14 | quietLogs(); | |
| 15 | ||
| 16 | let dataDir: string; | |
| 17 | let forge: Forge; | |
| 18 | ||
| 19 | function boot(username: string) { | |
| 20 | const user = getUserByName(forge.db, username)!; | |
| 21 | const { app } = createForgeApp( | |
| 22 | forge, | |
| 23 | { sshUsername: "t", remoteAddress: "127.0.0.1", user }, | |
| 24 | { terminal: headlessTerminal(140, 32), exitHooks: false }, | |
| 25 | ); | |
| 26 | app.start(); | |
| 27 | const key = (k: string, ctrl = false) => app.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false }); | |
| 28 | return { | |
| 29 | text: () => app.screen.toText(), | |
| 30 | palette: async (word: string) => { | |
| 31 | key("k", true); | |
| 32 | for (const ch of word) key(ch); | |
| 33 | await settleApp(app); | |
| 34 | }, | |
| 35 | enter: async () => { | |
| 36 | key("enter"); | |
| 37 | await settleApp(app); | |
| 38 | }, | |
| 39 | stop: () => app.stop(), | |
| 40 | }; | |
| 41 | } | |
| 42 | ||
| 43 | beforeAll(async () => { | |
| 44 | dataDir = await mkdtemp(join(tmpdir(), "forge-tui-splits-")); | |
| 45 | forge = createApp({ dataDir, port: 0 }); | |
| 46 | for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["smith", false]] as const) { | |
| 47 | const made = await createUser(forge.db, { | |
| 48 | origin: "seed", | |
| 49 | trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst, | |
| 50 | }); | |
| 51 | if (!made.ok) throw new Error(made.error); | |
| 52 | } | |
| 53 | const repoId = (forge.db | |
| 54 | .query<{ id: string }, []>("SELECT id FROM repo LIMIT 1") | |
| 55 | .get()?.id) ?? ""; | |
| 56 | forge.db.run("PRAGMA foreign_keys = OFF"); | |
| 57 | const agentId = getUserByName(forge.db, "smith")!.id; | |
| 58 | for (const [model, version] of [["claude-opus-5-5", "abc123"], ["", ""]] as const) { | |
| 59 | const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId }); | |
| 60 | claimRun(forge.db, run.id); | |
| 61 | noteRunProgress(forge.db, run.id, { model, promptVersion: version }); | |
| 62 | finishRun(forge.db, run.id, { state: "ok", summary: "", costUsd: 0.1 }); | |
| 63 | } | |
| 64 | }); | |
| 65 | afterAll(async () => { | |
| 66 | forge.close(); | |
| 67 | await rm(dataDir, { recursive: true, force: true }); | |
| 68 | }); | |
| 69 | ||
| 70 | test("an admin's terminal shows the cuts, with the empty key as default / not recorded", async () => { | |
| 71 | const h = boot("alice"); | |
| 72 | await h.palette("Crew scorecard"); | |
| 73 | await h.enter(); | |
| 74 | const text = h.text(); | |
| 75 | expect(text).toContain("model: claude-opus-5-5"); | |
| 76 | expect(text).toContain("model: default"); | |
| 77 | expect(text).toContain("prompt: abc123"); | |
| 78 | expect(text).toContain("prompt: not recorded"); | |
| 79 | h.stop(); | |
| 80 | }); | |
| 81 | ||
| 82 | test("a non-admin has no command", async () => { | |
| 83 | const h = boot("bob"); | |
| 84 | await h.palette("Crew scorecard"); | |
| 85 | expect(h.text()).not.toContain("agent runs per model and per prompt version"); | |
| 86 | h.stop(); | |
| 87 | }); |
new above
offline
Empty-key rows render as muted text only; on a phone this table scrolls horizontally inside table-scroll, which is fine, but add a test for this branch.