Public: anyone can read this; the crew can change it.
The terminal's crew splits screen is proven by a test: an admin sees runs by model and prompt version, anyone else is not offered it #7941
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/7936-the-terminal-s-crew-splits-screen-is-pro → main 2 commits fork 4 threads · 4 unresolved
1 file
changed +160
−63
test/tui-crew-splits.test.ts +160 −63
| @@ -1,24 +1,43 @@ | ||
| 1 | /** The crew scorecard in the terminal (#2682): an admin sees the per-model and per-prompt cuts; others have no command. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 1 | /** | |
| 2 | * The crew scorecard in the terminal (#2682, #7936): an admin sees the | |
| 3 | * per-model, per-prompt, per-day and wasted cuts, each row carrying the | |
| 4 | * numbers `agentSplits`, `modelDays` and `wastedDays` answer; with no runs it | |
| 5 | * says so in one line; anyone else is neither offered the screen nor shown it. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | 8 | import { mkdtemp, rm } from "node:fs/promises"; |
| 4 | 9 | import { tmpdir } from "node:os"; |
| 5 | 10 | import { join } from "node:path"; |
| 6 | 11 | import { createApp, type App as Forge } from "../src/app.ts"; |
| 7 | import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts"; | |
| 12 | import { modelDays } from "../src/core/model_days.ts"; | |
| 13 | import { agentSplits, claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts"; | |
| 8 | 14 | import { createUser, getUserByName } from "../src/core/users.ts"; |
| 15 | import { wastedDays } from "../src/core/wasted_days.ts"; | |
| 9 | 16 | import { createForgeApp } from "../src/tui/forge.ts"; |
| 17 | import { crewSplitsScreen } from "../src/tui/screens/crewsplits.ts"; | |
| 10 | 18 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 11 | 19 | import { quietLogs } from "./helpers/globals.ts"; |
| 12 | 20 | import { settleApp } from "./helpers/tui.ts"; |
| 13 | 21 | |
| 14 | 22 | quietLogs(); |
| 15 | 23 | |
| 16 | let dataDir: string; | |
| 17 | let forge: Forge; | |
| 24 | /** A fresh forge with alice (admin), bob and smith (the agent), no runs. */ | |
| 25 | async function freshForge(): Promise<{ forge: Forge; dataDir: string }> { | |
| 26 | const dataDir = await mkdtemp(join(tmpdir(), "forge-tui-splits-")); | |
| 27 | const forge = createApp({ dataDir, port: 0 }); | |
| 28 | for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["smith", false]] as const) { | |
| 29 | const made = await createUser(forge.db, { | |
| 30 | origin: "seed", | |
| 31 | trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst, | |
| 32 | }); | |
| 33 | if (!made.ok) throw new Error(made.error); | |
| 34 | } | |
| 35 | return { forge, dataDir }; | |
| 36 | } | |
| 18 | 37 | |
| 19 | function boot(username: string) { | |
| 38 | function boot(forge: Forge, username: string) { | |
| 20 | 39 | const user = getUserByName(forge.db, username)!; |
| 21 | const { app } = createForgeApp( | |
| 40 | const { app, session } = createForgeApp( | |
| 22 | 41 | forge, |
| 23 | 42 | { sshUsername: "t", remoteAddress: "127.0.0.1", user }, |
| 24 | 43 | { terminal: headlessTerminal(140, 32), exitHooks: false }, |
| @@ -26,7 +45,10 @@ function boot(username: string) { | ||
| 26 | 45 | app.start(); |
| 27 | 46 | const key = (k: string, ctrl = false) => app.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false }); |
| 28 | 47 | return { |
| 48 | app, | |
| 49 | session, | |
| 29 | 50 | text: () => app.screen.toText(), |
| 51 | lines: () => app.screen.toText().split("\n"), | |
| 30 | 52 | palette: async (word: string) => { |
| 31 | 53 | key("k", true); |
| 32 | 54 | for (const ch of word) key(ch); |
| @@ -40,65 +62,140 @@ function boot(username: string) { | ||
| 40 | 62 | }; |
| 41 | 63 | } |
| 42 | 64 | |
| 43 | beforeAll(async () => { | |
| 44 | dataDir = await mkdtemp(join(tmpdir(), "forge-tui-splits-")); | |
| 45 | forge = createApp({ dataDir, port: 0 }); | |
| 46 | for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["smith", false]] as const) { | |
| 47 | const made = await createUser(forge.db, { | |
| 48 | origin: "seed", | |
| 49 | trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst, | |
| 50 | }); | |
| 51 | if (!made.ok) throw new Error(made.error); | |
| 52 | } | |
| 53 | const repoId = (forge.db | |
| 54 | .query<{ id: string }, []>("SELECT id FROM repo LIMIT 1") | |
| 55 | .get()?.id) ?? ""; | |
| 56 | forge.db.run("PRAGMA foreign_keys = OFF"); | |
| 57 | const agentId = getUserByName(forge.db, "smith")!.id; | |
| 58 | for (const [model, version] of [["claude-opus-5-5", "abc123"], ["", ""]] as const) { | |
| 59 | const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId }); | |
| 60 | claimRun(forge.db, run.id); | |
| 61 | noteRunProgress(forge.db, run.id, { model, promptVersion: version }); | |
| 62 | finishRun(forge.db, run.id, { state: "ok", summary: "", costUsd: 0.1 }); | |
| 63 | } | |
| 64 | }); | |
| 65 | afterAll(async () => { | |
| 66 | forge.close(); | |
| 67 | await rm(dataDir, { recursive: true, force: true }); | |
| 68 | }); | |
| 69 | ||
| 70 | test("an admin's terminal shows the cuts, with the empty key as default / not recorded", async () => { | |
| 71 | const h = boot("alice"); | |
| 65 | async function openScorecard(forge: Forge) { | |
| 66 | const h = boot(forge, "alice"); | |
| 72 | 67 | await h.palette("Crew scorecard"); |
| 73 | 68 | await h.enter(); |
| 74 | const text = h.text(); | |
| 75 | expect(text).toContain("model: claude-opus-5-5"); | |
| 76 | expect(text).toContain("model: default"); | |
| 77 | expect(text).toContain("prompt: abc123"); | |
| 78 | expect(text).toContain("prompt: not recorded"); | |
| 79 | expect(text).toContain(`day ${new Date().toISOString().slice(0, 10)}: claude-opus-5-5`); | |
| 80 | expect(text).toContain("1 runs"); | |
| 81 | h.stop(); | |
| 69 | return h; | |
| 70 | } | |
| 71 | ||
| 72 | /** The one screen line naming `title`, which must also carry each of `parts`. */ | |
| 73 | function expectRow(lines: readonly string[], title: string, ...parts: string[]) { | |
| 74 | const line = lines.find((l) => l.includes(title)); | |
| 75 | expect(line, title).toBeDefined(); | |
| 76 | for (const part of parts) expect(line, `${title} / ${part}`).toContain(part); | |
| 77 | } | |
| 78 | ||
| 79 | const today = () => new Date().toISOString().slice(0, 10); | |
| 80 | ||
| 81 | describe("with no agent runs", () => { | |
| 82 | let forge: Forge; | |
| 83 | let dataDir: string; | |
| 84 | beforeAll(async () => ({ forge, dataDir } = await freshForge())); | |
| 85 | afterAll(async () => { | |
| 86 | forge.close(); | |
| 87 | await rm(dataDir, { recursive: true, force: true }); | |
| 88 | }); | |
| 89 | ||
| 90 | test("the screen says so in one line, not an empty table", async () => { | |
| 91 | const h = await openScorecard(forge); | |
| 92 | const lines = h.lines(); | |
| 93 | expect(lines.filter((l) => l.includes("No agent runs yet.")).length).toBe(1); | |
| 94 | expect(lines.some((l) => /\b(model|prompt|day|wasted) .*runs\b/.test(l))).toBe(false); | |
| 95 | h.stop(); | |
| 96 | }); | |
| 82 | 97 | }); |
| 83 | 98 | |
| 84 | test("an admin's terminal lists the wasted runs per day", async () => { | |
| 85 | const repoId = forge.db.query<{ id: string }, []>("SELECT id FROM repo LIMIT 1").get()?.id ?? ""; | |
| 86 | const agentId = getUserByName(forge.db, "smith")!.id; | |
| 87 | const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId }); | |
| 88 | claimRun(forge.db, run.id); | |
| 89 | finishRun(forge.db, run.id, { state: "ok", summary: "no changes pushed", costUsd: 0.4 }); | |
| 90 | const h = boot("alice"); | |
| 91 | await h.palette("Crew scorecard"); | |
| 92 | await h.enter(); | |
| 93 | expect(h.text()).toContain(`wasted ${new Date().toISOString().slice(0, 10)}`); | |
| 94 | expect(h.text()).toContain("1 of 3 runs"); | |
| 95 | expect(h.text()).toContain("$0.40 spent"); | |
| 96 | h.stop(); | |
| 97 | }); | |
| 99 | describe("over seeded runs", () => { | |
| 100 | let forge: Forge; | |
| 101 | let dataDir: string; | |
| 102 | beforeAll(async () => { | |
| 103 | ({ forge, dataDir } = await freshForge()); | |
| 104 | const repoId = forge.db.query<{ id: string }, []>("SELECT id FROM repo LIMIT 1").get()?.id ?? ""; | |
| 105 | // A fresh forge may have no repo, so the runs hang off repo "" — off for | |
| 106 | // the seed only, and put back as it was so the tests read a normal forge. | |
| 107 | const fk = forge.db.query<{ foreign_keys: number }, []>("PRAGMA foreign_keys").get()?.foreign_keys ?? 1; | |
| 108 | forge.db.run("PRAGMA foreign_keys = OFF"); | |
| 109 | const agentId = getUserByName(forge.db, "smith")!.id; | |
| 110 | const seeds = [ | |
| 111 | { model: "claude-opus-5-5", version: "v1", state: "ok", summary: "", cost: 0.1 }, | |
| 112 | { model: "claude-opus-5-5", version: "v2", state: "failed", summary: "", cost: 0.2 }, | |
| 113 | { model: "claude-sonnet-5-5", version: "v1", state: "ok", summary: "no changes pushed", cost: 0.4 }, | |
| 114 | { model: "", version: "", state: "ok", summary: "", cost: 0.1 }, | |
| 115 | ] as const; | |
| 116 | for (const s of seeds) { | |
| 117 | const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId }); | |
| 118 | claimRun(forge.db, run.id); | |
| 119 | noteRunProgress(forge.db, run.id, { model: s.model, promptVersion: s.version }); | |
| 120 | finishRun(forge.db, run.id, { state: s.state, summary: s.summary, costUsd: s.cost }); | |
| 121 | } | |
| 122 | forge.db.run(`PRAGMA foreign_keys = ${fk ? "ON" : "OFF"}`); | |
| 123 | }); | |
| 124 | afterAll(async () => { | |
| 125 | forge.close(); | |
| 126 | await rm(dataDir, { recursive: true, force: true }); | |
| 127 | }); | |
| 98 | 128 | |
| 99 | test("a non-admin has no command", async () => { | |
| 100 | const h = boot("bob"); | |
| 101 | await h.palette("Crew scorecard"); | |
| 102 | expect(h.text()).not.toContain("agent runs per model and per prompt version"); | |
| 103 | h.stop(); | |
| 129 | test("an admin sees each split with the numbers the core answers", async () => { | |
| 130 | const byModel = agentSplits(forge.db, "model"); | |
| 131 | const byPrompt = agentSplits(forge.db, "prompt_version"); | |
| 132 | const days = modelDays(forge.db); | |
| 133 | const wasted = wastedDays(forge.db); | |
| 134 | // The seed is what the core sees. | |
| 135 | expect(byModel.map((r) => [r.key, r.runs, r.ok, r.failed]).sort()).toEqual([ | |
| 136 | ["", 1, 1, 0], | |
| 137 | ["claude-opus-5-5", 2, 1, 1], | |
| 138 | ["claude-sonnet-5-5", 1, 1, 0], | |
| 139 | ]); | |
| 140 | expect(byPrompt.map((r) => [r.key, r.runs, r.ok, r.failed]).sort()).toEqual([ | |
| 141 | ["", 1, 1, 0], | |
| 142 | ["v1", 2, 2, 0], | |
| 143 | ["v2", 1, 0, 1], | |
| 144 | ]); | |
| 145 | expect(days.length).toBe(3); | |
| 146 | expect(wasted.length).toBe(1); | |
| 147 | ||
| 148 | const h = await openScorecard(forge); | |
| 149 | const lines = h.lines(); | |
| 150 | expect(lines.join("\n")).toContain("Agent runs per model, then per prompt version"); | |
| 151 | for (const [label, blank, splits] of [["model", "default", byModel], ["prompt", "not recorded", byPrompt]] as const) | |
| 152 | for (const r of splits) | |
| 153 | expectRow( | |
| 154 | lines, | |
| 155 | `${label}: ${r.key === "" ? blank : r.key}`, | |
| 156 | `${r.runs} runs`, | |
| 157 | `${r.ok} landed`, | |
| 158 | `${r.failed} failed`, | |
| 159 | `$${r.spentUsd.toFixed(2)} spent`, | |
| 160 | `$${(r.spentUsd / r.runs).toFixed(2)} per run`, | |
| 161 | ); | |
| 162 | for (const d of days) | |
| 163 | expectRow(lines, `day ${d.day}: ${d.model === "" ? "default" : d.model}`, `${d.runs} runs`, `${d.landed} landed`, `$${d.spentUsd.toFixed(2)} spent`); | |
| 164 | for (const w of wasted) | |
| 165 | expectRow(lines, `wasted ${w.day}`, `${w.wasted} of ${w.runs} runs`, `$${w.spentUsd.toFixed(2)} spent`); | |
| 166 | ||
| 167 | // And the numbers themselves, so a wrong core cannot agree with a wrong screen. | |
| 168 | expectRow(lines, "model: claude-opus-5-5", "2 runs", "1 landed", "1 failed", "$0.30 spent", "$0.15 per run"); | |
| 169 | expectRow(lines, "model: claude-sonnet-5-5", "1 runs", "1 landed", "0 failed", "$0.40 spent", "$0.40 per run"); | |
| 170 | expectRow(lines, "model: default", "1 runs", "1 landed", "0 failed", "$0.10 spent", "$0.10 per run"); | |
| 171 | expectRow(lines, "prompt: v1", "2 runs", "2 landed", "0 failed", "$0.50 spent", "$0.25 per run"); | |
| 172 | expectRow(lines, "prompt: v2", "1 runs", "0 landed", "1 failed", "$0.20 spent", "$0.20 per run"); | |
| 173 | expectRow(lines, "prompt: not recorded", "1 runs", "1 landed", "0 failed", "$0.10 spent", "$0.10 per run"); | |
| 174 | // A day's "landed" counts merged pulls, and the seed merges none. | |
| 175 | expectRow(lines, `day ${today()}: claude-opus-5-5`, "2 runs", "0 landed", "$0.30 spent"); | |
| 176 | expectRow(lines, `wasted ${today()}`, "1 of 4 runs", "$0.40 spent"); | |
| 177 | h.stop(); | |
| 178 | }); | |
| 179 | ||
| 180 | test("a non-admin's palette does not offer the scorecard; an admin's does", async () => { | |
| 181 | const admin = boot(forge, "alice"); | |
| 182 | await admin.palette("Crew scorecard"); | |
| 183 | expect(admin.text()).toContain("agent runs per model and per prompt version"); | |
| 184 | admin.stop(); | |
| 185 | const h = boot(forge, "bob"); | |
| 186 | await h.palette("Crew scorecard"); | |
| 187 | expect(h.text()).not.toContain("agent runs per model and per prompt version"); | |
| 188 | h.stop(); | |
| 189 | }); | |
| 190 | ||
| 191 | test("the screen itself refuses a non-admin: the gate line, and no rows", async () => { | |
| 192 | const h = boot(forge, "bob"); | |
| 193 | h.app.ctx.push(crewSplitsScreen(h.session, h.app.ctx)); | |
| 194 | await settleApp(h.app); | |
| 195 | const text = h.text(); | |
| 196 | expect(text).toContain("The crew scorecard is read by a forge admin."); | |
| 197 | expect(text).not.toMatch(/\b(model|prompt|day|wasted) .*runs\b/); | |
| 198 | expect(text).not.toContain("claude-opus-5-5"); | |
| 199 | h.stop(); | |
| 200 | }); | |
| 104 | 201 | }); |
new above
offline