Public: anyone can read this; the crew can change it.

The terminal's crew splits screen is proven by a test: an admin sees runs by model and prompt version, anyone else is not offered it #7941

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/7936-the-terminal-s-crew-splits-screen-is-pro → main 2 commits fork 4 threads · 4 unresolved

  • Merged. scout/7936-the-terminal-s-crew-splits-screen-is-pro was merged into main as 1f40c9f.

Review

Sign in to review this change.
1 file changed +160 −63
test/tui-crew-splits.test.ts +160 −63
@@ -1,24 +1,43 @@
1 /** The crew scorecard in the terminal (#2682): an admin sees the per-model and per-prompt cuts; others have no command. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
1 /**
2 * The crew scorecard in the terminal (#2682, #7936): an admin sees the
3 * per-model, per-prompt, per-day and wasted cuts, each row carrying the
4 * numbers `agentSplits`, `modelDays` and `wastedDays` answer; with no runs it
5 * says so in one line; anyone else is neither offered the screen nor shown it.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
38 import { mkdtemp, rm } from "node:fs/promises";
49 import { tmpdir } from "node:os";
510 import { join } from "node:path";
611 import { createApp, type App as Forge } from "../src/app.ts";
7 import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts";
12 import { modelDays } from "../src/core/model_days.ts";
13 import { agentSplits, claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts";
814 import { createUser, getUserByName } from "../src/core/users.ts";
15 import { wastedDays } from "../src/core/wasted_days.ts";
916 import { createForgeApp } from "../src/tui/forge.ts";
17 import { crewSplitsScreen } from "../src/tui/screens/crewsplits.ts";
1018 import { headlessTerminal } from "../src/tui/term/terminal.ts";
1119 import { quietLogs } from "./helpers/globals.ts";
1220 import { settleApp } from "./helpers/tui.ts";
1321
1422 quietLogs();
1523
16 let dataDir: string;
17 let forge: Forge;
24 /** A fresh forge with alice (admin), bob and smith (the agent), no runs. */
25 async function freshForge(): Promise<{ forge: Forge; dataDir: string }> {
26 const dataDir = await mkdtemp(join(tmpdir(), "forge-tui-splits-"));
27 const forge = createApp({ dataDir, port: 0 });
28 for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["smith", false]] as const) {
29 const made = await createUser(forge.db, {
30 origin: "seed",
31 trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst,
32 });
33 if (!made.ok) throw new Error(made.error);
34 }
35 return { forge, dataDir };
36 }
1837
19 function boot(username: string) {
38 function boot(forge: Forge, username: string) {
2039 const user = getUserByName(forge.db, username)!;
21 const { app } = createForgeApp(
40 const { app, session } = createForgeApp(
2241 forge,
2342 { sshUsername: "t", remoteAddress: "127.0.0.1", user },
2443 { terminal: headlessTerminal(140, 32), exitHooks: false },
@@ -26,7 +45,10 @@ function boot(username: string) {
2645 app.start();
2746 const key = (k: string, ctrl = false) => app.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
2847 return {
48 app,
49 session,
2950 text: () => app.screen.toText(),
51 lines: () => app.screen.toText().split("\n"),
3052 palette: async (word: string) => {
3153 key("k", true);
3254 for (const ch of word) key(ch);
@@ -40,65 +62,140 @@ function boot(username: string) {
4062 };
4163 }
4264
43 beforeAll(async () => {
44 dataDir = await mkdtemp(join(tmpdir(), "forge-tui-splits-"));
45 forge = createApp({ dataDir, port: 0 });
46 for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["smith", false]] as const) {
47 const made = await createUser(forge.db, {
48 origin: "seed",
49 trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst,
50 });
51 if (!made.ok) throw new Error(made.error);
52 }
53 const repoId = (forge.db
54 .query<{ id: string }, []>("SELECT id FROM repo LIMIT 1")
55 .get()?.id) ?? "";
56 forge.db.run("PRAGMA foreign_keys = OFF");
57 const agentId = getUserByName(forge.db, "smith")!.id;
58 for (const [model, version] of [["claude-opus-5-5", "abc123"], ["", ""]] as const) {
59 const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId });
60 claimRun(forge.db, run.id);
61 noteRunProgress(forge.db, run.id, { model, promptVersion: version });
62 finishRun(forge.db, run.id, { state: "ok", summary: "", costUsd: 0.1 });
63 }
64 });
65 afterAll(async () => {
66 forge.close();
67 await rm(dataDir, { recursive: true, force: true });
68 });
69
70 test("an admin's terminal shows the cuts, with the empty key as default / not recorded", async () => {
71 const h = boot("alice");
65 async function openScorecard(forge: Forge) {
66 const h = boot(forge, "alice");
7267 await h.palette("Crew scorecard");
7368 await h.enter();
74 const text = h.text();
75 expect(text).toContain("model: claude-opus-5-5");
76 expect(text).toContain("model: default");
77 expect(text).toContain("prompt: abc123");
78 expect(text).toContain("prompt: not recorded");
79 expect(text).toContain(`day ${new Date().toISOString().slice(0, 10)}: claude-opus-5-5`);
80 expect(text).toContain("1 runs");
81 h.stop();
69 return h;
70 }
71
72 /** The one screen line naming `title`, which must also carry each of `parts`. */
73 function expectRow(lines: readonly string[], title: string, ...parts: string[]) {
74 const line = lines.find((l) => l.includes(title));
75 expect(line, title).toBeDefined();
76 for (const part of parts) expect(line, `${title} / ${part}`).toContain(part);
77 }
78
79 const today = () => new Date().toISOString().slice(0, 10);
80
81 describe("with no agent runs", () => {
82 let forge: Forge;
83 let dataDir: string;
84 beforeAll(async () => ({ forge, dataDir } = await freshForge()));
85 afterAll(async () => {
86 forge.close();
87 await rm(dataDir, { recursive: true, force: true });
88 });
89
90 test("the screen says so in one line, not an empty table", async () => {
91 const h = await openScorecard(forge);
92 const lines = h.lines();
93 expect(lines.filter((l) => l.includes("No agent runs yet.")).length).toBe(1);
94 expect(lines.some((l) => /\b(model|prompt|day|wasted) .*runs\b/.test(l))).toBe(false);
95 h.stop();
96 });
8297 });
8398
84 test("an admin's terminal lists the wasted runs per day", async () => {
85 const repoId = forge.db.query<{ id: string }, []>("SELECT id FROM repo LIMIT 1").get()?.id ?? "";
86 const agentId = getUserByName(forge.db, "smith")!.id;
87 const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId });
88 claimRun(forge.db, run.id);
89 finishRun(forge.db, run.id, { state: "ok", summary: "no changes pushed", costUsd: 0.4 });
90 const h = boot("alice");
91 await h.palette("Crew scorecard");
92 await h.enter();
93 expect(h.text()).toContain(`wasted ${new Date().toISOString().slice(0, 10)}`);
94 expect(h.text()).toContain("1 of 3 runs");
95 expect(h.text()).toContain("$0.40 spent");
96 h.stop();
97 });
99 describe("over seeded runs", () => {
100 let forge: Forge;
101 let dataDir: string;
102 beforeAll(async () => {
103 ({ forge, dataDir } = await freshForge());
104 const repoId = forge.db.query<{ id: string }, []>("SELECT id FROM repo LIMIT 1").get()?.id ?? "";
105 // A fresh forge may have no repo, so the runs hang off repo "" — off for
106 // the seed only, and put back as it was so the tests read a normal forge.
107 const fk = forge.db.query<{ foreign_keys: number }, []>("PRAGMA foreign_keys").get()?.foreign_keys ?? 1;
108 forge.db.run("PRAGMA foreign_keys = OFF");
109 const agentId = getUserByName(forge.db, "smith")!.id;
110 const seeds = [
111 { model: "claude-opus-5-5", version: "v1", state: "ok", summary: "", cost: 0.1 },
112 { model: "claude-opus-5-5", version: "v2", state: "failed", summary: "", cost: 0.2 },
113 { model: "claude-sonnet-5-5", version: "v1", state: "ok", summary: "no changes pushed", cost: 0.4 },
114 { model: "", version: "", state: "ok", summary: "", cost: 0.1 },
115 ] as const;
116 for (const s of seeds) {
117 const run = createRun(forge.db, { repoId, kind: "agent", actorId: agentId });
118 claimRun(forge.db, run.id);
119 noteRunProgress(forge.db, run.id, { model: s.model, promptVersion: s.version });
120 finishRun(forge.db, run.id, { state: s.state, summary: s.summary, costUsd: s.cost });
121 }
122 forge.db.run(`PRAGMA foreign_keys = ${fk ? "ON" : "OFF"}`);
123 });
124 afterAll(async () => {
125 forge.close();
126 await rm(dataDir, { recursive: true, force: true });
127 });
98128
99 test("a non-admin has no command", async () => {
100 const h = boot("bob");
101 await h.palette("Crew scorecard");
102 expect(h.text()).not.toContain("agent runs per model and per prompt version");
103 h.stop();
129 test("an admin sees each split with the numbers the core answers", async () => {
130 const byModel = agentSplits(forge.db, "model");
131 const byPrompt = agentSplits(forge.db, "prompt_version");
132 const days = modelDays(forge.db);
133 const wasted = wastedDays(forge.db);
134 // The seed is what the core sees.
135 expect(byModel.map((r) => [r.key, r.runs, r.ok, r.failed]).sort()).toEqual([
136 ["", 1, 1, 0],
137 ["claude-opus-5-5", 2, 1, 1],
138 ["claude-sonnet-5-5", 1, 1, 0],
139 ]);
140 expect(byPrompt.map((r) => [r.key, r.runs, r.ok, r.failed]).sort()).toEqual([
141 ["", 1, 1, 0],
142 ["v1", 2, 2, 0],
143 ["v2", 1, 0, 1],
144 ]);
145 expect(days.length).toBe(3);
146 expect(wasted.length).toBe(1);
147
148 const h = await openScorecard(forge);
149 const lines = h.lines();
150 expect(lines.join("\n")).toContain("Agent runs per model, then per prompt version");
151 for (const [label, blank, splits] of [["model", "default", byModel], ["prompt", "not recorded", byPrompt]] as const)
152 for (const r of splits)
153 expectRow(
154 lines,
155 `${label}: ${r.key === "" ? blank : r.key}`,
156 `${r.runs} runs`,
157 `${r.ok} landed`,
158 `${r.failed} failed`,
159 `$${r.spentUsd.toFixed(2)} spent`,
160 `$${(r.spentUsd / r.runs).toFixed(2)} per run`,
161 );
162 for (const d of days)
163 expectRow(lines, `day ${d.day}: ${d.model === "" ? "default" : d.model}`, `${d.runs} runs`, `${d.landed} landed`, `$${d.spentUsd.toFixed(2)} spent`);
164 for (const w of wasted)
165 expectRow(lines, `wasted ${w.day}`, `${w.wasted} of ${w.runs} runs`, `$${w.spentUsd.toFixed(2)} spent`);
166
167 // And the numbers themselves, so a wrong core cannot agree with a wrong screen.
168 expectRow(lines, "model: claude-opus-5-5", "2 runs", "1 landed", "1 failed", "$0.30 spent", "$0.15 per run");
169 expectRow(lines, "model: claude-sonnet-5-5", "1 runs", "1 landed", "0 failed", "$0.40 spent", "$0.40 per run");
170 expectRow(lines, "model: default", "1 runs", "1 landed", "0 failed", "$0.10 spent", "$0.10 per run");
171 expectRow(lines, "prompt: v1", "2 runs", "2 landed", "0 failed", "$0.50 spent", "$0.25 per run");
172 expectRow(lines, "prompt: v2", "1 runs", "0 landed", "1 failed", "$0.20 spent", "$0.20 per run");
173 expectRow(lines, "prompt: not recorded", "1 runs", "1 landed", "0 failed", "$0.10 spent", "$0.10 per run");
174 // A day's "landed" counts merged pulls, and the seed merges none.
175 expectRow(lines, `day ${today()}: claude-opus-5-5`, "2 runs", "0 landed", "$0.30 spent");
176 expectRow(lines, `wasted ${today()}`, "1 of 4 runs", "$0.40 spent");
177 h.stop();
178 });
179
180 test("a non-admin's palette does not offer the scorecard; an admin's does", async () => {
181 const admin = boot(forge, "alice");
182 await admin.palette("Crew scorecard");
183 expect(admin.text()).toContain("agent runs per model and per prompt version");
184 admin.stop();
185 const h = boot(forge, "bob");
186 await h.palette("Crew scorecard");
187 expect(h.text()).not.toContain("agent runs per model and per prompt version");
188 h.stop();
189 });
190
191 test("the screen itself refuses a non-admin: the gate line, and no rows", async () => {
192 const h = boot(forge, "bob");
193 h.app.ctx.push(crewSplitsScreen(h.session, h.app.ctx));
194 await settleApp(h.app);
195 const text = h.text();
196 expect(text).toContain("The crew scorecard is read by a forge admin.");
197 expect(text).not.toMatch(/\b(model|prompt|day|wasted) .*runs\b/);
198 expect(text).not.toContain("claude-opus-5-5");
199 h.stop();
200 });
104201 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs