trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns12g rEventsFlows

The scorecard splits each crew member's figures by the model and prompt that ran #3189

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/3169-the-scorecard-splits-each-crew-member-s → main 2 commits fork

  • Merged. scout/3169-the-scorecard-splits-each-crew-member-s was merged into main as 4fad6bf.

Review

Sign in to review this change.
6 files changed +226 −43
README.md +9 −2
@@ -29,7 +29,13 @@ reverts, time to land — is on Settings → Crew (`/settings/crew`) and in the
2929 terminal's palette as "Crew scorecard per agent"; both are a forge admin's only.
3030 Landed, review rounds and red checks stand as the last 30 days (pulls opened
3131 since then) beside all time, so tuning an agent shows as movement; reverts and
32 time to land are all time.
32 time to land are all time. Under each agent, a sub-row per model and prompt
33 version its pulls ran on (the agent's latest run on the pull decides) carries
34 the same figures, so a change of model or brief shows against the one it
35 replaced; pulls opened before runs recorded either read "unrecorded", and are
36 counted, never dropped. The sub-rows list every pairing ever used; a pairing
37 unused in the last 30 days shows dashes there. The terminal shows them as
38 indented `↳` rows.
3339
3440 Git hosting, code browsing, and a JSON API in one fast Bun process, with
3541 **zero runtime dependencies** — the `package.json` dependency list is empty and
@@ -931,7 +937,8 @@ standing system-prompt text (not the issue or branch, so two runs on one brief
931937 share a version and editing the brief starts a new one). Settings → Crew shows
932938 the scorecard cut both ways ("Per model", "Per prompt version": runs, landed,
933939 failed, spent, per run), and the terminal's `Crew scorecard` command shows the
934 same to an admin. Runs from before this was recorded show as *default* /
940 same to an admin. The scorecard of pulls landed, review rounds and red checks
941 is cut the same way as sub-rows under each agent (see above). Runs from before this was recorded show as *default* /
935942 *not recorded*; they are not backfilled.
936943
937944 **Tell the crew from any page.** In a repository you may direct, the web palette
src/core/scorecard.ts +69 −9
@@ -49,11 +49,59 @@ export function recentWindow(now: number = Date.now()): ScorecardWindow {
4949 return { since: new Date(now - RECENT_DAYS * 86_400_000) };
5050 }
5151
52 /** A scorecard for the pulls an agent opened on one model and prompt version. */
53 export interface SplitScorecard extends Scorecard {
54 /** The model its latest run on the pull ran on; empty when unrecorded. */
55 readonly model: string;
56 /** The prompt version of that run; empty when unrecorded. */
57 readonly promptVersion: string;
58 }
59
60 /** "model · prompt version"; a half the runs never recorded reads "unrecorded". */
61 export function splitLabel(p: SplitScorecard): string {
62 const half = (v: string) => (v === "" ? "unrecorded" : v);
63 return `${half(p.model)} · ${half(p.promptVersion)}`;
64 }
65
5266 /** Each agent's scorecard, keyed by its user id: one grouped query. */
5367 export function crewScorecards(
5468 db: Database,
5569 window: ScorecardWindow = {},
5670 ): Map<string, Scorecard> {
71 return new Map(scorecardRows(db, window, false).map((r) => [r.authorId, r.card]));
72 }
73
74 /**
75 * Each agent's scorecard cut by (model, prompt version), keyed by user id.
76 * A pull belongs to the pairing of the agent's latest run on it; a pull with
77 * no recorded run (opened before the migration) is the pairing ('', ''),
78 * never dropped. Largest pairing first.
79 */
80 export function crewScorecardSplits(
81 db: Database,
82 window: ScorecardWindow = {},
83 ): Map<string, SplitScorecard[]> {
84 const out = new Map<string, SplitScorecard[]>();
85 for (const r of scorecardRows(db, window, true)) {
86 const list = out.get(r.authorId) ?? [];
87 list.push({ ...r.card, model: r.model, promptVersion: r.promptVersion });
88 out.set(r.authorId, list);
89 }
90 for (const list of out.values())
91 list.sort(
92 (a, b) =>
93 b.opened - a.opened ||
94 a.model.localeCompare(b.model) ||
95 a.promptVersion.localeCompare(b.promptVersion),
96 );
97 return out;
98 }
99
100 function scorecardRows(
101 db: Database,
102 window: ScorecardWindow,
103 split: boolean,
104 ): { authorId: string; model: string; promptVersion: string; card: Scorecard }[] {
57105 // created_at is stored as ISO-8601 UTC, so the string compare is the time compare.
58106 // An invalid Date throws here rather than widen the window to all time.
59107 const since = window.since === undefined ? null : window.since.toISOString();
@@ -61,6 +109,8 @@ export function crewScorecards(
61109 .query<
62110 {
63111 author_id: string;
112 model: string;
113 prompt_version: string;
64114 opened: number;
65115 landed: number;
66116 rounds: number;
@@ -72,6 +122,8 @@ export function crewScorecards(
72122 [string | null]
73123 >(
74124 `SELECT issue.author_id AS author_id,
125 ${split ? latest("model") : "''"} AS model,
126 ${split ? latest("prompt_version") : "''"} AS prompt_version,
75127 COUNT(*) AS opened,
76128 SUM(pull.merged_at IS NOT NULL) AS landed,
77129 COALESCE(SUM(CASE WHEN pull.merged_at IS NOT NULL THEN
@@ -95,15 +147,16 @@ export function crewScorecards(
95147 JOIN pull ON pull.issue_id = issue.id
96148 JOIN user ON user.id = issue.author_id AND user.is_agent = 1
97149 WHERE ?1 IS NULL OR issue.created_at >= ?1
98 GROUP BY issue.author_id`,
150 GROUP BY issue.author_id${split ? ", model, prompt_version" : ""}`,
99151 )
100152 .all(since);
101 return new Map(
102 rows.map((r) => {
153 return rows.map((r) => {
103154 const checks = r.checks_ok + r.checks_failed;
104 return [
105 r.author_id,
106 {
155 return {
156 authorId: r.author_id,
157 model: r.model,
158 promptVersion: r.prompt_version,
159 card: {
107160 opened: r.opened,
108161 landed: r.landed,
109162 reviewRounds: r.landed === 0 ? null : r.rounds / r.landed,
@@ -114,9 +167,16 @@ export function crewScorecards(
114167 ? null
115168 : Math.max(0, Math.round(r.mean_land_days * 86_400_000)),
116169 },
117 ];
118 }),
119 );
170 };
171 });
172 }
173
174 /** The column of the agent's latest run on the pull; '' when it has none. `column` is a literal. */
175 function latest(column: "model" | "prompt_version"): string {
176 return `COALESCE((SELECT run.${column} FROM run
177 WHERE run.issue_id = issue.id AND run.kind = 'agent'
178 AND run.actor_id = issue.author_id
179 ORDER BY run.id DESC LIMIT 1), '')`;
120180 }
121181
122182 /** A time to land in the largest unit that keeps it readable: both doors say it the same way. */
src/tui/screens/scorecard.ts +42 −15
@@ -2,17 +2,21 @@
22 * The crew's scorecard in the terminal, a forge admin's (#2677): each
33 * agent's pulls landed of opened, review rounds and red checks over the
44 * last `RECENT_DAYS` days beside all time, then reverts and time to land
5 * over all time, as Settings → Crew shows them (`crewScorecards`). The gate
5 * over all time, as Settings → Crew shows them (`crewScorecards`), with a sub-row per
6 * model · prompt version (`crewScorecardSplits`). The gate
67 * is the page's: a forge admin; anyone else sees nothing.
78 */
89
910 import { listAllAgents } from "../../actions/crew.ts";
1011 import {
1112 crewScorecards,
13 crewScorecardSplits,
1214 RECENT_DAYS,
1315 recentWindow,
1416 spanText,
17 splitLabel,
1518 type Scorecard,
19 type SplitScorecard,
1620 } from "../../core/scorecard.ts";
1721 import type { Agent } from "../../core/agents.ts";
1822 import type { Ctx, Screen } from "../app/app.ts";
@@ -30,31 +34,52 @@ const rounds = (s: Scorecard | undefined) =>
3034 const red = (s: Scorecard | undefined) =>
3135 s?.redCheckRate == null ? "—" : `${Math.round(100 * s.redCheckRate)}%`;
3236
37 /** The figures of one scorecard, as a list row's second and third lines. */
38 function figures(
39 title: string,
40 r: Scorecard | undefined,
41 s: Scorecard,
42 ): ListRow {
43 const d = `${RECENT_DAYS}d`;
44 return {
45 title,
46 secondary: `landed ${d} ${landed(r)} · all ${landed(s)}`,
47 meta: [
48 `review rounds ${d} ${rounds(r)} · all ${rounds(s)}`,
49 `red checks ${d} ${red(r)} · all ${red(s)}`,
50 `reverts ${s.reverts}`,
51 `time to land ${s.meanLandMs === null ? "—" : spanText(s.meanLandMs)}`,
52 ].join(" "),
53 };
54 }
55
3356 /**
3457 * One row per agent; one with no pull yet says so rather than a row of
3558 * zeros. Each windowed figure reads "<recent>d … · all …", the page's order.
59 * Under an agent, one indented row per model · prompt version its pulls ran
60 * on, with the same figures ("unrecorded" for pulls from before runs
61 * recorded them).
3662 */
3763 export function scorecardRows(
3864 agents: readonly Agent[],
3965 cards: ReadonlyMap<string, Scorecard>,
4066 recent: ReadonlyMap<string, Scorecard>,
67 splits: ReadonlyMap<string, readonly SplitScorecard[]> = new Map(),
68 recentSplits: ReadonlyMap<string, readonly SplitScorecard[]> = new Map(),
4169 ): ListRow[] {
42 const d = `${RECENT_DAYS}d`;
43 return agents.map((agent) => {
70 return agents.flatMap((agent) => {
4471 const s = cards.get(agent.user.id);
4572 if (s === undefined)
46 return { title: agent.user.username, secondary: "no pull yet" };
47 const r = recent.get(agent.user.id);
48 return {
49 title: agent.user.username,
50 secondary: `landed ${d} ${landed(r)} · all ${landed(s)}`,
51 meta: [
52 `review rounds ${d} ${rounds(r)} · all ${rounds(s)}`,
53 `red checks ${d} ${red(r)} · all ${red(s)}`,
54 `reverts ${s.reverts}`,
55 `time to land ${s.meanLandMs === null ? "—" : spanText(s.meanLandMs)}`,
56 ].join(" "),
57 };
73 return [{ title: agent.user.username, secondary: "no pull yet" }];
74 const lately = new Map(
75 (recentSplits.get(agent.user.id) ?? []).map((p) => [splitLabel(p), p]),
76 );
77 return [
78 figures(agent.user.username, recent.get(agent.user.id), s),
79 ...(splits.get(agent.user.id) ?? []).map((p) =>
80 figures(` ↳ ${splitLabel(p)}`, lately.get(splitLabel(p)), p),
81 ),
82 ];
5883 });
5984 }
6085
@@ -69,6 +94,8 @@ export function scorecardScreen(f: Session, ctx: Ctx): Screen {
6994 listed.value,
7095 crewScorecards(f.db),
7196 crewScorecards(f.db, recentWindow()),
97 crewScorecardSplits(f.db),
98 crewScorecardSplits(f.db, recentWindow()),
7299 )
73100 : [];
74101 return {
src/web/pages/crew.ts +46 −16
@@ -26,10 +26,13 @@ import {
2626 } from "../../core/runs.ts";
2727 import {
2828 crewScorecards,
29 crewScorecardSplits,
30 splitLabel,
2931 RECENT_DAYS,
3032 recentWindow,
3133 spanText,
3234 type Scorecard,
35 type SplitScorecard,
3336 } from "../../core/scorecard.ts";
3437 import { crewFor } from "../../crew/dispatch.ts";
3538 import { MODELS } from "../../crew/models.ts";
@@ -229,19 +232,25 @@ function totalsCells(t: AgentTotals | undefined): Html {
229232 </td>`;
230233 }
231234
235 const pairKey = (p: SplitScorecard) => `${p.model}\0${p.promptVersion}`;
236
232237 /**
233238 * The scorecard: how each agent's pulls fare once opened — so the owner
234239 * can tune who does what. Landed, review rounds and red checks stand as the
235240 * last 30 days beside all time, so a change to an agent shows as movement.
236241 * What they cost is the table above's ("Per landed"), not repeated here. An
237242 * agent with no pull yet is a row of dashes, never a row of misleading
238 * zeros. The agent's name stays in view while a phone scrolls the figures
243 * zeros. Under each agent, one sub-row per model and prompt version its pulls
244 * ran on, with the same figures; pulls from before runs recorded either read
245 * "unrecorded". The agent's name stays in view while a phone scrolls the figures
239246 * sideways.
240247 */
241248 function scorecardCard(
242249 agents: readonly Agent[],
243250 cards: ReadonlyMap<string, Scorecard>,
244251 recent: ReadonlyMap<string, Scorecard>,
252 splits: ReadonlyMap<string, readonly SplitScorecard[]>,
253 recentSplits: ReadonlyMap<string, readonly SplitScorecard[]>,
245254 ): Html {
246255 const dash = html`<td class="num muted">—</td>`;
247256 const lately = `${RECENT_DAYS} days`;
@@ -262,6 +271,14 @@ function scorecardCard(
262271 s?.redCheckRate == null
263272 ? dash
264273 : html`<td class="num">${Math.round(100 * s.redCheckRate)}%</td>`;
274 const figures = (r: Scorecard | undefined, s: Scorecard | undefined) =>
275 html`${landed(r)}${landed(s)}${rounds(r)}${rounds(s)}${red(r)}${red(s)}
276 ${s === undefined
277 ? html`${dash}${dash}`
278 : html`<td class="num">${s.reverts}</td>
279 ${s.meanLandMs === null
280 ? dash
281 : html`<td class="num">${spanText(s.meanLandMs)}</td>`}`}`;
265282 return card(
266283 html`<div class="table-scroll">
267284 <table class="table" id="crew-scorecard" data-sticky-lead>
@@ -282,21 +299,26 @@ function scorecardCard(
282299 ${agents.map((agent) => {
283300 const s = cards.get(agent.user.id);
284301 const r = recent.get(agent.user.id);
302 const pairs = splits.get(agent.user.id) ?? [];
303 const lately = new Map(
304 (recentSplits.get(agent.user.id) ?? []).map((p) => [pairKey(p), p]),
305 );
285306 return html`<tr>
286 <td>
287 ${userLink(agent.user.username, {
288 avatar: false,
289 mark: markOf(agent.user),
290 })}
291 </td>
292 ${landed(r)}${landed(s)}${rounds(r)}${rounds(s)}${red(r)}${red(s)}
293 ${s === undefined
294 ? html`${dash}${dash}`
295 : html`<td class="num">${s.reverts}</td>
296 ${s.meanLandMs === null
297 ? dash
298 : html`<td class="num">${spanText(s.meanLandMs)}</td>`}`}
299 </tr>`;
307 <td>
308 ${userLink(agent.user.username, {
309 avatar: false,
310 mark: markOf(agent.user),
311 })}
312 </td>
313 ${figures(r, s)}
314 </tr>
315 ${pairs.map((p) => {
316 const label = splitLabel(p);
317 return html`<tr data-sub>
318 <td class="muted small mono" title="${label}">${label}</td>
319 ${figures(lately.get(pairKey(p)), p)}
320 </tr>`;
321 })}`;
300322 })}
301323 </tbody>
302324 </table>
@@ -381,6 +403,8 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html {
381403 const totals = agentTotals(ctx.app.db);
382404 const scorecards = crewScorecards(ctx.app.db);
383405 const recentScorecards = crewScorecards(ctx.app.db, recentWindow());
406 const splitScorecards = crewScorecardSplits(ctx.app.db);
407 const recentSplitScorecards = crewScorecardSplits(ctx.app.db, recentWindow());
384408 // Where the money goes today: every app running on its own, its spend
385409 // against its ceiling. The ceilings are the one control on the drain, and
386410 // this is the page where the crew is wielded, so they are read here.
@@ -479,7 +503,13 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html {
479503 { pad: false },
480504 )}
481505 ${when(agents.length > 0, () =>
482 scorecardCard(agents, scorecards, recentScorecards),
506 scorecardCard(
507 agents,
508 scorecards,
509 recentScorecards,
510 splitScorecards,
511 recentSplitScorecards,
512 ),
483513 )}
484514 ${splitsCard(
485515 "Per model",
src/web/style/components.ts +8
@@ -401,6 +401,14 @@ a.row:hover .row-title { color: var(--accent); }
401401 position: sticky; left: 0; z-index: 1; background: var(--card);
402402 }
403403
404 /* A sub-row (data-sub) is a split of the row above: indented under it, its
405 label cut with an ellipsis so a long "model · prompt" cannot widen the
406 sticky lead cell past a phone's width. */
407 .table tr[data-sub] td:first-child {
408 padding-left: 24px; max-width: 14rem;
409 overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
410 }
411
404412 /* ---- tabs (navigation links, never role=tab) ---- */
405413 .tabs {
406414 display: flex; gap: 2px; align-items: stretch;
test/crew-scorecard.test.ts +52 −1
@@ -10,7 +10,11 @@ import {
1010 createRun,
1111 finishRun,
1212 } from "../src/core/runs.ts";
13 import { crewScorecards, recentWindow } from "../src/core/scorecard.ts";
13 import {
14 crewScorecards,
15 crewScorecardSplits,
16 recentWindow,
17 } from "../src/core/scorecard.ts";
1418 import { createUser, getUserByName } from "../src/core/users.ts";
1519 import { createForgeApp } from "../src/tui/forge.ts";
1620 import { headlessTerminal } from "../src/tui/term/terminal.ts";
@@ -278,4 +282,51 @@ describe("a scorecard per crew member", () => {
278282 expect(t.text()).not.toContain("all 2 of 3");
279283 t.stop();
280284 });
285
286 test("one member on two models gets a row each, with their own landed counts", async () => {
287 await fetch(`${base}/api/v1/agents`, {
288 method: "POST",
289 headers: { cookie, "content-type": "application/json" },
290 body: JSON.stringify({ username: "lee", model: "opus" }),
291 });
292 const lee = getUserByName(app.db, "lee")!.id;
293 const ran = (issueId: string, model: string, promptVersion: string) => {
294 const run = createRun(app.db, { repoId, kind: "agent", issueId, actorId: lee });
295 claimRun(app.db, run.id);
296 app.db
297 .query(`UPDATE run SET model = ?2, prompt_version = ?3 WHERE id = ?1`)
298 .run(run.id, model, promptVersion);
299 };
300 const merged = "2026-10-05T00:00:00.000Z";
301 ran(pull(lee, "A", { mergedAt: merged }), "opus", "p1");
302 ran(pull(lee, "B", { mergedAt: merged }), "opus", "p1");
303 ran(pull(lee, "C", { mergedAt: merged }), "haiku", "p1");
304 ran(pull(lee, "D"), "haiku", "p1");
305 pull(lee, "Before the migration", { mergedAt: merged });
306
307 const rows = crewScorecardSplits(app.db).get(lee)!;
308 const by = (m: string) => rows.find((r) => r.model === m)!;
309 expect([by("opus").opened, by("opus").landed]).toEqual([2, 2]);
310 expect([by("haiku").opened, by("haiku").landed]).toEqual([2, 1]);
311 // A pull with no recorded run is shown as unrecorded, not dropped.
312 expect([by("").opened, by("").landed, by("").promptVersion]).toEqual([1, 1, ""]);
313 expect(rows.reduce((n, r) => n + r.opened, 0)).toBe(5);
314
315 const page = await fetch(`${base}/settings/crew`, { headers: { cookie } }).then((r) => r.text());
316 const table = page.slice(page.indexOf(`id="crew-scorecard"`));
317 expect(table).toContain("opus");
318 expect(table).toContain("haiku");
319 expect(table).toContain("unrecorded");
320
321 // The terminal shows the same sub-rows under the member.
322 const t = terminal("ada");
323 await t.palette("Crew scorecard per agent");
324 await t.enter();
325 const text = t.text();
326 expect(text).toContain("↳ opus · p1");
327 expect(text).toContain("↳ haiku · p1");
328 expect(text).toContain("↳ unrecorded · unrecorded");
329 expect(text).toContain("↳ opus · p1 landed 30d 2 of 2 · all 2 of 2");
330 t.stop();
331 });
281332 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs