trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns15g rEventsFlows

Crew teams the owner can dial: design, data science, security, platform - with a scorecard per crew member #2687

Merged masoncrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/2677-crew-teams-the-owner-can-dial-design-dat → main 3 commits fork

  • Merged. mason/2677-crew-teams-the-owner-can-dial-design-dat was merged into main as 2856e59.

Review

Sign in to review this change.
8 files changed +502 −1
CLAUDE.md +3 −1
@@ -136,7 +136,9 @@ src/
136136 reviews, orgs/teams, webhooks, keys (ssh public keys), search
137137 (repos/issues/`git grep`, all bounded), events (outbox+bus),
138138 reconcile, schema, usage (what each agent run spent: tokens,
139 cost, wall time, by subscription/app/agent/issue) — pure
139 cost, wall time, by subscription/app/agent/issue), scorecard (how each
140 agent's pulls fare: review rounds, red checks, reverts, time to
141 land) — pure
140142 data, no authz, no events
141143 actions/ one function per platform mutation: authorize (access.ts tiers),
142144 call core, `bus.append`. API, web, and TUI are adapters over
README.md +4
@@ -24,6 +24,10 @@ child's own data, and `/forges` says how many are running and why any is not.
2424 In the terminal an admin's palette offers "Forges you sent": each forge with its
2525 claim state and the same apps-running note, never a claim link.
2626
27 Each agent's scorecard — pulls landed of opened, review rounds, red checks,
28 reverts, time to land — is on Settings → Crew (`/settings/crew`) and in the
29 terminal's palette as "Crew scorecard per agent"; both are a forge admin's only.
30
2731 Git hosting, code browsing, and a JSON API in one fast Bun process, with
2832 **zero runtime dependencies** — the `package.json` dependency list is empty and
2933 stays that way. The one exception is dev-only: `@anthropic-ai/sandbox-runtime`
src/core/scorecard.ts added +108
@@ -0,0 +1,108 @@
1 /**
2 * Scorecard — how each crew member's pulls fare after it opens them: the
3 * numbers an owner tunes a crew by (#2677). `agentTotals` (`core/runs.ts`)
4 * says what an agent spent and how many of its pulls landed; this says how
5 * hard they were to land and whether they stayed landed.
6 *
7 * Every figure is over the pulls an agent authored, all time:
8 *
9 * reviewRounds = count(changes_requested reviews on its landed pulls) ÷ landed
10 * redCheckRate = failed ÷ finished (ok + failed) check runs on its pulls
11 * reverts = its landed pulls that a merged pull took back
12 * (`pull.reverts_issue_id`, named explicitly, never sniffed)
13 * meanLandMs = mean of merged_at − opened (issue.created_at) over landed
14 *
15 * A ratio over nothing is null, said as a dash by the page. Only agents
16 * appear; an agent that has opened no pull is absent.
17 *
18 * Pure data: no authz, no events.
19 */
20
21 import type { Database } from "bun:sqlite";
22
23 export interface Scorecard {
24 /** Pulls the agent opened. */
25 readonly opened: number;
26 /** Of those, merged. */
27 readonly landed: number;
28 /** Mean changes-requested reviews per landed pull; null before one lands. */
29 readonly reviewRounds: number | null;
30 /** Share (0–1) of finished check runs on its pulls that failed; null with none. */
31 readonly redCheckRate: number | null;
32 /** Landed pulls a later merged pull reverted. */
33 readonly reverts: number;
34 /** Mean milliseconds from opening a pull to its merge; null before one lands. */
35 readonly meanLandMs: number | null;
36 }
37
38 /** Each agent's scorecard, keyed by its user id: one grouped query. */
39 export function crewScorecards(db: Database): Map<string, Scorecard> {
40 const rows = db
41 .query<
42 {
43 author_id: string;
44 opened: number;
45 landed: number;
46 rounds: number;
47 checks_ok: number;
48 checks_failed: number;
49 reverts: number;
50 mean_land_days: number | null;
51 },
52 []
53 >(
54 `SELECT issue.author_id AS author_id,
55 COUNT(*) AS opened,
56 SUM(pull.merged_at IS NOT NULL) AS landed,
57 COALESCE(SUM(CASE WHEN pull.merged_at IS NOT NULL THEN
58 (SELECT COUNT(*) FROM review
59 WHERE review.issue_id = issue.id
60 AND review.state = 'changes_requested') END), 0) AS rounds,
61 COALESCE(SUM((SELECT COUNT(*) FROM run
62 WHERE run.issue_id = issue.id AND run.kind = 'check'
63 AND run.state = 'ok')), 0) AS checks_ok,
64 COALESCE(SUM((SELECT COUNT(*) FROM run
65 WHERE run.issue_id = issue.id AND run.kind = 'check'
66 AND run.state = 'failed')), 0) AS checks_failed,
67 SUM(pull.merged_at IS NOT NULL AND EXISTS (
68 SELECT 1 FROM pull AS back
69 WHERE back.reverts_issue_id = issue.id
70 AND back.merged_at IS NOT NULL)) AS reverts,
71 AVG(CASE WHEN pull.merged_at IS NOT NULL
72 THEN julianday(pull.merged_at) - julianday(issue.created_at) END)
73 AS mean_land_days
74 FROM issue
75 JOIN pull ON pull.issue_id = issue.id
76 JOIN user ON user.id = issue.author_id AND user.is_agent = 1
77 GROUP BY issue.author_id`,
78 )
79 .all();
80 return new Map(
81 rows.map((r) => {
82 const checks = r.checks_ok + r.checks_failed;
83 return [
84 r.author_id,
85 {
86 opened: r.opened,
87 landed: r.landed,
88 reviewRounds: r.landed === 0 ? null : r.rounds / r.landed,
89 redCheckRate: checks === 0 ? null : r.checks_failed / checks,
90 reverts: r.reverts,
91 meanLandMs:
92 r.mean_land_days === null
93 ? null
94 : Math.max(0, Math.round(r.mean_land_days * 86_400_000)),
95 },
96 ];
97 }),
98 );
99 }
100
101 /** A time to land in the largest unit that keeps it readable: both doors say it the same way. */
102 export function spanText(ms: number): string {
103 const minutes = ms / 60_000;
104 if (minutes < 90) return `${Math.max(1, Math.round(minutes))}m`;
105 const hours = minutes / 60;
106 if (hours < 48) return `${Math.round(hours)}h`;
107 return `${Math.round(hours / 24)}d`;
108 }
src/tui/forge.ts +9
@@ -45,6 +45,7 @@ import { dashboardScreen } from "./screens/dashboard.ts";
4545 import { mayManageFlags } from "../core/flags.ts";
4646 import { accessScreen } from "./screens/access.ts";
4747 import { flagsScreen } from "./screens/flags.ts";
48 import { scorecardScreen } from "./screens/scorecard.ts";
4849 import { cloudScreen } from "./screens/cloud.ts";
4950 import { crewSplitsScreen } from "./screens/crewsplits.ts";
5051 import { usageScreen } from "./screens/usage.ts";
@@ -375,6 +376,14 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
375376 hint: "agent runs per model and per prompt version",
376377 run: () => ctx.push(crewSplitsScreen(session, ctx)),
377378 });
379 // Settings → Crew's scorecard per agent, as the page has it: a forge admin (#2677).
380 if (session.user?.isAdmin === true)
381 commands.push({
382 id: "scorecard",
383 title: "Crew scorecard per agent",
384 hint: "each agent's pulls landed, review rounds, red checks, reverts, time to land",
385 run: () => ctx.push(scorecardScreen(session, ctx)),
386 });
378387 // Proposal tokens for a repository you write, as the API and MCP have them (#2148).
379388 if (session.user !== undefined)
380389 commands.push({
src/tui/screens/scorecard.ts added +68
@@ -0,0 +1,68 @@
1 /**
2 * The crew's scorecard in the terminal, a forge admin's (#2677): each
3 * agent's pulls landed of opened, review rounds, red checks, reverts and
4 * time to land, as Settings → Crew shows them (`crewScorecards`). The gate
5 * is the page's: a forge admin; anyone else sees nothing.
6 */
7
8 import { listAllAgents } from "../../actions/crew.ts";
9 import {
10 crewScorecards,
11 spanText,
12 type Scorecard,
13 } from "../../core/scorecard.ts";
14 import type { Agent } from "../../core/agents.ts";
15 import type { Ctx, Screen } from "../app/app.ts";
16 import { binding } from "../app/keys.ts";
17 import { style } from "../term/style.ts";
18 import { truncate } from "../term/width.ts";
19 import type { Session } from "../session.ts";
20 import { fill, len, split } from "../ui/layout.ts";
21 import { ListState, drawList, type ListRow } from "./common.ts";
22
23 /** One row per agent; one with no pull yet says so rather than a row of zeros. */
24 export function scorecardRows(
25 agents: readonly Agent[],
26 cards: ReadonlyMap<string, Scorecard>,
27 ): ListRow[] {
28 return agents.map((agent) => {
29 const s = cards.get(agent.user.id);
30 if (s === undefined)
31 return { title: agent.user.username, secondary: "no pull yet" };
32 return {
33 title: agent.user.username,
34 secondary: `${s.landed} landed of ${s.opened}`,
35 meta: [
36 `review rounds ${s.reviewRounds === null ? "—" : s.reviewRounds.toFixed(1)}`,
37 `red checks ${s.redCheckRate === null ? "—" : `${Math.round(100 * s.redCheckRate)}%`}`,
38 `reverts ${s.reverts}`,
39 `time to land ${s.meanLandMs === null ? "—" : spanText(s.meanLandMs)}`,
40 ].join(" "),
41 };
42 });
43 }
44
45 export function scorecardScreen(f: Session, ctx: Ctx): Screen {
46 const state = new ListState();
47 const allowed = f.user?.isAdmin === true;
48 // Read once when the screen opens, as the page reads it once per request.
49 const listed = allowed ? listAllAgents(f.app, f.user) : undefined;
50 const rows =
51 listed?.ok === true ? scorecardRows(listed.value, crewScorecards(f.db)) : [];
52 return {
53 title: "Crew scorecard per agent",
54 keys: [binding(["j", "k"], "move cursor")],
55 render(area, s) {
56 const c = ctx.theme.c;
57 const [head, list] = split(area, "v", [len(1), fill(1)]);
58 const text = !allowed
59 ? "The crew's scorecard is read by a forge admin."
60 : rows.length === 0
61 ? "No agents yet."
62 : "Over every pull each agent opened, as Settings → Crew shows it.";
63 s.text(head!.x, head!.y, truncate(text, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w);
64 drawList(s, list!, c, state, rows);
65 },
66 key: (e) => state.key(e, rows.length),
67 };
68 }
src/web/pages/crew.ts +80
@@ -24,6 +24,11 @@ import {
2424 type AgentTotals,
2525 type SplitTotals,
2626 } from "../../core/runs.ts";
27 import {
28 crewScorecards,
29 spanText,
30 type Scorecard,
31 } from "../../core/scorecard.ts";
2732 import { crewFor } from "../../crew/dispatch.ts";
2833 import { MODELS } from "../../crew/models.ts";
2934 import type { Ctx } from "../../http/context.ts";
@@ -222,6 +227,79 @@ function totalsCells(t: AgentTotals | undefined): Html {
222227 </td>`;
223228 }
224229
230 /**
231 * The scorecard: how each agent's pulls fare once opened — so the owner
232 * can tune who does what. What they cost is the table above's ("Per
233 * landed"), not repeated here. An agent with no pull yet is a row of
234 * dashes, never a row of misleading zeros. The agent's name stays in view
235 * while a phone scrolls the figures sideways.
236 */
237 function scorecardCard(
238 agents: readonly Agent[],
239 cards: ReadonlyMap<string, Scorecard>,
240 ): Html {
241 const dash = html`<td class="num muted">—</td>`;
242 return card(
243 html`<div class="table-scroll">
244 <table class="table" id="crew-scorecard" data-sticky-lead>
245 <thead>
246 <tr>
247 <th>Agent</th>
248 <th>Pulls landed</th>
249 <th>Review rounds</th>
250 <th>Red checks</th>
251 <th>Reverts</th>
252 <th>Time to land</th>
253 </tr>
254 </thead>
255 <tbody>
256 ${agents.map((agent) => {
257 const s = cards.get(agent.user.id);
258 return html`<tr>
259 <td>
260 ${userLink(agent.user.username, {
261 avatar: false,
262 mark: markOf(agent.user),
263 })}
264 </td>
265 ${s === undefined
266 ? html`${dash}${dash}${dash}${dash}${dash}`
267 : html`<td class="num">
268 ${s.landed}
269 <span class="muted small">of ${s.opened}</span>
270 </td>
271 ${s.reviewRounds === null
272 ? dash
273 : html`<td class="num">${s.reviewRounds.toFixed(1)}</td>`}
274 ${s.redCheckRate === null
275 ? dash
276 : html`<td class="num">
277 ${Math.round(100 * s.redCheckRate)}%
278 </td>`}
279 <td class="num">${s.reverts}</td>
280 ${s.meanLandMs === null
281 ? dash
282 : html`<td class="num">${spanText(s.meanLandMs)}</td>`}`}
283 </tr>`;
284 })}
285 </tbody>
286 </table>
287 </div>`,
288 {
289 pad: false,
290 extra: "mt mb",
291 head: html`${icon("activity")}
292 <h2>Scorecard</h2>
293 <span class="muted small"
294 >Over every pull each agent opened: review rounds are change
295 requests per landed pull, red checks the share of finished checks
296 that failed, reverts the landed pulls a merged pull took back, time
297 to land the mean from opening to merge.</span
298 >`,
299 },
300 );
301 }
302
225303 /** Today's spend against each ceiling: the drain, and the knob for it. */
226304 function spendingCard(
227305 spending: readonly { repo: Repo; status: AutonomyStatus }[],
@@ -284,6 +362,7 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html {
284362 const listed = listAllAgents(ctx.app, ctx.auth.user);
285363 const agents = listed.ok ? listed.value : [];
286364 const totals = agentTotals(ctx.app.db);
365 const scorecards = crewScorecards(ctx.app.db);
287366 // Where the money goes today: every app running on its own, its spend
288367 // against its ceiling. The ceilings are the one control on the drain, and
289368 // this is the page where the crew is wielded, so they are read here.
@@ -381,6 +460,7 @@ function crewView(ctx: Ctx, values: AgentValues, error?: string): Html {
381460 </div>`,
382461 { pad: false },
383462 )}
463 ${when(agents.length > 0, () => scorecardCard(agents, scorecards))}
384464 ${splitsCard(
385465 "Per model",
386466 "Model",
src/web/style/components.ts +5
@@ -360,6 +360,11 @@ a.row:hover .row-title { color: var(--accent); }
360360 .table tbody tr:first-child td { border-top: none; }
361361 .table[data-dense] td, .table[data-dense] th { padding: 5px 10px; }
362362 .table-scroll { overflow-x: auto; scrollbar-width: thin; }
363 /* The row's name stays in view while a narrow screen scrolls the figures. */
364 .table[data-sticky-lead] th:first-child,
365 .table[data-sticky-lead] td:first-child {
366 position: sticky; left: 0; z-index: 1; background: var(--card);
367 }
363368
364369 /* ---- tabs (navigation links, never role=tab) ---- */
365370 .tabs {
test/crew-scorecard.test.ts added +225
@@ -0,0 +1,225 @@
1 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2 import { mkdtemp, rm } from "node:fs/promises";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { createApp, type App } from "../src/app.ts";
6 import { createIssue } from "../src/core/issues.ts";
7 import { submitReview } from "../src/core/reviews.ts";
8 import {
9 claimRun,
10 createRun,
11 finishRun,
12 } from "../src/core/runs.ts";
13 import { crewScorecards } from "../src/core/scorecard.ts";
14 import { createUser, getUserByName } from "../src/core/users.ts";
15 import { createForgeApp } from "../src/tui/forge.ts";
16 import { headlessTerminal } from "../src/tui/term/terminal.ts";
17 import { serve, type Servers } from "../src/http/server.ts";
18 import { adminCookie } from "./helpers/admin.ts";
19 import { quietLogs } from "./helpers/globals.ts";
20 import { settleApp } from "./helpers/tui.ts";
21
22 quietLogs();
23
24 describe("a scorecard per crew member", () => {
25 let dataDir: string;
26 let app: App;
27 let servers: Servers;
28 let base: string;
29 let cookie = "";
30 let repoId = "";
31 let agentId = "";
32 let adminId = "";
33
34 /** A pull row as the forge keeps it, without the git it was opened from. */
35 function pull(
36 authorId: string,
37 title: string,
38 opts: { openedAt?: string; mergedAt?: string; reverts?: string } = {},
39 ): string {
40 const issue = createIssue(app.db, { repoId, authorId, title, kind: "pull" });
41 if (!issue.ok) throw new Error(issue.error);
42 if (opts.openedAt)
43 app.db
44 .query(`UPDATE issue SET created_at = ?2 WHERE id = ?1`)
45 .run(issue.value.id, opts.openedAt);
46 app.db
47 .query(
48 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid,
49 merged_at, reverts_issue_id)
50 VALUES (?1, 'main', ?2, 'a', 'b', ?3, ?4)`,
51 )
52 .run(issue.value.id, `h${issue.value.number}`, opts.mergedAt ?? null, opts.reverts ?? null);
53 return issue.value.id;
54 }
55
56 function check(issueId: string, state: "ok" | "failed"): void {
57 const run = createRun(app.db, { repoId, kind: "check", issueId });
58 claimRun(app.db, run.id);
59 finishRun(app.db, run.id, { state, summary: state });
60 }
61
62 beforeAll(async () => {
63 dataDir = await mkdtemp(join(tmpdir(), "forge-scorecard-"));
64 app = createApp({ dataDir, port: 0 });
65 servers = await serve(app);
66 base = servers.url;
67 cookie = await adminCookie(app, {
68 username: "ada",
69 email: "[email protected]",
70 password: "analytical-engine",
71 });
72 adminId = getUserByName(app.db, "ada")!.id;
73 const headers = { cookie, "content-type": "application/json" };
74 const repo = (await fetch(`${base}/api/v1/repos`, {
75 method: "POST",
76 headers,
77 body: JSON.stringify({ name: "hello", visibility: "public" }),
78 }).then((r) => r.json())) as { id: string };
79 repoId = repo.id;
80 await fetch(`${base}/api/v1/agents`, {
81 method: "POST",
82 headers,
83 body: JSON.stringify({ username: "smith", model: "opus" }),
84 });
85 agentId = getUserByName(app.db, "smith")!.id;
86 });
87
88 afterAll(async () => {
89 servers.stop();
90 app.close();
91 await rm(dataDir, { recursive: true, force: true });
92 });
93
94 test("an agent with no pull is a row of dashes, not zeros", async () => {
95 expect(crewScorecards(app.db).get(agentId)).toBeUndefined();
96 const page = await fetch(`${base}/settings/crew`, {
97 headers: { cookie },
98 }).then((r) => r.text());
99 expect(page).toContain(`id="crew-scorecard"`);
100 for (const head of [
101 "Pulls landed",
102 "Review rounds",
103 "Red checks",
104 "Reverts",
105 "Time to land",
106 ])
107 expect(page).toContain(`<th>${head}</th>`);
108 // What a landed pull costs is the agent table's "Per landed"; one figure, one column.
109 expect(page).not.toContain("<th>Landed per $</th>");
110 // The agent's name stays in view while a phone scrolls the figures.
111 expect(page).toContain(`id="crew-scorecard" data-sticky-lead`);
112 });
113
114 test("a landed pull counts for its author, and only its author", () => {
115 pull(agentId, "First change", {
116 openedAt: "2026-10-01T00:00:00.000Z",
117 mergedAt: "2026-10-01T02:00:00.000Z",
118 });
119 const card = crewScorecards(app.db).get(agentId)!;
120 expect(card.opened).toBe(1);
121 expect(card.landed).toBe(1);
122 expect(card.meanLandMs).toBe(2 * 3_600_000);
123 // A person's pulls are not on the crew's scorecard.
124 pull(adminId, "A person's change", { mergedAt: "2026-10-02T00:00:00.000Z" });
125 expect(crewScorecards(app.db).has(adminId)).toBe(false);
126 });
127
128 test("review rounds, red checks and reverts add up", () => {
129 const second = pull(agentId, "Second change", {
130 openedAt: "2026-10-03T00:00:00.000Z",
131 mergedAt: "2026-10-03T04:00:00.000Z",
132 });
133 for (const state of ["changes_requested", "changes_requested", "approved"] as const)
134 submitReview(app.db, { issueId: second, authorId: adminId, state, body: "", commitOid: "b" });
135 // A change request on a pull that never landed is not a round of landing one.
136 const open = pull(agentId, "Still open");
137 submitReview(app.db, { issueId: open, authorId: adminId, state: "changes_requested", body: "", commitOid: "b" });
138 check(second, "failed");
139 check(second, "ok");
140 check(open, "ok");
141 check(open, "ok");
142 // Taken back by a merged revert; an unmerged revert takes nothing back.
143 pull(adminId, "Revert second", { mergedAt: "2026-10-04T00:00:00.000Z", reverts: second });
144 const first = app.db
145 .query<{ id: string }, [string]>(`SELECT id FROM issue WHERE title = ?1`)
146 .get("First change")!.id;
147 pull(adminId, "Revert first, not merged", { reverts: first });
148
149 const card = crewScorecards(app.db).get(agentId)!;
150 expect(card.opened).toBe(3);
151 expect(card.landed).toBe(2);
152 expect(card.reviewRounds).toBe(1); // two change requests over two landed
153 expect(card.redCheckRate).toBe(0.25); // one failed of four finished
154 expect(card.reverts).toBe(1);
155 expect(card.meanLandMs).toBe(3 * 3_600_000); // (2h + 4h) / 2
156 });
157
158 test("the crew page shows the scorecard's numbers", async () => {
159 const page = await fetch(`${base}/settings/crew`, {
160 headers: { cookie },
161 }).then((r) => r.text());
162 const table = page.slice(page.indexOf(`id="crew-scorecard"`));
163 expect(table).toContain("of 3");
164 expect(table).toContain("1.0");
165 expect(table).toContain("25%");
166 expect(table).toContain("3h");
167 });
168
169 /** The forge's terminal, signed in as one account, on a headless screen. */
170 function terminal(username: string) {
171 const user = getUserByName(app.db, username)!;
172 const { app: tui } = createForgeApp(
173 app,
174 { sshUsername: "t", remoteAddress: "127.0.0.1", user },
175 { terminal: headlessTerminal(160, 32), exitHooks: false },
176 );
177 tui.start();
178 const key = (k: string, ctrl = false) =>
179 tui.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
180 return {
181 text: () => tui.screen.toText(),
182 palette: async (word: string) => {
183 key("k", true);
184 for (const ch of word) key(ch);
185 await settleApp(tui);
186 },
187 enter: async () => {
188 key("enter");
189 await settleApp(tui);
190 },
191 stop: () => tui.stop(),
192 };
193 }
194
195 test("the terminal shows an admin the same scorecard", async () => {
196 const t = terminal("ada");
197 await t.palette("Crew scorecard per agent");
198 await t.enter();
199 const text = t.text();
200 expect(text).toContain("smith");
201 expect(text).toContain("2 landed of 3");
202 expect(text).toContain("review rounds 1.0");
203 expect(text).toContain("red checks 25%");
204 expect(text).toContain("reverts 1");
205 expect(text).toContain("time to land 3h");
206 t.stop();
207 });
208
209 test("someone who is not an admin has no scorecard in the terminal", async () => {
210 const made = await createUser(app.db, {
211 origin: "seed",
212 trusted: true,
213 username: "bob",
214 email: "[email protected]",
215 password: "a-good-passphrase",
216 });
217 if (!made.ok) throw new Error(made.error);
218 const t = terminal("bob");
219 await t.palette("Crew scorecard per agent");
220 expect(t.text()).not.toContain("review rounds, red checks");
221 await t.enter();
222 expect(t.text()).not.toContain("2 landed of 3");
223 t.stop();
224 });
225 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs