trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns14g rEventsFlows

Escalate to Opus by need, not by agent: P0s, planning, and any pull sent back twice run on Opus; everything else on the agent's own model #7576

Merged masoncrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/7570-escalate-to-opus-by-need-not-by-agent-p0 → main 4 commits fork 1 thread · 1 unresolved

  • Merged. mason/7570-escalate-to-opus-by-need-not-by-agent-p0 was merged into main as 75774b0.

Review

Sign in to review this change.
14 files changed +396 −7
README.md +1
@@ -37,6 +37,7 @@ The terminal's scorecard also ends each agent's row with `spend $ 5h · today ·
3737 its cost over the last 5 hours, since 00:00 UTC and the last 7 days, as
3838 `/settings/spend` reads it; `—` means nothing was spent in that span (`spend —`,
3939 in all three).
40 Runs on P0, `opus`-labelled, planner and twice-bounced work use Opus (`docs/features/model-escalation.md`); the "Per model per day" spend and landed table is on Settings → Crew, in the terminal's "Crew scorecard" list, and in `GET /api/v1/crew/scorecard`.
4041 Settings → Crew also shows, each only when it has something, a "Handed to the
4142 planner" card (issues the crew parked twice: repo, issue link, when last
4243 parked, why) and a "Restore drill" card (the last drill's pass or fail and
docs/features/model-escalation.md added +11
@@ -0,0 +1,11 @@
1 # Escalate to Opus by need
2
3 A run picks its model from the work, not only the agent (`src/crew/model_choice.ts`):
4
5 - an issue serving the P0 pillar, or labelled `opus` → `opus`
6 - a planner's run → `opus`
7 - the next round on a pull that has two or more changes-requested reviews → `opus` (not for the reviewer's own run)
8 - anything else → the agent's own model
9
10 The run row records `model` and `model_reason` (`agent`, `p0`, `label:opus`, `planner`, `bounced:N`).
11 Settings → Crew (`/settings/crew`, "Per model per day"), the terminal's "Crew scorecard" list (admin only) and `GET /api/v1/crew/scorecard` (`modelDays`) show runs, spend and landed pulls per model per day.
src/actions/scorecard.ts +3 −1
@@ -11,6 +11,7 @@ import {
1111 type Scorecard,
1212 type SplitScorecard,
1313 } from "../core/scorecard.ts";
14 import { modelDays, type ModelDay } from "../core/model_days.ts";
1415 import { listAgents } from "../core/agents.ts";
1516 import type { User } from "../core/users.ts";
1617
@@ -40,7 +41,7 @@ const figures = (c: Scorecard): Scorecard => ({
4041 export function readScorecards(
4142 app: App,
4243 actor: User | undefined,
43 ): { agents: AgentScorecard[] } | undefined {
44 ): { agents: AgentScorecard[]; modelDays: ModelDay[] } | undefined {
4445 if (actor?.isAdmin !== true) return undefined;
4546 const window = recentWindow();
4647 const all = crewScorecards(app.db);
@@ -49,6 +50,7 @@ export function readScorecards(
4950 const recentSplits = crewScorecardSplits(app.db, window);
5051 const key = (p: SplitScorecard) => `${p.model}\u0000${p.promptVersion}`;
5152 return {
53 modelDays: modelDays(app.db),
5254 agents: listAgents(app.db).map((agent) => {
5355 const id = agent.user.id;
5456 const lately = new Map((recentSplits.get(id) ?? []).map((p) => [key(p), p]));
src/core/migration-pins/20261027140000-run-model-reason.pin added +1
@@ -0,0 +1,1 @@
1 41f6145d061298b139c990f4a88b43ab131b26c816b297ca3152e635d38e98d1
src/core/migrations/20261027140000-run-model-reason.ts added +5
@@ -0,0 +1,5 @@
1 // Why an agent run ran on the model it did: its agent's own, or an escalation
2 // to Opus by need (#7570).
3 export default `
4 ALTER TABLE run ADD COLUMN model_reason TEXT NOT NULL DEFAULT '';
5 `;
src/core/model_days.ts added +54
@@ -0,0 +1,54 @@
1 /**
sentinelcrew

This read model reaches the page and the API but not src/tui/screens/. Add a terminal view that reads modelDays, gated to admins as readScorecards is.
2 * Spend and landed pulls per model per day (#7570), so the switch from one
3 * model to another is measured. A run counts on the day it was created, a
4 * pull on the day it merged, under the model of the latest agent run on it.
5 * An unrecorded model is the empty string.
6 */
7 import type { Database } from "bun:sqlite";
8
9 export interface ModelDay {
10 /** UTC date, `2026-10-27`. */
11 readonly day: string;
12 readonly model: string;
13 readonly runs: number;
14 readonly spentUsd: number;
15 readonly landed: number;
16 }
17
18 /** The last `days` days, newest day first, then the model by name. */
19 export function modelDays(db: Database, days = 7, now = Date.now()): ModelDay[] {
20 const since = new Date(now - days * 86_400_000).toISOString();
21 const rows = new Map<string, { day: string; model: string; runs: number; spentUsd: number; landed: number }>();
22 const at = (day: string, model: string) => {
23 const key = `${day}\0${model}`;
24 let row = rows.get(key);
25 if (row === undefined) rows.set(key, (row = { day, model, runs: 0, spentUsd: 0, landed: 0 }));
26 return row;
27 };
28 for (const r of db
29 .query<{ day: string; model: string; runs: number; spent: number }, [string]>(
30 `SELECT substr(created_at, 1, 10) AS day, model, COUNT(*) AS runs,
31 COALESCE(SUM(cost_usd), 0) AS spent
32 FROM run WHERE kind = 'agent' AND created_at >= ?1 GROUP BY day, model`,
33 )
34 .all(since)) {
35 const row = at(r.day, r.model);
36 row.runs = r.runs;
37 row.spentUsd = r.spent;
38 }
39 for (const r of db
40 .query<{ day: string; model: string; landed: number }, [string]>(
41 `SELECT substr(pull.merged_at, 1, 10) AS day,
42 COALESCE((SELECT run.model FROM run
43 WHERE run.issue_id = issue.id AND run.kind = 'agent'
44 ORDER BY run.id DESC LIMIT 1), '') AS model,
45 COUNT(*) AS landed
46 FROM pull JOIN issue ON issue.id = pull.issue_id
47 WHERE pull.merged_at >= ?1 GROUP BY day, model`,
48 )
49 .all(since))
50 at(r.day, r.model).landed = r.landed;
51 return [...rows.values()].sort(
52 (a, b) => b.day.localeCompare(a.day) || a.model.localeCompare(b.model),
53 );
54 }
src/core/runs.ts +5 −1
@@ -400,6 +400,8 @@ export function noteRunProgress(
400400 node?: string | undefined;
401401 /** The model an agent run ran on (empty: the runner's default), and its prompt version. */
402402 model?: string | undefined;
403 /** Why that model: the agent's own, or what escalated the run to Opus. */
404 modelReason?: string | undefined;
403405 promptVersion?: string | undefined;
404406 },
405407 ): void {
@@ -411,7 +413,8 @@ export function noteRunProgress(
411413 placement = COALESCE(?8, placement),
412414 node = COALESCE(?9, node),
413415 model = COALESCE(?10, model),
414 prompt_version = COALESCE(?11, prompt_version)
416 prompt_version = COALESCE(?11, prompt_version),
417 model_reason = COALESCE(?12, model_reason)
415418 WHERE id = ?1`,
416419 ).run(
417420 runId,
@@ -425,6 +428,7 @@ export function noteRunProgress(
425428 progress.node?.slice(0, 253) ?? null,
426429 progress.model?.slice(0, 100) ?? null,
427430 progress.promptVersion?.slice(0, 40) ?? null,
431 progress.modelReason?.slice(0, 60) ?? null,
428432 );
429433 }
430434
src/crew/model_choice.ts added +71
@@ -0,0 +1,71 @@
1 /**
2 * Which model a run uses (#7570). The agent's own model is the default;
3 * work that costs more to get wrong escalates to Opus: a P0 issue or one
4 * labelled `opus`, a planner's run, and the next round of a pull that has
5 * been sent back for changes twice. The reason is recorded on the run row.
6 */
7 import type { Database } from "bun:sqlite";
8 import type { Repo } from "../core/repos.ts";
9 import { labelsFor, type Issue } from "../core/issues.ts";
10 import { REVIEW_ROUNDS } from "../core/reviews.ts";
11 import { pillarsOf, servedBy } from "../core/pillars.ts";
12
13 /** The model escalated runs ask Claude Code for. */
14 export const OPUS_MODEL = "opus";
15 /** Changes-requested reviews after which a pull's next round runs on Opus. */
16 export const BOUNCED_ROUNDS = REVIEW_ROUNDS;
17 /** The label that asks for Opus by hand. */
18 export const OPUS_LABEL = "opus";
19
20 export interface ModelFacts {
21 readonly agentModel: string;
22 readonly role: string | undefined;
23 readonly p0: boolean;
24 readonly labelled: boolean;
25 readonly bouncedRounds: number;
26 }
27
28 export interface ModelChoice {
29 /** Empty: the runner's default. */
30 readonly model: string;
31 /** `agent`, or what escalated: `p0`, `label:opus`, `planner`, `bounced:N`. */
32 readonly reason: string;
33 }
34
35 /** Pure: the facts of the work in, the model and why out. */
36 export function chooseModel(facts: ModelFacts): ModelChoice {
37 if (facts.p0) return { model: OPUS_MODEL, reason: "p0" };
38 if (facts.labelled) return { model: OPUS_MODEL, reason: `label:${OPUS_LABEL}` };
39 if (facts.role === "planner") return { model: OPUS_MODEL, reason: "planner" };
40 if (facts.bouncedRounds >= BOUNCED_ROUNDS)
41 return { model: OPUS_MODEL, reason: `bounced:${facts.bouncedRounds}` };
42 return { model: facts.agentModel, reason: "agent" };
43 }
44
45 /** Reads the facts about the issues a run is for (the pull, and the issue it closes). */
46 export function modelFacts(
47 db: Database,
48 repo: Repo,
49 issues: readonly Issue[],
50 agentModel: string,
51 role: string | undefined,
52 reviewing = false,
53 ): ModelFacts {
54 const ids = issues.map((i) => i.id);
55 const serving = servedBy(db, pillarsOf(db, repo));
56 const p0 = ids.some((id) => /^P0\b/.test(serving.get(id) ?? ""));
57 const labelled = ids.some((id) =>
58 labelsFor(db, id).some((l) => l.name.toLowerCase() === OPUS_LABEL),
59 );
60 // A reviewer reads a bounced pull; it is not the next round of the work.
61 let bouncedRounds = 0;
62 for (const issue of reviewing ? [] : issues) {
63 if (issue.kind !== "pull") continue;
64 bouncedRounds = db
65 .query<{ n: number }, [string]>(
66 "SELECT COUNT(*) AS n FROM review WHERE issue_id = ?1 AND state = 'changes_requested'",
67 )
68 .get(issue.id)!.n;
69 }
70 return { agentModel, role, p0, labelled, bouncedRounds };
71 }
src/crew/runner.ts +14 −2
@@ -32,6 +32,7 @@ import { announceRun } from "../actions/runs.ts";
3232 import { Stopwatch } from "../apps/timing.ts";
3333 import { previewEnv } from "../apps/data.ts";
3434 import { appsRuntime } from "../apps/runtime.ts";
35 import { chooseModel, modelFacts } from "./model_choice.ts";
3536 import { ownApp, ownAppUrls } from "./app_hosts.ts";
3637 import { crewRole, getAgent, type Agent } from "../core/agents.ts";
3738 import { getAppByRepo, type AppRecord } from "../core/apps.ts";
@@ -1327,6 +1328,16 @@ async function work(
13271328 // One deadline for the run, whatever it takes to get there: the token
13281329 // dies ten minutes after it, so a second invocation gets what is left.
13291330 const deadline = Date.now() + options.runTimeoutMs;
1331 // The agent's own model, unless the work asks for Opus (#7570).
1332 const facts = modelFacts(
1333 app.db,
1334 repo,
1335 taskIssue === undefined ? [issue] : [issue, taskIssue],
1336 agent.model,
1337 brief.role,
1338 reviewing,
1339 );
1340 const chosen = chooseModel(facts);
13301341 const invoke = async (
13311342 prompt: string,
13321343 budgetUsd: number,
@@ -1338,7 +1349,7 @@ async function work(
13381349 cwd: dir,
13391350 prompt,
13401351 appendSystemPrompt: systemPrompt(brief),
1341 model: agent.model || undefined,
1352 model: chosen.model || undefined,
13421353 budgetUsd,
13431354 resume,
13441355 env,
@@ -1372,7 +1383,8 @@ async function work(
13721383 `claude: budget $${budget.toFixed(2)}, $${reserve.toFixed(2)} of it kept for the wrap-up${brief.resuming ? `, resuming ${previous?.sessionId}` : ""}`,
13731384 );
13741385 noteRunProgress(app.db, run.id, {
1375 model: agent.model,
1386 model: chosen.model,
1387 modelReason: chosen.reason,
13761388 promptVersion: promptVersion(agent, brief.role),
13771389 sandbox: {
13781390 enforced: sandbox.enforced,
src/tui/screens/crewsplits.ts +12 −2
@@ -1,9 +1,10 @@
11 /**
22 * The crew scorecard in the terminal, a forge admin's (#2682): agent runs
3 * cut by the model they ran on and by prompt version, as Settings → Crew
3 * cut by the model they ran on, by prompt version and by model per day, as Settings → Crew
44 * has them (`agentSplits`). The gate is the page's: an admin.
55 */
66
7 import { modelDays, type ModelDay } from "../../core/model_days.ts";
78 import { agentSplits, type SplitTotals } from "../../core/runs.ts";
89 import type { Ctx, Screen } from "../app/app.ts";
910 import { binding } from "../app/keys.ts";
@@ -25,6 +26,14 @@ export function splitRows(
2526 }));
2627 }
2728
29 export function modelDayRows(days: readonly ModelDay[]): ListRow[] {
30 return days.map((d) => ({
31 title: `day ${d.day}: ${d.model === "" ? "default" : d.model}`,
32 secondary: `${d.runs} runs`,
33 meta: `${d.landed} landed $${d.spentUsd.toFixed(2)} spent`,
34 }));
35 }
36
2837 export function crewSplitsScreen(f: Session, ctx: Ctx): Screen {
2938 const state = new ListState();
3039 const allowed = f.user?.isAdmin === true;
@@ -33,6 +42,7 @@ export function crewSplitsScreen(f: Session, ctx: Ctx): Screen {
3342 ? [
3443 ...splitRows("model", "default", agentSplits(f.db, "model")),
3544 ...splitRows("prompt", "not recorded", agentSplits(f.db, "prompt_version")),
45 ...modelDayRows(modelDays(f.db)),
3646 ]
3747 : [];
3848 return {
@@ -45,7 +55,7 @@ export function crewSplitsScreen(f: Session, ctx: Ctx): Screen {
4555 ? "The crew scorecard is read by a forge admin."
4656 : rows.length === 0
4757 ? "No agent runs yet."
48 : "Agent runs per model, then per prompt version.";
58 : "Agent runs per model, then per prompt version, then per model per day.";
4959 s.text(head!.x, head!.y, truncate(text, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w);
5060 drawList(s, list!, c, state, rows);
5161 },
src/web/pages/crew.ts +36 −1
@@ -35,6 +35,7 @@ import {
3535 type Scorecard,
3636 type SplitScorecard,
3737 } from "../../core/scorecard.ts";
38 import { modelDays, type ModelDay } from "../../core/model_days.ts";
3839 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
3940 import { listTeams, type Team } from "../../core/teams.ts";
4041 import { crewFor } from "../../crew/dispatch.ts";
@@ -436,6 +437,7 @@ function loadScorecards(ctx: Ctx) {
436437 recentSplitScorecards: crewScorecardSplits(ctx.app.db, recentWindow()),
437438 byModel: agentSplits(ctx.app.db, "model"),
438439 byPrompt: agentSplits(ctx.app.db, "prompt_version"),
440 byModelDay: modelDays(ctx.app.db),
439441 };
440442 }
441443
@@ -473,7 +475,7 @@ function crewView(
473475 splitScorecards,
474476 recentSplitScorecards,
475477 } = data.scorecards;
476 const { scorecards, byModel, byPrompt } = data.scorecards;
478 const { scorecards, byModel, byPrompt, byModelDay } = data.scorecards;
477479 const { spending } = data;
478480 const usernameError =
479481 error !== undefined && /username/i.test(error) ? error : undefined;
@@ -584,6 +586,7 @@ function crewView(
584586 byModel,
585587 () => html`<span class="muted">default</span>`,
586588 )}
589 ${modelDaysCard(byModelDay)}
587590 ${splitsCard(
588591 "Per prompt version",
589592 "Prompt version",
@@ -672,6 +675,38 @@ function drillCard(drill: CrewHandover["drill"]): Html {
672675 );
673676 }
674677
678 /** Spend and landed pulls per model per day: the escalation, measured. */
679 function modelDaysCard(rows: readonly ModelDay[]): Html {
680 if (rows.length === 0) return html``;
681 return card(
682 html`<div class="table-scroll">
683 <table class="table" id="crew-model-days">
684 <thead>
685 <tr>
686 <th>Day</th>
687 <th>Model</th>
688 <th>Runs</th>
689 <th>Spent</th>
690 <th>Landed</th>
691 </tr>
692 </thead>
693 <tbody>
694 ${rows.map(
695 (r) => html`<tr>
696 <td class="mono">${r.day}</td>
697 <td class="mono">${r.model === "" ? html`<span class="muted">default</span>` : r.model}</td>
698 <td class="num">${r.runs}</td>
699 <td class="num">$${r.spentUsd.toFixed(2)}</td>
700 <td class="num">${r.landed}</td>
701 </tr>`,
702 )}
703 </tbody>
704 </table>
705 </div>`,
706 { pad: false, head: html`<h2>Per model per day</h2>` },
707 );
708 }
709
675710 /** The scorecard cut by one thing the runs ran on; nothing before a first run. */
676711 function splitsCard(
677712 title: string,
test/crew-scorecard.test.ts +15
@@ -9,6 +9,7 @@ import {
99 claimRun,
1010 createRun,
1111 finishRun,
12 noteRunProgress,
1213 } from "../src/core/runs.ts";
1314 import {
1415 crewScorecards,
@@ -95,6 +96,20 @@ describe("a scorecard per crew member", () => {
9596 await rm(dataDir, { recursive: true, force: true });
9697 });
9798
99 test("the page has no model-per-day table before a run, and has one for an admin after", async () => {
100 const get = () =>
101 fetch(`${base}/settings/crew`, { headers: { cookie } }).then((r) => r.text());
102 expect(await get()).not.toContain(`id="crew-model-days"`);
103 const run = createRun(app.db, { repoId, kind: "agent", actorId: agentId });
104 claimRun(app.db, run.id);
105 noteRunProgress(app.db, run.id, { model: "opus", modelReason: "p0", costUsd: 0.5 });
106 finishRun(app.db, run.id, { state: "ok", summary: "" });
107 const page = await get();
108 expect(page).toContain(`id="crew-model-days"`);
109 expect(page).toContain("Per model per day");
110 app.db.query("DELETE FROM run WHERE id = ?1").run(run.id);
111 });
112
98113 test("an agent with no pull is a row of dashes, not zeros", async () => {
99114 expect(crewScorecards(app.db).get(agentId)).toBeUndefined();
100115 const page = await fetch(`${base}/settings/crew`, {
test/model-choice.test.ts added +166
@@ -0,0 +1,166 @@
1 /**
2 * A run picks its model from the work (#7570): P0 or `opus`-labelled work,
3 * a planner's run and a pull sent back twice run on Opus; the rest on the
4 * agent's own model. The run row records which and why, and the scorecard
5 * reads spend and landed pulls per model per day.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { createRepo } from "../src/actions/repos.ts";
13 import { chooseModel, modelFacts } from "../src/crew/model_choice.ts";
14 import { addDependency } from "../src/core/dependencies.ts";
15 import { createIssue, createLabel, setIssueLabels, type Issue } from "../src/core/issues.ts";
16 import { modelDays } from "../src/core/model_days.ts";
17 import { submitReview } from "../src/core/reviews.ts";
18 import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts";
19 import { getRepoById, type Repo } from "../src/core/repos.ts";
20 import type { User } from "../src/core/users.ts";
21 import { readScorecards } from "../src/actions/scorecard.ts";
22 import { seedAdmin } from "./helpers/admin.ts";
23 import { quietLogs } from "./helpers/globals.ts";
24
25 quietLogs();
26
27 const plain = {
28 agentModel: "sonnet",
29 role: "builder",
30 p0: false,
31 labelled: false,
32 bouncedRounds: 0,
33 };
34
35 describe("which model a run uses", () => {
36 test("an ordinary run uses the agent's own model", () => {
37 expect(chooseModel(plain)).toEqual({ model: "sonnet", reason: "agent" });
38 expect(chooseModel({ ...plain, agentModel: "" })).toEqual({ model: "", reason: "agent" });
39 expect(chooseModel({ ...plain, bouncedRounds: 1 }).reason).toBe("agent");
40 });
41
42 test("P0, the opus label, the planner and a twice-bounced pull use opus", () => {
43 expect(chooseModel({ ...plain, p0: true })).toEqual({ model: "opus", reason: "p0" });
44 expect(chooseModel({ ...plain, labelled: true }).reason).toBe("label:opus");
45 expect(chooseModel({ ...plain, role: "planner" }).reason).toBe("planner");
46 expect(chooseModel({ ...plain, bouncedRounds: 2 })).toEqual({
47 model: "opus",
48 reason: "bounced:2",
49 });
50 });
51 });
52
53 describe("the facts of the work", () => {
54 let dataDir: string;
55 let app: App;
56 let repo: Repo;
57 let admin: User;
58
59 const open = (title: string, kind: "issue" | "pull" = "issue"): Issue => {
60 const made = createIssue(app.db, { repoId: repo.id, authorId: admin.id, title, kind });
61 if (!made.ok) throw new Error(made.error);
62 return made.value;
63 };
64
65 beforeAll(async () => {
66 dataDir = await mkdtemp(join(tmpdir(), "forge-model-choice-"));
67 app = createApp({ dataDir, port: 0 });
68 const ada = await seedAdmin(app, { username: "ada", email: "[email protected]" });
69 if (!ada.ok) throw new Error(ada.error);
70 admin = ada.value;
71 const made = await createRepo(app, admin, { name: "hello", visibility: "public" });
72 if (!made.ok) throw new Error(JSON.stringify(made.error));
73 repo = getRepoById(app.db, made.value.id)!;
74 });
75
76 afterAll(async () => {
77 app.close();
78 await rm(dataDir, { recursive: true, force: true });
79 });
80
81 test("an issue serving P0 picks opus, one serving P1 does not", () => {
82 const trust = open("P0 Trust: contained");
83 const feed = open("P1 Feed: front door");
84 const urgent = open("Contain the cell");
85 const calm = open("Tidy the feed");
86 addDependency(app.db, trust, urgent, admin.id);
87 addDependency(app.db, feed, calm, admin.id);
88 const pick = (i: Issue) =>
89 chooseModel(modelFacts(app.db, repo, [i], "sonnet", "builder"));
90 expect(pick(urgent)).toEqual({ model: "opus", reason: "p0" });
91 expect(pick(calm)).toEqual({ model: "sonnet", reason: "agent" });
92 });
93
94 test("the opus label picks opus", () => {
95 const labelled = open("Hard one");
96 const label = createLabel(app.db, repo.id, "opus", "7c3aed");
97 if (!label.ok) throw new Error(label.error);
98 setIssueLabels(app.db, labelled.id, [label.value.id]);
99 expect(
100 chooseModel(modelFacts(app.db, repo, [labelled], "sonnet", "builder")).reason,
101 ).toBe("label:opus");
102 });
103
104 test("a pull sent back twice picks opus for its next round", () => {
105 const pull = open("A change", "pull");
106 const review = (state: "changes_requested" | "approved") =>
107 submitReview(app.db, { issueId: pull.id, authorId: admin.id, state, commitOid: "a" });
108 const pick = () => chooseModel(modelFacts(app.db, repo, [pull], "sonnet", "builder"));
109 review("changes_requested");
110 expect(pick().reason).toBe("agent");
111 review("changes_requested");
112 expect(pick()).toEqual({ model: "opus", reason: "bounced:2" });
113 });
114
115 test("a reviewer on a twice-bounced pull gets the agent's own model", () => {
116 const pull = open("Reviewed", "pull");
117 for (let n = 0; n < 2; n++)
118 submitReview(app.db, { issueId: pull.id, authorId: admin.id, state: "changes_requested", commitOid: "a" });
119 const pick = (reviewing: boolean) =>
120 chooseModel(modelFacts(app.db, repo, [pull], "sonnet", "reviewer", reviewing));
121 expect(pick(false).reason).toBe("bounced:2");
122 expect(pick(true)).toEqual({ model: "sonnet", reason: "agent" });
123 });
124
125 test("a landed pull counts on its merge day under the latest run's model", () => {
126 const pull = open("Landed", "pull");
127 for (const model of ["sonnet", "opus"]) {
128 const run = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: pull.id, actorId: admin.id });
129 claimRun(app.db, run.id);
130 noteRunProgress(app.db, run.id, { model });
131 finishRun(app.db, run.id, { state: "ok", summary: "" });
132 }
133 const merged = new Date(Date.now() - 86_400_000).toISOString();
134 app.db
135 .query(
136 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
137 VALUES (?1, 'main', 'landed', 'a', 'b', ?2)`,
138 )
139 .run(pull.id, merged);
140 const days = modelDays(app.db, 7);
141 const landed = days.filter((d) => d.landed > 0);
142 expect(landed).toEqual([
143 expect.objectContaining({ day: merged.slice(0, 10), model: "opus", landed: 1 }),
144 ]);
145 });
146
147 test("the run records the model and why; the scorecard reads spend and landed per model per day", () => {
148 const run = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: open("Work").id, actorId: admin.id });
149 claimRun(app.db, run.id);
150 noteRunProgress(app.db, run.id, { model: "opus", modelReason: "p0", costUsd: 1.5 });
151 finishRun(app.db, run.id, { state: "ok", summary: "done" });
152 const row = app.db
153 .query<{ model: string; model_reason: string }, [string]>(
154 "SELECT model, model_reason FROM run WHERE id = ?1",
155 )
156 .get(run.id)!;
157 expect(row).toEqual({ model: "opus", model_reason: "p0" });
158 const days = modelDays(app.db);
159 const today = new Date().toISOString().slice(0, 10);
160 expect(days.find((d) => d.day === today && d.model === "opus")).toMatchObject({
161 runs: 2,
162 spentUsd: 1.5,
163 });
164 expect(readScorecards(app, admin)?.modelDays).toEqual(days);
165 });
166 });
test/tui-crew-splits.test.ts +2
@@ -76,6 +76,8 @@ test("an admin's terminal shows the cuts, with the empty key as default / not re
7676 expect(text).toContain("model: default");
7777 expect(text).toContain("prompt: abc123");
7878 expect(text).toContain("prompt: not recorded");
79 expect(text).toContain(`day ${new Date().toISOString().slice(0, 10)}: claude-opus-5-5`);
80 expect(text).toContain("1 runs");
7981 h.stop();
8082 });
8183

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs