Public: anyone can read this; the crew can change it.

The week's model allowance is spent on purpose: the crew stood still on 10 of 28 days because it was gone in four or five #14281

opened this in answer to

Check · Review · Land · Live — Pushed since the last review. Next: reviewer, pushed since the last review.

Ready to land

scout/forge:scout/14250-the-week-s-model-allowance-is-spent-on-p → main 3 commits fork

Review

Sign in to review this change.
7 files changed +186 −10
docs/features/week-pace.md +2
@@ -12,3 +12,5 @@ The settings are the plain values in `WEEK` (`src/crew/week.ts`); the decision i
1212 ## Bets (#8713)
1313
1414 Bets are not in the forge yet, so the release above falls back to the lowest-numbered gate-held issue. When bets land, "behind" releases the next scope of a backed bet, or offers the next bet to back, in place of that fallback; the change is the one candidate read in `releaseForPace` (`src/crew/autonomy.ts`). Appetites then sit inside the week's capacity the decision already works out (`capUsd` less the reserve), so no bet's share can exceed it.
15
16 **What the week bought (#14250).** The efficiency sentence on Home and the terminal's dashboard also says what share of the week's agent spend is on work that has not landed (a run on a pull closed without merging, or on a closed issue no merged pull closes; work still open, work merged before the week, and runs with no issue are not counted as lost, though they stay in the spend), and, at the rate so far, what the allowance left should buy: landings per point of the week used times the points left up to the weekly budget. This is ask 3 only; it comes first because the owner's choice to hold or spend the reserve needs the figure before the pipeline work (asks 1, 2 and 4) changes what it measures, and the issue is narrowed to it. Left out: the split of large issues, the stop after a red pull's third run (a red is handed back only when the same failure repeats, `sameRedTwice`), and checks while the allowance is gone.
src/core/migration-pins/20261117090000-run-agent-window.pin added +1
@@ -0,0 +1,1 @@
1 df1ffa5202a6d72af73bbcb50872530381e7fb3e11bd23b6c0a98b0a7469080c
src/core/migrations/20261117090000-run-agent-window.ts added +6
@@ -0,0 +1,6 @@
1 // The week sentence on Home and the terminal reads the agent runs since the week opened, twice
2 // (their count and cost, and what of it was lost). Without an index on the time it read every run
3 // of the repository; this one answers from its own order. Additive.
4 export default `
5 CREATE INDEX IF NOT EXISTS run_agent_window ON run(repo_id, created_at) WHERE kind = 'agent';
6 `;
src/crew/week.ts +34 −5
@@ -94,11 +94,40 @@ export interface Efficiency {
9494 readonly landings: number;
9595 readonly runs: number;
9696 readonly costUsd: number;
97 /** Of `costUsd`, what agent runs spent on issues and pulls that have not landed (#14250). */
98 readonly unlandedUsd: number;
9799 }
98100
99 /** `$3.20 and 2.5 runs per landing (4 landings)`; says so plainly when nothing landed. */
100 export function efficiencyWords(e: Efficiency): string {
101 if (e.landings === 0) return `Nothing has landed this week (${e.runs} runs, ${usd(e.costUsd)}).`;
102 const runs = Math.round((e.runs / e.landings) * 10) / 10;
103 return `Each landing cost ${usd(e.costUsd / e.landings)} and ${runs} runs this week (${e.landings} landed, pulls closed as already on the base not counted).`;
101 /** What the week has used and may use, in percent of the provider's week. */
102 export interface Allowance {
103 readonly usedPct: number;
104 readonly mayUsePct: number;
105 }
106
107 /**
108 * `$3.20 and 2.5 runs per landing (4 landings)`; says so plainly when nothing landed. Given
109 * the week's allowance it adds, on the same line, what share of the spend has not landed (runs on
110 * issues and pulls only) and what the allowance left is expected to buy at the rate so far
111 * (#14250), so holding a reserve or spending it is an informed choice.
112 */
113 export function efficiencyWords(e: Efficiency, allowance?: Allowance): string {
114 const lost =
115 e.costUsd <= 0
116 ? ""
117 : ` ${Math.round((e.unlandedUsd / e.costUsd) * 100)}% of the spend (${usd(e.unlandedUsd)}) has not landed.`;
118 if (e.landings === 0) return `Nothing has landed this week (${e.runs} runs, ${usd(e.costUsd)}).${lost}`;
119 const runs = Math.round((e.runs / e.landings) * 10) / 10;
120 const base = `Each landing cost ${usd(e.costUsd / e.landings)} and ${runs} runs this week (${e.landings} landed, pulls closed as already on the base not counted).${lost}`;
121 return `${base}${expectedBuy(e, allowance)}`;
122 }
123
124 /**
125 * What the allowance left should buy at this week's rate: landings per point used, times points
126 * left.
127 */
128 function expectedBuy(e: Efficiency, a: Allowance | undefined): string {
129 if (a === undefined || a.usedPct <= 0) return "";
130 const left = Math.max(0, a.mayUsePct - a.usedPct);
131 const more = Math.round((e.landings / a.usedPct) * left);
132 return ` ${(e.landings / a.usedPct).toFixed(2)} landed per point of the week used; the ${left} points left should buy about ${more} more.`;
104133 }
src/crew/week_read.ts +41 −2
@@ -4,6 +4,7 @@
44 */
55
66 import type { App } from "../app.ts";
7 import { closesFromBody } from "../core/pulls.ts";
78 import { listAutonomousRepos, type Repo } from "../core/repos.ts";
89 import { measureFor } from "../core/measure.ts";
910 import { getHeadroom, WINDOW_MS } from "../core/standing.ts";
@@ -83,7 +84,45 @@ export function efficiencySince(app: App, repo: Repo, since: Date): Efficiency {
8384 WHERE repo_id = ?1 AND kind = 'agent' AND created_at >= ?2`,
8485 )
8586 .get(repo.id, at);
86 return { landings, runs: runs?.n ?? 0, costUsd: runs?.usd ?? 0 };
87 return { landings, runs: runs?.n ?? 0, costUsd: runs?.usd ?? 0, unlandedUsd: unlandedSince(app, repo, at) };
88 }
89
90 /**
91 * What agent runs since `at` cost on work that is lost: a run on a pull closed without merging,
92 * or on a closed issue no merged pull closes. Work still in flight (an open pull or issue) and
93 * pulls that merged, in or before the window, are not lost; neither are runs with no issue
94 * (checks and other non-issue agent runs), which are left out of the figure though they stay in
95 * the week's total spend. One grouped query reads the spend; a second, only when a closed issue
96 * has spend, reads the merged pulls' bodies to see which closed issues they close.
97 */
98 export function unlandedSince(app: App, repo: Repo, at: string): number {
99 const spent = app.db
100 .query<{ number: number; kind: string; state: string; merged: number; usd: number | null }, [string, string]>(
101 `SELECT i.number, i.kind, i.state, q.merged_at IS NOT NULL AS merged, SUM(r.cost_usd) AS usd
102 FROM run r JOIN issue i ON i.id = r.issue_id LEFT JOIN pull q ON q.issue_id = i.id
103 WHERE r.repo_id = ?1 AND r.kind = 'agent' AND r.created_at >= ?2 AND i.state = 'closed'
104 GROUP BY i.id`,
105 )
106 .all(repo.id, at);
107 if (spent.length === 0) return 0;
108 let closed: Set<number> | null = null;
109 const closedByMerge = (): Set<number> => {
110 if (closed !== null) return closed;
111 closed = new Set();
112 const bodies = app.db
113 .query<{ body: string }, [string]>(
114 `SELECT i.body FROM pull q JOIN issue i ON i.id = q.issue_id
115 WHERE i.repo_id = ?1 AND q.merged_at IS NOT NULL AND i.body LIKE '%#%'`,
116 )
117 .all(repo.id);
118 for (const b of bodies) for (const n of closesFromBody(b.body)) closed.add(n);
119 return closed;
120 };
121 let usd = 0;
122 for (const s of spent) {
123 if (s.kind === "pull" ? s.merged === 0 : !closedByMerge().has(s.number)) usd += s.usd ?? 0;
124 }
125 return usd;
87126 }
88127
89128 /** The week's efficiency: since the week opened, or seven days back when no window is known. */
@@ -146,7 +185,7 @@ function busiest<T>(app: App, now: Date, read: (repo: Repo, week: WeekDecision)
146185 * terminal's dashboard print the same words.
147186 */
148187 export function weekWords(app: App, now: Date = new Date()): string | null {
149 return busiest(app, now, (repo, week) => `${week.sentence} ${efficiencyWords(weekEfficiency(app, repo, now))}${heldWords(app)}`);
188 return busiest(app, now, (repo, week) => `${week.sentence} ${efficiencyWords(weekEfficiency(app, repo, now), week)}${heldWords(app)}`);
150189 }
151190
152191 /** The owner's week on Home and the terminal. */
test/week-wiring.test.ts +14
@@ -20,6 +20,7 @@ import { openIssue } from "../src/actions/issues.ts";
2020 import { getAgent } from "../src/core/agents.ts";
2121 import { changeCapacity } from "../src/actions/capacity.ts";
2222 import { capacityDefault, setCapacity } from "../src/core/capacity.ts";
23 import { id } from "../src/kernel/id.ts";
2324 import { forgetWeek, weekWords } from "../src/crew/week_read.ts";
2425 import { createForgeApp } from "../src/tui/forge.ts";
2526 import { headlessTerminal } from "../src/tui/term/terminal.ts";
@@ -279,3 +280,16 @@ test("the autonomy card shows the week and the pace as two rows, once, with no d
279280 servers.stop();
280281 }
281282 });
283
284 test("home and the terminal's sentence says what share of the spend was lost and what the allowance left should buy", () => {
285 week(50);
286 const at = new Date(NOW.getTime() - DAY).toISOString();
287 const lost = openIssue(app, alice, repo, { title: "Lost work", body: "Never built." });
288 if (!lost.ok) throw new Error("issue not opened");
289 app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(lost.value.id);
290 app.db.query("INSERT INTO run (id, repo_id, issue_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, ?3, 'agent', 'ok', 4, ?4)").run(id(), repo.id, lost.value.id, at);
291 forgetWeek();
292 const words = weekWords(app, NOW)!;
293 expect(words).toMatch(/Nothing has landed this week \(\d+ runs, \$[\d.]+\)\. \d+% of the spend \(\$4\.00\) has not landed\./);
294 expect(String(weekLine(app, NOW))).toContain("has not landed");
295 });
test/week.test.ts +88 −3
@@ -7,7 +7,7 @@ import { createApp } from "../src/app.ts";
77 import { createRepo } from "../src/core/repos.ts";
88 import { createUser } from "../src/core/users.ts";
99 import { id } from "../src/kernel/id.ts";
10 import { efficiencySince } from "../src/crew/week_read.ts";
10 import { efficiencySince, unlandedSince } from "../src/crew/week_read.ts";
1111 import { efficiencyWords, weekDecision, type WeekInput } from "../src/crew/week.ts";
1212
1313 const DAY = 86_400_000;
@@ -75,8 +75,93 @@ describe("efficiency", () => {
7575 for (const cost of [1, 2, 3, 4, 5, 6])
7676 app.db.query(`INSERT INTO run (id, repo_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, 'agent', 'ok', ?3, ?4)`).run(id(), repo.id, cost, at);
7777 const e = efficiencySince(app, repo, new Date(NOW.getTime() - DAY));
78 expect(e).toEqual({ landings: 2, runs: 6, costUsd: 21 });
78 expect(e).toEqual({ landings: 2, runs: 6, costUsd: 21, unlandedUsd: 0 });
7979 expect(efficiencyWords(e)).toContain("$10.50 and 3 runs");
80 expect(efficiencyWords({ landings: 0, runs: 2, costUsd: 4 })).toContain("Nothing has landed");
80 expect(efficiencyWords({ landings: 0, runs: 2, costUsd: 4, unlandedUsd: 0 })).toContain("Nothing has landed");
81 });
82
83 const seed = async () => {
84 const app = createApp({ dataDir: mkdtempSync(join(tmpdir(), "forge-week-")), port: 0 });
85 const u = await createUser(app.db, { username: "alice", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true });
86 if (!u.ok) throw new Error(u.error);
87 const made = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "r", visibility: "public", description: "r" });
88 if (!made.ok) throw new Error(made.error);
89 const repo = made.value;
90 const at = NOW.toISOString();
91 const issue = (n: number, kind: string, state: string, body = "") => {
92 const issueId = id();
93 app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, ?4, 'T', ?5, ?6, ?7, ?8)`).run(issueId, repo.id, n, kind, body, u.value.id, state, at);
94 return issueId;
95 };
96 const merge = (issueId: string, n: number, mergedAt: string | null) =>
97 app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid, carried) VALUES (?1, ?2, ?3, 'base', '', ?4, ?5, 'new', 0)`).run(issueId, repo.defaultBranch, `b/${n}`, repo.id, mergedAt);
98 const spend = (issueId: string | null, cost: number, created = at) =>
99 app.db.query(`INSERT INTO run (id, repo_id, issue_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, ?3, 'agent', 'ok', ?4, ?5)`).run(id(), repo.id, issueId, cost, created);
100 return { app, repo, at, issue, merge, spend };
101 };
102 const since = new Date(NOW.getTime() - DAY);
103
104 test("spend on work that landed is not unlanded, and the words say what the allowance left should buy", async () => {
105 const { app, repo, at, issue, merge, spend } = await seed();
106 const built = issue(1, "issue", "closed");
107 const lost = issue(2, "issue", "closed");
108 const pullId = issue(3, "pull", "closed", "Closes #1");
109 merge(pullId, 3, at);
110 spend(built, 6);
111 spend(pullId, 2);
112 spend(lost, 2);
113 const e = efficiencySince(app, repo, since);
114 expect(e).toEqual({ landings: 1, runs: 3, costUsd: 10, unlandedUsd: 2 });
115 const words = efficiencyWords(e, { usedPct: 50, mayUsePct: 90 });
116 expect(words).toContain("20% of the spend ($2.00) has not landed");
117 expect(words).toContain("0.02 landed per point of the week used; the 40 points left should buy about 1 more");
118 expect(efficiencyWords(e)).not.toContain("points left");
119 });
120
121 test("a pull closed without merging is lost; one in flight, and one that merged before the window, are not", async () => {
122 const { app, repo, issue, merge, spend } = await seed();
123 const dead = issue(1, "pull", "closed");
124 merge(dead, 1, null);
125 const live = issue(2, "pull", "open");
126 merge(live, 2, null);
127 const early = issue(3, "pull", "closed", "Closes #4");
128 merge(early, 3, new Date(NOW.getTime() - 30 * DAY).toISOString());
129 const earlyIssue = issue(4, "issue", "closed");
130 const open = issue(5, "issue", "open");
131 spend(dead, 3);
132 spend(live, 5);
133 spend(early, 7);
134 spend(earlyIssue, 11);
135 spend(open, 13);
136 expect(unlandedSince(app, repo, since.toISOString())).toBe(3);
137 });
138
139 test("runs with no issue are left out of the lost figure", async () => {
140 const { app, repo, issue, spend } = await seed();
141 spend(null, 9);
142 spend(issue(1, "issue", "closed"), 1);
143 expect(unlandedSince(app, repo, since.toISOString())).toBe(1);
144 expect(efficiencySince(app, repo, since)).toEqual({ landings: 0, runs: 2, costUsd: 10, unlandedUsd: 1 });
145 });
146
147 test("with nothing landed the words still say what was lost, and the allowance left clamps at zero", () => {
148 const none = { landings: 0, runs: 4, costUsd: 8, unlandedUsd: 2 };
149 expect(efficiencyWords(none, { usedPct: 50, mayUsePct: 90 })).toBe("Nothing has landed this week (4 runs, $8.00). 25% of the spend ($2.00) has not landed.");
150 const some = { landings: 3, runs: 6, costUsd: 12, unlandedUsd: 0 };
151 expect(efficiencyWords(some, { usedPct: 0, mayUsePct: 90 })).not.toContain("points left");
152 expect(efficiencyWords(some, { usedPct: 95, mayUsePct: 90 })).toContain("the 0 points left should buy about 0 more");
153 });
154
155 test("one query reads a large run table in well under 50 ms", async () => {
156 const { app, repo, at, issue, merge, spend } = await seed();
157 const closed = Array.from({ length: 200 }, (_, k) => issue(k + 1, k % 2 ? "issue" : "pull", "closed", k % 2 ? "" : `Closes #${k + 2}`));
158 closed.forEach((c, k) => k % 2 === 0 && merge(c, k + 1, at));
159 app.db.transaction(() => {
160 for (let k = 0; k < 20_000; k++) spend(closed[k % 200]!, 1);
161 })();
162 const t = performance.now();
163 const usd = unlandedSince(app, repo, since.toISOString());
164 expect(performance.now() - t).toBeLessThan(50);
165 expect(usd).toBe(0);
81166 });
82167 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs