Public: anyone can read this; the crew can change it.
The week's model allowance is spent on purpose: the crew stood still on 10 of 28 days because it was gone in four or five #14281
Check · Review · Land · Live — Pushed since the last review.
scout/forge:scout/14250-the-week-s-model-allowance-is-spent-on-p → main 3 commits fork
7 files
changed +186
−10
docs/features/week-pace.md +2
| @@ -12,3 +12,5 @@ The settings are the plain values in `WEEK` (`src/crew/week.ts`); the decision i | ||
| 12 | 12 | ## Bets (#8713) |
| 13 | 13 | |
| 14 | 14 | Bets are not in the forge yet, so the release above falls back to the lowest-numbered gate-held issue. When bets land, "behind" releases the next scope of a backed bet, or offers the next bet to back, in place of that fallback; the change is the one candidate read in `releaseForPace` (`src/crew/autonomy.ts`). Appetites then sit inside the week's capacity the decision already works out (`capUsd` less the reserve), so no bet's share can exceed it. |
| 15 | ||
| 16 | **What the week bought (#14250).** The efficiency sentence on Home and the terminal's dashboard also says what share of the week's agent spend is on work that has not landed (a run on a pull closed without merging, or on a closed issue no merged pull closes; work still open, work merged before the week, and runs with no issue are not counted as lost, though they stay in the spend), and, at the rate so far, what the allowance left should buy: landings per point of the week used times the points left up to the weekly budget. This is ask 3 only; it comes first because the owner's choice to hold or spend the reserve needs the figure before the pipeline work (asks 1, 2 and 4) changes what it measures, and the issue is narrowed to it. Left out: the split of large issues, the stop after a red pull's third run (a red is handed back only when the same failure repeats, `sameRedTwice`), and checks while the allowance is gone. |
src/core/migration-pins/20261117090000-run-agent-window.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | df1ffa5202a6d72af73bbcb50872530381e7fb3e11bd23b6c0a98b0a7469080c |
src/core/migrations/20261117090000-run-agent-window.ts added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | // The week sentence on Home and the terminal reads the agent runs since the week opened, twice | |
| 2 | // (their count and cost, and what of it was lost). Without an index on the time it read every run | |
| 3 | // of the repository; this one answers from its own order. Additive. | |
| 4 | export default ` | |
| 5 | CREATE INDEX IF NOT EXISTS run_agent_window ON run(repo_id, created_at) WHERE kind = 'agent'; | |
| 6 | `; |
src/crew/week.ts +34 −5
| @@ -94,11 +94,40 @@ export interface Efficiency { | ||
| 94 | 94 | readonly landings: number; |
| 95 | 95 | readonly runs: number; |
| 96 | 96 | readonly costUsd: number; |
| 97 | /** Of `costUsd`, what agent runs spent on issues and pulls that have not landed (#14250). */ | |
| 98 | readonly unlandedUsd: number; | |
| 97 | 99 | } |
| 98 | 100 | |
| 99 | /** `$3.20 and 2.5 runs per landing (4 landings)`; says so plainly when nothing landed. */ | |
| 100 | export function efficiencyWords(e: Efficiency): string { | |
| 101 | if (e.landings === 0) return `Nothing has landed this week (${e.runs} runs, ${usd(e.costUsd)}).`; | |
| 102 | const runs = Math.round((e.runs / e.landings) * 10) / 10; | |
| 103 | return `Each landing cost ${usd(e.costUsd / e.landings)} and ${runs} runs this week (${e.landings} landed, pulls closed as already on the base not counted).`; | |
| 101 | /** What the week has used and may use, in percent of the provider's week. */ | |
| 102 | export interface Allowance { | |
| 103 | readonly usedPct: number; | |
| 104 | readonly mayUsePct: number; | |
| 105 | } | |
| 106 | ||
| 107 | /** | |
| 108 | * `$3.20 and 2.5 runs per landing (4 landings)`; says so plainly when nothing landed. Given | |
| 109 | * the week's allowance it adds, on the same line, what share of the spend has not landed (runs on | |
| 110 | * issues and pulls only) and what the allowance left is expected to buy at the rate so far | |
| 111 | * (#14250), so holding a reserve or spending it is an informed choice. | |
| 112 | */ | |
| 113 | export function efficiencyWords(e: Efficiency, allowance?: Allowance): string { | |
| 114 | const lost = | |
| 115 | e.costUsd <= 0 | |
| 116 | ? "" | |
| 117 | : ` ${Math.round((e.unlandedUsd / e.costUsd) * 100)}% of the spend (${usd(e.unlandedUsd)}) has not landed.`; | |
| 118 | if (e.landings === 0) return `Nothing has landed this week (${e.runs} runs, ${usd(e.costUsd)}).${lost}`; | |
| 119 | const runs = Math.round((e.runs / e.landings) * 10) / 10; | |
| 120 | const base = `Each landing cost ${usd(e.costUsd / e.landings)} and ${runs} runs this week (${e.landings} landed, pulls closed as already on the base not counted).${lost}`; | |
| 121 | return `${base}${expectedBuy(e, allowance)}`; | |
| 122 | } | |
| 123 | ||
| 124 | /** | |
| 125 | * What the allowance left should buy at this week's rate: landings per point used, times points | |
| 126 | * left. | |
| 127 | */ | |
| 128 | function expectedBuy(e: Efficiency, a: Allowance | undefined): string { | |
| 129 | if (a === undefined || a.usedPct <= 0) return ""; | |
| 130 | const left = Math.max(0, a.mayUsePct - a.usedPct); | |
| 131 | const more = Math.round((e.landings / a.usedPct) * left); | |
| 132 | return ` ${(e.landings / a.usedPct).toFixed(2)} landed per point of the week used; the ${left} points left should buy about ${more} more.`; | |
| 104 | 133 | } |
src/crew/week_read.ts +41 −2
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | import type { App } from "../app.ts"; |
| 7 | import { closesFromBody } from "../core/pulls.ts"; | |
| 7 | 8 | import { listAutonomousRepos, type Repo } from "../core/repos.ts"; |
| 8 | 9 | import { measureFor } from "../core/measure.ts"; |
| 9 | 10 | import { getHeadroom, WINDOW_MS } from "../core/standing.ts"; |
| @@ -83,7 +84,45 @@ export function efficiencySince(app: App, repo: Repo, since: Date): Efficiency { | ||
| 83 | 84 | WHERE repo_id = ?1 AND kind = 'agent' AND created_at >= ?2`, |
| 84 | 85 | ) |
| 85 | 86 | .get(repo.id, at); |
| 86 | return { landings, runs: runs?.n ?? 0, costUsd: runs?.usd ?? 0 }; | |
| 87 | return { landings, runs: runs?.n ?? 0, costUsd: runs?.usd ?? 0, unlandedUsd: unlandedSince(app, repo, at) }; | |
| 88 | } | |
| 89 | ||
| 90 | /** | |
| 91 | * What agent runs since `at` cost on work that is lost: a run on a pull closed without merging, | |
| 92 | * or on a closed issue no merged pull closes. Work still in flight (an open pull or issue) and | |
| 93 | * pulls that merged, in or before the window, are not lost; neither are runs with no issue | |
| 94 | * (checks and other non-issue agent runs), which are left out of the figure though they stay in | |
| 95 | * the week's total spend. One grouped query reads the spend; a second, only when a closed issue | |
| 96 | * has spend, reads the merged pulls' bodies to see which closed issues they close. | |
| 97 | */ | |
| 98 | export function unlandedSince(app: App, repo: Repo, at: string): number { | |
| 99 | const spent = app.db | |
| 100 | .query<{ number: number; kind: string; state: string; merged: number; usd: number | null }, [string, string]>( | |
| 101 | `SELECT i.number, i.kind, i.state, q.merged_at IS NOT NULL AS merged, SUM(r.cost_usd) AS usd | |
| 102 | FROM run r JOIN issue i ON i.id = r.issue_id LEFT JOIN pull q ON q.issue_id = i.id | |
| 103 | WHERE r.repo_id = ?1 AND r.kind = 'agent' AND r.created_at >= ?2 AND i.state = 'closed' | |
| 104 | GROUP BY i.id`, | |
| 105 | ) | |
| 106 | .all(repo.id, at); | |
| 107 | if (spent.length === 0) return 0; | |
| 108 | let closed: Set<number> | null = null; | |
| 109 | const closedByMerge = (): Set<number> => { | |
| 110 | if (closed !== null) return closed; | |
| 111 | closed = new Set(); | |
| 112 | const bodies = app.db | |
| 113 | .query<{ body: string }, [string]>( | |
| 114 | `SELECT i.body FROM pull q JOIN issue i ON i.id = q.issue_id | |
| 115 | WHERE i.repo_id = ?1 AND q.merged_at IS NOT NULL AND i.body LIKE '%#%'`, | |
| 116 | ) | |
| 117 | .all(repo.id); | |
| 118 | for (const b of bodies) for (const n of closesFromBody(b.body)) closed.add(n); | |
| 119 | return closed; | |
| 120 | }; | |
| 121 | let usd = 0; | |
| 122 | for (const s of spent) { | |
| 123 | if (s.kind === "pull" ? s.merged === 0 : !closedByMerge().has(s.number)) usd += s.usd ?? 0; | |
| 124 | } | |
| 125 | return usd; | |
| 87 | 126 | } |
| 88 | 127 | |
| 89 | 128 | /** The week's efficiency: since the week opened, or seven days back when no window is known. */ |
| @@ -146,7 +185,7 @@ function busiest<T>(app: App, now: Date, read: (repo: Repo, week: WeekDecision) | ||
| 146 | 185 | * terminal's dashboard print the same words. |
| 147 | 186 | */ |
| 148 | 187 | export function weekWords(app: App, now: Date = new Date()): string | null { |
| 149 | return busiest(app, now, (repo, week) => `${week.sentence} ${efficiencyWords(weekEfficiency(app, repo, now))}${heldWords(app)}`); | |
| 188 | return busiest(app, now, (repo, week) => `${week.sentence} ${efficiencyWords(weekEfficiency(app, repo, now), week)}${heldWords(app)}`); | |
| 150 | 189 | } |
| 151 | 190 | |
| 152 | 191 | /** The owner's week on Home and the terminal. */ |
test/week-wiring.test.ts +14
| @@ -20,6 +20,7 @@ import { openIssue } from "../src/actions/issues.ts"; | ||
| 20 | 20 | import { getAgent } from "../src/core/agents.ts"; |
| 21 | 21 | import { changeCapacity } from "../src/actions/capacity.ts"; |
| 22 | 22 | import { capacityDefault, setCapacity } from "../src/core/capacity.ts"; |
| 23 | import { id } from "../src/kernel/id.ts"; | |
| 23 | 24 | import { forgetWeek, weekWords } from "../src/crew/week_read.ts"; |
| 24 | 25 | import { createForgeApp } from "../src/tui/forge.ts"; |
| 25 | 26 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| @@ -279,3 +280,16 @@ test("the autonomy card shows the week and the pace as two rows, once, with no d | ||
| 279 | 280 | servers.stop(); |
| 280 | 281 | } |
| 281 | 282 | }); |
| 283 | ||
| 284 | test("home and the terminal's sentence says what share of the spend was lost and what the allowance left should buy", () => { | |
| 285 | week(50); | |
| 286 | const at = new Date(NOW.getTime() - DAY).toISOString(); | |
| 287 | const lost = openIssue(app, alice, repo, { title: "Lost work", body: "Never built." }); | |
| 288 | if (!lost.ok) throw new Error("issue not opened"); | |
| 289 | app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(lost.value.id); | |
| 290 | app.db.query("INSERT INTO run (id, repo_id, issue_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, ?3, 'agent', 'ok', 4, ?4)").run(id(), repo.id, lost.value.id, at); | |
| 291 | forgetWeek(); | |
| 292 | const words = weekWords(app, NOW)!; | |
| 293 | expect(words).toMatch(/Nothing has landed this week \(\d+ runs, \$[\d.]+\)\. \d+% of the spend \(\$4\.00\) has not landed\./); | |
| 294 | expect(String(weekLine(app, NOW))).toContain("has not landed"); | |
| 295 | }); |
test/week.test.ts +88 −3
| @@ -7,7 +7,7 @@ import { createApp } from "../src/app.ts"; | ||
| 7 | 7 | import { createRepo } from "../src/core/repos.ts"; |
| 8 | 8 | import { createUser } from "../src/core/users.ts"; |
| 9 | 9 | import { id } from "../src/kernel/id.ts"; |
| 10 | import { efficiencySince } from "../src/crew/week_read.ts"; | |
| 10 | import { efficiencySince, unlandedSince } from "../src/crew/week_read.ts"; | |
| 11 | 11 | import { efficiencyWords, weekDecision, type WeekInput } from "../src/crew/week.ts"; |
| 12 | 12 | |
| 13 | 13 | const DAY = 86_400_000; |
| @@ -75,8 +75,93 @@ describe("efficiency", () => { | ||
| 75 | 75 | for (const cost of [1, 2, 3, 4, 5, 6]) |
| 76 | 76 | app.db.query(`INSERT INTO run (id, repo_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, 'agent', 'ok', ?3, ?4)`).run(id(), repo.id, cost, at); |
| 77 | 77 | const e = efficiencySince(app, repo, new Date(NOW.getTime() - DAY)); |
| 78 | expect(e).toEqual({ landings: 2, runs: 6, costUsd: 21 }); | |
| 78 | expect(e).toEqual({ landings: 2, runs: 6, costUsd: 21, unlandedUsd: 0 }); | |
| 79 | 79 | expect(efficiencyWords(e)).toContain("$10.50 and 3 runs"); |
| 80 | expect(efficiencyWords({ landings: 0, runs: 2, costUsd: 4 })).toContain("Nothing has landed"); | |
| 80 | expect(efficiencyWords({ landings: 0, runs: 2, costUsd: 4, unlandedUsd: 0 })).toContain("Nothing has landed"); | |
| 81 | }); | |
| 82 | ||
| 83 | const seed = async () => { | |
| 84 | const app = createApp({ dataDir: mkdtempSync(join(tmpdir(), "forge-week-")), port: 0 }); | |
| 85 | const u = await createUser(app.db, { username: "alice", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true }); | |
| 86 | if (!u.ok) throw new Error(u.error); | |
| 87 | const made = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "r", visibility: "public", description: "r" }); | |
| 88 | if (!made.ok) throw new Error(made.error); | |
| 89 | const repo = made.value; | |
| 90 | const at = NOW.toISOString(); | |
| 91 | const issue = (n: number, kind: string, state: string, body = "") => { | |
| 92 | const issueId = id(); | |
| 93 | app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, ?4, 'T', ?5, ?6, ?7, ?8)`).run(issueId, repo.id, n, kind, body, u.value.id, state, at); | |
| 94 | return issueId; | |
| 95 | }; | |
| 96 | const merge = (issueId: string, n: number, mergedAt: string | null) => | |
| 97 | app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid, carried) VALUES (?1, ?2, ?3, 'base', '', ?4, ?5, 'new', 0)`).run(issueId, repo.defaultBranch, `b/${n}`, repo.id, mergedAt); | |
| 98 | const spend = (issueId: string | null, cost: number, created = at) => | |
| 99 | app.db.query(`INSERT INTO run (id, repo_id, issue_id, kind, state, cost_usd, created_at) VALUES (?1, ?2, ?3, 'agent', 'ok', ?4, ?5)`).run(id(), repo.id, issueId, cost, created); | |
| 100 | return { app, repo, at, issue, merge, spend }; | |
| 101 | }; | |
| 102 | const since = new Date(NOW.getTime() - DAY); | |
| 103 | ||
| 104 | test("spend on work that landed is not unlanded, and the words say what the allowance left should buy", async () => { | |
| 105 | const { app, repo, at, issue, merge, spend } = await seed(); | |
| 106 | const built = issue(1, "issue", "closed"); | |
| 107 | const lost = issue(2, "issue", "closed"); | |
| 108 | const pullId = issue(3, "pull", "closed", "Closes #1"); | |
| 109 | merge(pullId, 3, at); | |
| 110 | spend(built, 6); | |
| 111 | spend(pullId, 2); | |
| 112 | spend(lost, 2); | |
| 113 | const e = efficiencySince(app, repo, since); | |
| 114 | expect(e).toEqual({ landings: 1, runs: 3, costUsd: 10, unlandedUsd: 2 }); | |
| 115 | const words = efficiencyWords(e, { usedPct: 50, mayUsePct: 90 }); | |
| 116 | expect(words).toContain("20% of the spend ($2.00) has not landed"); | |
| 117 | expect(words).toContain("0.02 landed per point of the week used; the 40 points left should buy about 1 more"); | |
| 118 | expect(efficiencyWords(e)).not.toContain("points left"); | |
| 119 | }); | |
| 120 | ||
| 121 | test("a pull closed without merging is lost; one in flight, and one that merged before the window, are not", async () => { | |
| 122 | const { app, repo, issue, merge, spend } = await seed(); | |
| 123 | const dead = issue(1, "pull", "closed"); | |
| 124 | merge(dead, 1, null); | |
| 125 | const live = issue(2, "pull", "open"); | |
| 126 | merge(live, 2, null); | |
| 127 | const early = issue(3, "pull", "closed", "Closes #4"); | |
| 128 | merge(early, 3, new Date(NOW.getTime() - 30 * DAY).toISOString()); | |
| 129 | const earlyIssue = issue(4, "issue", "closed"); | |
| 130 | const open = issue(5, "issue", "open"); | |
| 131 | spend(dead, 3); | |
| 132 | spend(live, 5); | |
| 133 | spend(early, 7); | |
| 134 | spend(earlyIssue, 11); | |
| 135 | spend(open, 13); | |
| 136 | expect(unlandedSince(app, repo, since.toISOString())).toBe(3); | |
| 137 | }); | |
| 138 | ||
| 139 | test("runs with no issue are left out of the lost figure", async () => { | |
| 140 | const { app, repo, issue, spend } = await seed(); | |
| 141 | spend(null, 9); | |
| 142 | spend(issue(1, "issue", "closed"), 1); | |
| 143 | expect(unlandedSince(app, repo, since.toISOString())).toBe(1); | |
| 144 | expect(efficiencySince(app, repo, since)).toEqual({ landings: 0, runs: 2, costUsd: 10, unlandedUsd: 1 }); | |
| 145 | }); | |
| 146 | ||
| 147 | test("with nothing landed the words still say what was lost, and the allowance left clamps at zero", () => { | |
| 148 | const none = { landings: 0, runs: 4, costUsd: 8, unlandedUsd: 2 }; | |
| 149 | expect(efficiencyWords(none, { usedPct: 50, mayUsePct: 90 })).toBe("Nothing has landed this week (4 runs, $8.00). 25% of the spend ($2.00) has not landed."); | |
| 150 | const some = { landings: 3, runs: 6, costUsd: 12, unlandedUsd: 0 }; | |
| 151 | expect(efficiencyWords(some, { usedPct: 0, mayUsePct: 90 })).not.toContain("points left"); | |
| 152 | expect(efficiencyWords(some, { usedPct: 95, mayUsePct: 90 })).toContain("the 0 points left should buy about 0 more"); | |
| 153 | }); | |
| 154 | ||
| 155 | test("one query reads a large run table in well under 50 ms", async () => { | |
| 156 | const { app, repo, at, issue, merge, spend } = await seed(); | |
| 157 | const closed = Array.from({ length: 200 }, (_, k) => issue(k + 1, k % 2 ? "issue" : "pull", "closed", k % 2 ? "" : `Closes #${k + 2}`)); | |
| 158 | closed.forEach((c, k) => k % 2 === 0 && merge(c, k + 1, at)); | |
| 159 | app.db.transaction(() => { | |
| 160 | for (let k = 0; k < 20_000; k++) spend(closed[k % 200]!, 1); | |
| 161 | })(); | |
| 162 | const t = performance.now(); | |
| 163 | const usd = unlandedSince(app, repo, since.toISOString()); | |
| 164 | expect(performance.now() - t).toBeLessThan(50); | |
| 165 | expect(usd).toBe(0); | |
| 81 | 166 | }); |
| 82 | 167 | }); |
new above
offline