Public: anyone can read this; the crew can change it.
The apps shelf reads each app's newest crew run in under 10 ms: /trevato and /apps spend 320–400 ms in shelf-runs today #8402
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8379-the-apps-shelf-reads-each-app-s-newest-c → main 3 commits fork 1 thread · 1 unresolved
4 files
changed +123
−14
src/core/migration-pins/20261102170000-run-agent-latest.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 9b90c3cffc14eab72c80ef0fa70be3c76db145108118dbf394c4e44eb134f703 |
src/core/migrations/20261102170000-run-agent-latest.ts added +7
| @@ -0,0 +1,7 @@ | ||
| 1 | // The apps shelf asks for each repository's newest agent run (#8379). A | |
| 2 | // window over every agent run of every repository on the shelf cost | |
| 3 | // 320–400 ms; this index makes each repository's newest one a seek. | |
| 4 | // Additive: an index build, no table rewrite. | |
| 5 | export default ` | |
| 6 | CREATE INDEX IF NOT EXISTS run_agent_latest ON run(repo_id, created_at DESC, id DESC) WHERE kind = 'agent'; | |
| 7 | `; |
src/core/runs.ts +21 −14
| @@ -1189,26 +1189,33 @@ export function agentRunTotals( | ||
| 1189 | 1189 | return { runs: row?.runs ?? 0, spentUsd: row?.spent ?? 0 }; |
| 1190 | 1190 | } |
| 1191 | 1191 | |
| 1192 | /** The newest agent run of each repository, in one query. */ | |
| 1192 | /** | |
| 1193 | * The one statement behind `latestAgentRunsForRepos`, exported so its test | |
| 1194 | * explains this exact text and fails when it stops seeking `run_agent_latest`. | |
| 1195 | * `?1` is a JSON array of repository ids. | |
| 1196 | */ | |
| 1197 | export const LATEST_AGENT_RUNS_SQL = `SELECT run.* FROM run WHERE run.id IN ( | |
| 1198 | SELECT (SELECT r.id FROM run r | |
| 1199 | WHERE r.repo_id = ids.value AND r.kind = 'agent' | |
| 1200 | ORDER BY r.created_at DESC, r.id DESC LIMIT 1) | |
| 1201 | FROM json_each(?1) AS ids)`; | |
| 1202 | ||
| 1203 | /** | |
| 1204 | * The newest agent run of each repository, in one query: per repository one | |
| 1205 | * seek into `run_agent_latest`, never a window over every agent run (#8379). | |
| 1206 | * Ties on `created_at` break by the larger id, as they always have. | |
| 1207 | */ | |
| 1193 | 1208 | export function latestAgentRunsForRepos( |
| 1194 | 1209 | db: Database, |
| 1195 | 1210 | repoIds: readonly string[], |
| 1196 | 1211 | ): Map<string, Run> { |
| 1197 | 1212 | const out = new Map<string, Run>(); |
| 1198 | 1213 | const unique = [...new Set(repoIds)]; |
| 1199 | const CHUNK = 400; | |
| 1200 | for (let i = 0; i < unique.length; i += CHUNK) { | |
| 1201 | const batch = unique.slice(i, i + CHUNK); | |
| 1202 | const rows = db | |
| 1203 | .query<RunRow, string[]>( | |
| 1204 | `SELECT * FROM ( | |
| 1205 | SELECT run.*, ROW_NUMBER() OVER (PARTITION BY repo_id ORDER BY created_at DESC, id DESC) AS rn | |
| 1206 | FROM run WHERE kind = 'agent' AND repo_id IN (${batch.map((_, n) => `?${n + 1}`).join(",")}) | |
| 1207 | ) WHERE rn = 1`, | |
| 1208 | ) | |
| 1209 | .all(...batch); | |
| 1210 | for (const row of rows) out.set(row.repo_id, toRun(row)); | |
| 1211 | } | |
| 1214 | if (unique.length === 0) return out; | |
| 1215 | const rows = db | |
| 1216 | .query<RunRow, [string]>(LATEST_AGENT_RUNS_SQL) | |
| 1217 | .all(JSON.stringify(unique)); | |
| 1218 | for (const row of rows) out.set(row.repo_id, toRun(row)); | |
| 1212 | 1219 | return out; |
| 1213 | 1220 | } |
| 1214 | 1221 |
test/runs-latest.test.ts added +94
| @@ -0,0 +1,94 @@ | ||
| 1 | /** | |
| 2 | * The apps shelf reads each repository's newest agent run (#8379): one seek | |
| 3 | * per repository into `run_agent_latest`, the same answer the window over | |
| 4 | * every agent run gave, ties included. The cost is bounded by work, not a | |
| 5 | * clock (#7824): one statement for the whole shelf, and a plan that seeks the | |
| 6 | * index with no sort, so it does not grow with a repository's run count. | |
| 7 | */ | |
| 8 | import { expect, test } from "bun:test"; | |
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { LATEST_AGENT_RUNS_SQL, latestAgentRunsForRepos } from "../src/core/runs.ts"; | |
| 11 | import { migrations } from "../src/core/schema.ts"; | |
| 12 | import { openDb } from "../src/kernel/db.ts"; | |
| 13 | import { statements } from "./helpers/statements.ts"; | |
| 14 | import { expectWork } from "./helpers/work.ts"; | |
| 15 | ||
| 16 | /** The query the shelf ran before #8379, kept here as the oracle. */ | |
| 17 | function windowed(db: Database, repoIds: readonly string[]): Map<string, string> { | |
| 18 | const rows = db | |
| 19 | .query<{ id: string; repo_id: string }, string[]>( | |
| 20 | `SELECT id, repo_id FROM ( | |
| 21 | SELECT run.*, ROW_NUMBER() OVER (PARTITION BY repo_id ORDER BY created_at DESC, id DESC) AS rn | |
| 22 | FROM run WHERE kind = 'agent' AND repo_id IN (${repoIds.map((_, n) => `?${n + 1}`).join(",")}) | |
| 23 | ) WHERE rn = 1`, | |
| 24 | ) | |
| 25 | .all(...repoIds); | |
| 26 | return new Map(rows.map((r) => [r.repo_id, r.id])); | |
| 27 | } | |
| 28 | ||
| 29 | function seeded(): { db: Database; repoIds: string[] } { | |
| 30 | const db = openDb(":memory:", migrations); | |
| 31 | db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1','u','u@x','x')"); | |
| 32 | const repoIds = Array.from({ length: 20 }, (_, i) => `r${String(i).padStart(2, "0")}`); | |
| 33 | const repo = db.prepare("INSERT INTO repo (id, owner_id, name) VALUES (?1, 'u1', ?1)"); | |
| 34 | for (const id of repoIds) repo.run(id); | |
| 35 | const run = db.prepare("INSERT INTO run (id, repo_id, kind, created_at) VALUES (?1, ?2, ?3, ?4)"); | |
| 36 | const at = (n: number) => new Date(Date.UTC(2026, 9, 1) + n * 60_000).toISOString(); | |
| 37 | db.transaction(() => { | |
| 38 | // 5,000 agent runs on one repository, with deploys and checks between them. | |
| 39 | for (let n = 0; n < 5000; n++) { | |
| 40 | run.run(`a-${String(n).padStart(5, "0")}`, "r00", "agent", at(n)); | |
| 41 | if (n % 10 === 0) run.run(`d-${n}`, "r00", "deploy", at(n + 10_000)); | |
| 42 | } | |
| 43 | // Two agent runs at the same instant: the larger id wins, as before. | |
| 44 | run.run("t-a", "r01", "agent", at(5)); | |
| 45 | run.run("t-b", "r01", "agent", at(5)); | |
| 46 | run.run("t-0", "r01", "agent", at(1)); | |
| 47 | // A newer check never stands in for the newest agent run. | |
| 48 | run.run("c-1", "r02", "agent", at(1)); | |
| 49 | run.run("c-2", "r02", "check", at(9)); | |
| 50 | // r03 has only a deploy; r04..r19 one or two agent runs each. | |
| 51 | run.run("d-only", "r03", "deploy", at(3)); | |
| 52 | for (let i = 4; i < 20; i++) { | |
| 53 | run.run(`x-${i}-1`, repoIds[i]!, "agent", at(i)); | |
| 54 | if (i % 2 === 0) run.run(`x-${i}-0`, repoIds[i]!, "agent", at(i - 1)); | |
| 55 | } | |
| 56 | })(); | |
| 57 | return { db, repoIds }; | |
| 58 | } | |
| 59 | ||
| 60 | test("the newest agent run of each repository is the one the window picked, ties included", () => { | |
| 61 | const { db, repoIds } = seeded(); | |
| 62 | const got = latestAgentRunsForRepos(db, repoIds); | |
| 63 | expect(new Map([...got].map(([repo, r]) => [repo, r.id]))).toEqual(windowed(db, repoIds)); | |
| 64 | expect(got.get("r00")?.id).toBe("a-04999"); | |
| 65 | expect(got.get("r01")?.id).toBe("t-b"); | |
| 66 | expect(got.get("r02")?.id).toBe("c-1"); | |
| 67 | expect(got.has("r03")).toBe(false); | |
| 68 | expect(latestAgentRunsForRepos(db, [])).toEqual(new Map()); | |
| 69 | expect(latestAgentRunsForRepos(db, ["r01", "r01", "nope"]).size).toBe(1); | |
| 70 | }); | |
| 71 | ||
| 72 | test("the read is one statement that seeks run_agent_latest, however many runs", async () => { | |
| 73 | const { db, repoIds } = seeded(); | |
| 74 | const plan = db | |
| 75 | .query<{ detail: string }, [string]>(`EXPLAIN QUERY PLAN ${LATEST_AGENT_RUNS_SQL}`) | |
| 76 | .all(JSON.stringify(repoIds)) | |
| 77 | .map((r) => r.detail) | |
| 78 | .join("\n"); | |
| 79 | expect(plan).not.toContain("TEMP B-TREE"); | |
| 80 | expect(plan).toContain("COVERING INDEX run_agent_latest (repo_id=?)"); | |
| 81 | expect(plan).not.toContain("SCAN r"); | |
| 82 | expectWork(await statements(db, () => void latestAgentRunsForRepos(db, repoIds)), { max: 1 }); | |
| 83 | // The statement explained above is the one the function runs, not a copy. | |
| 84 | const ran: string[] = []; | |
| 85 | const door = db as unknown as { query: (sql: string) => unknown }; | |
| 86 | const original = db.query.bind(db); | |
| 87 | door.query = (sql: string) => (ran.push(sql), original(sql)); | |
| 88 | try { | |
| 89 | latestAgentRunsForRepos(db, repoIds); | |
| 90 | } finally { | |
| 91 | door.query = original; | |
| 92 | } | |
| 93 | expect(ran).toEqual([LATEST_AGENT_RUNS_SQL]); | |
| 94 | }); |
new above
offline
Lift this SQL into an exported constant and have the test EXPLAIN that constant instead of a pasted copy, so the plan test fails when this function stops using the index.