Public: anyone can read this; the crew can change it.
Repository pages: two indexes and four query fixes take 17% off their SQL; and one-file-per-repository becomes the archive format, not the live storage #12961
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12960-repository-pages-two-indexes-and-four-qu → main 5 commits fork
7 files
changed +142
−10
src/core/from_app.ts +1 −1
| @@ -58,7 +58,7 @@ export function waitingFromApp(db: Database, repoId?: string): WaitingFromApp[] | ||
| 58 | 58 | JOIN issue_label il ON il.issue_id = i.id |
| 59 | 59 | JOIN label l ON l.id = il.label_id AND l.name = ?2 COLLATE NOCASE |
| 60 | 60 | WHERE i.kind = 'issue' AND i.state = 'open' AND n.hidden = 0 AND n.visitor LIKE 'u:%' |
| 61 | AND (?1 IS NULL OR i.repo_id = ?1) | |
| 61 | ${repoId === undefined ? "AND ?1 IS NULL" : "AND i.repo_id = ?1"} | |
| 62 | 62 | GROUP BY i.id |
| 63 | 63 | ORDER BY i.created_at, i.id`, |
| 64 | 64 | ) |
src/core/migration-pins/20261115100000-usage-issue-review-approved.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 699e2c008d4a64cdf149b9cbc1ecfadb826dfd25fde7871e233df3c2826afdc0 |
src/core/migrations/20261115100000-usage-issue-review-approved.ts added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | // Two reads that walked a whole table to answer for one issue (#12960): | |
| 2 | // `issueUsage` summed `usage` by issue with no index on it (every row, six | |
| 3 | // times a page), and the landing line's latest approval per pull read every | |
| 4 | // approved review. Partial, so the indexes hold only the rows those reads | |
| 5 | // want. Idempotent. Additive. Undone with two DROP INDEX. | |
| 6 | export default ` | |
| 7 | CREATE INDEX IF NOT EXISTS usage_issue ON usage(issue_id) WHERE issue_id IS NOT NULL; | |
| 8 | CREATE INDEX IF NOT EXISTS review_approved_at ON review(issue_id, created_at) WHERE state = 'approved'; | |
| 9 | `; |
src/flow/alarm_lane.ts +2 −2
| @@ -108,10 +108,10 @@ function groups(failures: readonly Failure[]): Group[] { | ||
| 108 | 108 | } |
| 109 | 109 | |
| 110 | 110 | /** When a pull of the repository last merged. */ |
| 111 | function lastMerge(db: Database, repoId: string): string | null { | |
| 111 | export function lastMerge(db: Database, repoId: string): string | null { | |
| 112 | 112 | return ( |
| 113 | 113 | db |
| 114 | .query<{ at: string | null }, [string]>(`SELECT MAX(p.merged_at) AS at FROM pull p JOIN issue i ON i.id = p.issue_id WHERE i.repo_id = ?1`) | |
| 114 | .query<{ at: string | null }, [string]>(`SELECT MAX(p.merged_at) AS at FROM pull p WHERE p.issue_id IN (SELECT id FROM issue WHERE repo_id = ?1)`) | |
| 115 | 115 | .get(repoId)?.at ?? null |
| 116 | 116 | ); |
| 117 | 117 | } |
src/flow/bugs.ts +6 −6
| @@ -41,21 +41,21 @@ interface BugRow { | ||
| 41 | 41 | * The one open-bug query. `named` joins the repository and its owner (for the |
| 42 | 42 | * alarm); the count needs neither, and without them it reads only issue and labels. |
| 43 | 43 | */ |
| 44 | function readBugRows(db: Database, named: boolean, publicOnly: boolean): BugRow[] { | |
| 44 | function readBugRows(db: Database, named: boolean, publicOnly: boolean, repoId?: string): BugRow[] { | |
| 45 | 45 | const repo = named ? "JOIN repo r ON r.id = i.repo_id JOIN user u ON u.id = r.owner_id" : ""; |
| 46 | 46 | const who = named ? "u.username AS owner, r.name AS name" : "NULL AS owner, NULL AS name"; |
| 47 | 47 | return db |
| 48 | .query<BugRow, []>( | |
| 48 | .query<BugRow, [string | null]>( | |
| 49 | 49 | `SELECT ${who}, ${named ? "i.number" : "0 AS number"}, i.created_at, |
| 50 | 50 | EXISTS (SELECT 1 FROM issue_label h JOIN label hl ON hl.id = h.label_id |
| 51 | 51 | WHERE h.issue_id = i.id AND hl.name COLLATE NOCASE IN ${HIGH_NAMES}) AS high |
| 52 | 52 | FROM issue i ${repo} |
| 53 | 53 | JOIN issue_label il ON il.issue_id = i.id |
| 54 | 54 | JOIN label l ON l.id = il.label_id AND l.name = 'bug' COLLATE NOCASE |
| 55 | WHERE i.kind = 'issue' AND i.state = 'open' ${named && publicOnly ? "AND r.visibility = 'public' AND i.sealed = 0" : ""} | |
| 55 | WHERE i.kind = 'issue' AND i.state = 'open' ${repoId === undefined ? "AND ?1 IS NULL" : "AND i.repo_id = ?1"} ${named && publicOnly ? "AND r.visibility = 'public' AND i.sealed = 0" : ""} | |
| 56 | 56 | ORDER BY i.created_at`, |
| 57 | 57 | ) |
| 58 | .all(); | |
| 58 | .all(repoId ?? null); | |
| 59 | 59 | } |
| 60 | 60 | |
| 61 | 61 | const dueMs = (r: BugRow): number => Date.parse(r.created_at) + (r.high === 1 ? HIGH_BUG_MS : LOW_BUG_MS); |
| @@ -66,8 +66,8 @@ const dueMs = (r: BugRow): number => Date.parse(r.created_at) + (r.high === 1 ? | ||
| 66 | 66 | * repository is its writers' (`core/sealed.ts`), and a reader who cannot open |
| 67 | 67 | * it is not told its number either. |
| 68 | 68 | */ |
| 69 | export function openBugs(db: Database, opts: { readonly publicOnly?: boolean } = {}): OpenBug[] { | |
| 70 | return readBugRows(db, true, opts.publicOnly === true).map((r) => ({ | |
| 69 | export function openBugs(db: Database, opts: { readonly publicOnly?: boolean; readonly repoId?: string } = {}): OpenBug[] { | |
| 70 | return readBugRows(db, true, opts.publicOnly === true, opts.repoId).map((r) => ({ | |
| 71 | 71 | repo: `${r.owner}/${r.name}`, |
| 72 | 72 | number: r.number, |
| 73 | 73 | createdAt: r.created_at, |
src/flow/landing_line.ts +1 −1
| @@ -32,7 +32,7 @@ export function landingLine( | ||
| 32 | 32 | .query<{ id: string; number: number; at: string }, [string]>( |
| 33 | 33 | `SELECT i.id AS id, i.number AS number, a.at AS at |
| 34 | 34 | FROM pull p JOIN issue i ON i.id = p.issue_id |
| 35 | JOIN (SELECT issue_id, MAX(created_at) AS at FROM review WHERE state = 'approved' GROUP BY issue_id) a | |
| 35 | JOIN (SELECT issue_id, MAX(created_at) AS at FROM review WHERE state = 'approved' AND issue_id IN (SELECT id FROM issue WHERE repo_id = ?1) GROUP BY issue_id) a | |
| 36 | 36 | ON a.issue_id = i.id |
| 37 | 37 | WHERE i.repo_id = ?1 AND i.state = 'open' AND p.merged_at IS NULL |
| 38 | 38 | AND NOT EXISTS (SELECT 1 FROM review c WHERE c.issue_id = i.id AND c.state = 'changes_requested' AND c.created_at > a.at) |
test/repo-bound-reads.test.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** | |
| 2 | * A repository page's reads are bound by the repository (#12960): none scans a | |
| 3 | * table that holds every repository's rows (issue, pull, review, usage, | |
| 4 | * comment, run, event) without an index. Judged by `EXPLAIN QUERY PLAN` of the | |
| 5 | * SQL the functions run, on a database that holds two repositories' rows, never | |
| 6 | * the clock, as `hot-reads-indexed.test.ts` does. | |
| 7 | */ | |
| 8 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 9 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { unansweredFromApp, waitingFromApp } from "../src/core/from_app.ts"; | |
| 14 | import { issueUsage } from "../src/core/usage.ts"; | |
| 15 | import { landingLine } from "../src/flow/landing_line.ts"; | |
| 16 | ||
| 17 | let root = ""; | |
| 18 | let app: App; | |
| 19 | ||
| 20 | const OLD = "2026-01-01T00:00:00.000Z"; | |
| 21 | const NEW = "2026-11-01T00:00:00.000Z"; | |
| 22 | ||
| 23 | /** | |
| 24 | * Two repositories, `trevato/forge` (a) and `other/x` (b), each with issues, | |
| 25 | * pulls, labels, reviews and spend. | |
| 26 | */ | |
| 27 | function seed(): void { | |
| 28 | const db = app.db; | |
| 29 | db.exec("PRAGMA foreign_keys = OFF"); | |
| 30 | const run = (sql: string, ...args: (string | number | null)[]) => void db.query(sql).run(...args); | |
| 31 | for (const [id, name] of [["ua", "trevato"], ["ub", "other"]] as const) { | |
| 32 | run(`INSERT INTO user (id, username, email, password_hash) VALUES (?, ?, ?, 'x')`, id, name, `${name}@x.test`); | |
| 33 | } | |
| 34 | run(`INSERT INTO user (id, username, email, password_hash, is_agent) VALUES ('ag', 'bot', '[email protected]', 'x', 1)`); | |
| 35 | run(`INSERT INTO repo (id, owner_id, name) VALUES ('a', 'ua', 'forge'), ('b', 'ub', 'x')`); | |
| 36 | for (const r of ["a", "b"]) run(`INSERT INTO label (id, repo_id, name) VALUES (?, ?, 'from-app')`, `l-${r}`, r); | |
| 37 | // Issues 1-3 of each repository; 1 is a from-app issue, 2 an open pull approved, 3 a pull merged. | |
| 38 | for (const r of ["a", "b"]) { | |
| 39 | run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 1, 'issue', 't', 'ua', ?)`, `${r}1`, r, OLD); | |
| 40 | run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 2, 'pull', 't', 'ua', ?)`, `${r}2`, r, OLD); | |
| 41 | run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 3, 'pull', 't', 'ua', ?)`, `${r}3`, r, OLD); | |
| 42 | run(`INSERT INTO issue_label (issue_id, label_id) VALUES (?, ?)`, `${r}1`, `l-${r}`); | |
| 43 | run(`INSERT INTO app_note (id, app_id, repo_id, issue_number, visitor, name, created_at) VALUES (?, 'app', ?, 1, 'u:7', 'n', ?)`, `n-${r}`, r, OLD); | |
| 44 | for (const n of [2, 3]) { | |
| 45 | run(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?, 'main', 'h', 'o', 'o', ?)`, `${r}${n}`, n === 3 ? (r === "a" ? OLD : NEW) : null); | |
| 46 | } | |
| 47 | run(`INSERT INTO review (id, issue_id, author_id, state, created_at) VALUES (?, ?, 'ua', 'approved', ?)`, `rv-${r}`, `${r}2`, OLD); | |
| 48 | run(`INSERT INTO usage (run_id, repo_id, issue_id, outcome, at, tokens_in, tokens_out, cost_usd) VALUES (?, ?, ?, 'ok', ?, 10, 5, ?)`, `u-${r}1`, r, `${r}1`, OLD, r === "a" ? 1 : 2); | |
| 49 | run(`INSERT INTO usage (run_id, repo_id, issue_id, outcome, at, tokens_in, tokens_out, cost_usd) VALUES (?, ?, NULL, 'ok', ?, 1, 1, 9)`, `u-${r}0`, r, OLD); | |
| 50 | } | |
| 51 | db.exec("PRAGMA foreign_keys = ON"); | |
| 52 | } | |
| 53 | ||
| 54 | beforeAll(() => { | |
| 55 | root = mkdtempSync(join(tmpdir(), "forge-repo-bound-")); | |
| 56 | app = createApp({ dataDir: root, port: 0 }); | |
| 57 | seed(); | |
| 58 | }); | |
| 59 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 60 | ||
| 61 | /** | |
| 62 | * The plan lines of every SELECT `work` runs (only those containing `only`), | |
| 63 | * parameters bound as text. | |
| 64 | */ | |
| 65 | function plans(work: () => void, only = ""): string[] { | |
| 66 | const db = app.db; | |
| 67 | const original = db.query.bind(db); | |
| 68 | const sqls: string[] = []; | |
| 69 | (db as unknown as { query: unknown }).query = (sql: string) => { | |
| 70 | sqls.push(sql); | |
| 71 | return original(sql); | |
| 72 | }; | |
| 73 | try { | |
| 74 | work(); | |
| 75 | } finally { | |
| 76 | (db as unknown as { query: unknown }).query = original; | |
| 77 | } | |
| 78 | return sqls | |
| 79 | .filter((sql) => /^\s*SELECT/i.test(sql) && sql.includes(only)) | |
| 80 | .flatMap((sql) => { | |
| 81 | const n = Math.max(0, ...[...sql.matchAll(/\?(\d+)/g)].map((m) => Number(m[1]))); | |
| 82 | return original(`EXPLAIN QUERY PLAN ${sql}`) | |
| 83 | .all(...Array.from({ length: n }, () => "x")) | |
| 84 | .map((row) => (row as { detail: string }).detail); | |
| 85 | }); | |
| 86 | } | |
| 87 | ||
| 88 | /** Tables that hold every repository's rows: a scan of one is a read of the forge. */ | |
| 89 | const WIDE = /^SCAN (issue|pull|review|usage|comment|run|event|check_gate|issue_label|app_note)\b(?!.*USING (COVERING )?INDEX)/; | |
| 90 | const noWideScan = (lines: string[]) => { | |
| 91 | expect(lines.length).toBeGreaterThan(0); | |
| 92 | for (const line of lines) expect(line).not.toMatch(WIDE); | |
| 93 | }; | |
| 94 | ||
| 95 | const pull = { id: "a2", number: 2, state: "open", mergedAt: null }; | |
| 96 | ||
| 97 | test("an issue's spend is its own, read from the usage index", () => { | |
| 98 | expect(issueUsage(app.db, "a1")).toEqual({ runs: 1, tokensIn: 10, tokensOut: 5, costUsd: 1 }); | |
| 99 | expect(issueUsage(app.db, "b1")?.costUsd).toBe(2); | |
| 100 | expect(issueUsage(app.db, "nobody")).toBeUndefined(); | |
| 101 | const lines = plans(() => void issueUsage(app.db, "a1")); | |
| 102 | expect(lines.join("\n")).toMatch(/SEARCH usage USING (COVERING )?INDEX usage_issue/); | |
| 103 | noWideScan(lines); | |
| 104 | }); | |
| 105 | ||
| 106 | test("the landing line reads one repository's approvals, by index", () => { | |
| 107 | expect(landingLine(app.db, pull, "a")?.says).toStartWith("first in the queue"); | |
| 108 | expect(landingLine(app.db, { ...pull, id: "b2" }, "a")).toBeNull(); | |
| 109 | const lines = plans(() => void landingLine(app.db, pull, "a"), "state = 'approved'"); | |
| 110 | expect(lines.join("\n")).toMatch(/SEARCH review USING (COVERING )?INDEX review_approved_at/); | |
| 111 | noWideScan(lines); | |
| 112 | }); | |
| 113 | ||
| 114 | test("a repository's waiting from-app issues are its own, bound by index", () => { | |
| 115 | expect(unansweredFromApp(app.db, "a")?.repoId).toBe("a"); | |
| 116 | expect(unansweredFromApp(app.db, "b")?.repoId).toBe("b"); | |
| 117 | expect(unansweredFromApp(app.db, "none")).toBeUndefined(); | |
| 118 | expect(waitingFromApp(app.db).map((w) => w.repoId).sort()).toEqual(["a", "b"]); | |
| 119 | const lines = plans(() => void unansweredFromApp(app.db, "a")); | |
| 120 | expect(lines.join("\n")).toMatch(/SEARCH i USING (COVERING )?INDEX issue_\w+ \(repo_id=\?\)/); | |
| 121 | noWideScan(lines); | |
| 122 | }); |
new above
offline