Public: anyone can read this; the crew can change it.

Repository pages: two indexes and four query fixes take 17% off their SQL; and one-file-per-repository becomes the archive format, not the live storage #12961

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12960-repository-pages-two-indexes-and-four-qu → main 5 commits fork

  • Merged. mason/12960-repository-pages-two-indexes-and-four-qu was merged into main as fb3637c.

Review

Sign in to review this change.
7 files changed +142 −10
src/core/from_app.ts +1 −1
@@ -58,7 +58,7 @@ export function waitingFromApp(db: Database, repoId?: string): WaitingFromApp[]
5858 JOIN issue_label il ON il.issue_id = i.id
5959 JOIN label l ON l.id = il.label_id AND l.name = ?2 COLLATE NOCASE
6060 WHERE i.kind = 'issue' AND i.state = 'open' AND n.hidden = 0 AND n.visitor LIKE 'u:%'
61 AND (?1 IS NULL OR i.repo_id = ?1)
61 ${repoId === undefined ? "AND ?1 IS NULL" : "AND i.repo_id = ?1"}
6262 GROUP BY i.id
6363 ORDER BY i.created_at, i.id`,
6464 )
src/core/migration-pins/20261115100000-usage-issue-review-approved.pin added +1
@@ -0,0 +1,1 @@
1 699e2c008d4a64cdf149b9cbc1ecfadb826dfd25fde7871e233df3c2826afdc0
src/core/migrations/20261115100000-usage-issue-review-approved.ts added +9
@@ -0,0 +1,9 @@
1 // Two reads that walked a whole table to answer for one issue (#12960):
2 // `issueUsage` summed `usage` by issue with no index on it (every row, six
3 // times a page), and the landing line's latest approval per pull read every
4 // approved review. Partial, so the indexes hold only the rows those reads
5 // want. Idempotent. Additive. Undone with two DROP INDEX.
6 export default `
7 CREATE INDEX IF NOT EXISTS usage_issue ON usage(issue_id) WHERE issue_id IS NOT NULL;
8 CREATE INDEX IF NOT EXISTS review_approved_at ON review(issue_id, created_at) WHERE state = 'approved';
9 `;
src/flow/alarm_lane.ts +2 −2
@@ -108,10 +108,10 @@ function groups(failures: readonly Failure[]): Group[] {
108108 }
109109
110110 /** When a pull of the repository last merged. */
111 function lastMerge(db: Database, repoId: string): string | null {
111 export function lastMerge(db: Database, repoId: string): string | null {
112112 return (
113113 db
114 .query<{ at: string | null }, [string]>(`SELECT MAX(p.merged_at) AS at FROM pull p JOIN issue i ON i.id = p.issue_id WHERE i.repo_id = ?1`)
114 .query<{ at: string | null }, [string]>(`SELECT MAX(p.merged_at) AS at FROM pull p WHERE p.issue_id IN (SELECT id FROM issue WHERE repo_id = ?1)`)
115115 .get(repoId)?.at ?? null
116116 );
117117 }
src/flow/bugs.ts +6 −6
@@ -41,21 +41,21 @@ interface BugRow {
4141 * The one open-bug query. `named` joins the repository and its owner (for the
4242 * alarm); the count needs neither, and without them it reads only issue and labels.
4343 */
44 function readBugRows(db: Database, named: boolean, publicOnly: boolean): BugRow[] {
44 function readBugRows(db: Database, named: boolean, publicOnly: boolean, repoId?: string): BugRow[] {
4545 const repo = named ? "JOIN repo r ON r.id = i.repo_id JOIN user u ON u.id = r.owner_id" : "";
4646 const who = named ? "u.username AS owner, r.name AS name" : "NULL AS owner, NULL AS name";
4747 return db
48 .query<BugRow, []>(
48 .query<BugRow, [string | null]>(
4949 `SELECT ${who}, ${named ? "i.number" : "0 AS number"}, i.created_at,
5050 EXISTS (SELECT 1 FROM issue_label h JOIN label hl ON hl.id = h.label_id
5151 WHERE h.issue_id = i.id AND hl.name COLLATE NOCASE IN ${HIGH_NAMES}) AS high
5252 FROM issue i ${repo}
5353 JOIN issue_label il ON il.issue_id = i.id
5454 JOIN label l ON l.id = il.label_id AND l.name = 'bug' COLLATE NOCASE
55 WHERE i.kind = 'issue' AND i.state = 'open' ${named && publicOnly ? "AND r.visibility = 'public' AND i.sealed = 0" : ""}
55 WHERE i.kind = 'issue' AND i.state = 'open' ${repoId === undefined ? "AND ?1 IS NULL" : "AND i.repo_id = ?1"} ${named && publicOnly ? "AND r.visibility = 'public' AND i.sealed = 0" : ""}
5656 ORDER BY i.created_at`,
5757 )
58 .all();
58 .all(repoId ?? null);
5959 }
6060
6161 const dueMs = (r: BugRow): number => Date.parse(r.created_at) + (r.high === 1 ? HIGH_BUG_MS : LOW_BUG_MS);
@@ -66,8 +66,8 @@ const dueMs = (r: BugRow): number => Date.parse(r.created_at) + (r.high === 1 ?
6666 * repository is its writers' (`core/sealed.ts`), and a reader who cannot open
6767 * it is not told its number either.
6868 */
69 export function openBugs(db: Database, opts: { readonly publicOnly?: boolean } = {}): OpenBug[] {
70 return readBugRows(db, true, opts.publicOnly === true).map((r) => ({
69 export function openBugs(db: Database, opts: { readonly publicOnly?: boolean; readonly repoId?: string } = {}): OpenBug[] {
70 return readBugRows(db, true, opts.publicOnly === true, opts.repoId).map((r) => ({
7171 repo: `${r.owner}/${r.name}`,
7272 number: r.number,
7373 createdAt: r.created_at,
src/flow/landing_line.ts +1 −1
@@ -32,7 +32,7 @@ export function landingLine(
3232 .query<{ id: string; number: number; at: string }, [string]>(
3333 `SELECT i.id AS id, i.number AS number, a.at AS at
3434 FROM pull p JOIN issue i ON i.id = p.issue_id
35 JOIN (SELECT issue_id, MAX(created_at) AS at FROM review WHERE state = 'approved' GROUP BY issue_id) a
35 JOIN (SELECT issue_id, MAX(created_at) AS at FROM review WHERE state = 'approved' AND issue_id IN (SELECT id FROM issue WHERE repo_id = ?1) GROUP BY issue_id) a
3636 ON a.issue_id = i.id
3737 WHERE i.repo_id = ?1 AND i.state = 'open' AND p.merged_at IS NULL
3838 AND NOT EXISTS (SELECT 1 FROM review c WHERE c.issue_id = i.id AND c.state = 'changes_requested' AND c.created_at > a.at)
test/repo-bound-reads.test.ts added +122
@@ -0,0 +1,122 @@
1 /**
2 * A repository page's reads are bound by the repository (#12960): none scans a
3 * table that holds every repository's rows (issue, pull, review, usage,
4 * comment, run, event) without an index. Judged by `EXPLAIN QUERY PLAN` of the
5 * SQL the functions run, on a database that holds two repositories' rows, never
6 * the clock, as `hot-reads-indexed.test.ts` does.
7 */
8 import { afterAll, beforeAll, expect, test } from "bun:test";
9 import { mkdtempSync, rmSync } from "node:fs";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { unansweredFromApp, waitingFromApp } from "../src/core/from_app.ts";
14 import { issueUsage } from "../src/core/usage.ts";
15 import { landingLine } from "../src/flow/landing_line.ts";
16
17 let root = "";
18 let app: App;
19
20 const OLD = "2026-01-01T00:00:00.000Z";
21 const NEW = "2026-11-01T00:00:00.000Z";
22
23 /**
24 * Two repositories, `trevato/forge` (a) and `other/x` (b), each with issues,
25 * pulls, labels, reviews and spend.
26 */
27 function seed(): void {
28 const db = app.db;
29 db.exec("PRAGMA foreign_keys = OFF");
30 const run = (sql: string, ...args: (string | number | null)[]) => void db.query(sql).run(...args);
31 for (const [id, name] of [["ua", "trevato"], ["ub", "other"]] as const) {
32 run(`INSERT INTO user (id, username, email, password_hash) VALUES (?, ?, ?, 'x')`, id, name, `${name}@x.test`);
33 }
34 run(`INSERT INTO user (id, username, email, password_hash, is_agent) VALUES ('ag', 'bot', '[email protected]', 'x', 1)`);
35 run(`INSERT INTO repo (id, owner_id, name) VALUES ('a', 'ua', 'forge'), ('b', 'ub', 'x')`);
36 for (const r of ["a", "b"]) run(`INSERT INTO label (id, repo_id, name) VALUES (?, ?, 'from-app')`, `l-${r}`, r);
37 // Issues 1-3 of each repository; 1 is a from-app issue, 2 an open pull approved, 3 a pull merged.
38 for (const r of ["a", "b"]) {
39 run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 1, 'issue', 't', 'ua', ?)`, `${r}1`, r, OLD);
40 run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 2, 'pull', 't', 'ua', ?)`, `${r}2`, r, OLD);
41 run(`INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, 3, 'pull', 't', 'ua', ?)`, `${r}3`, r, OLD);
42 run(`INSERT INTO issue_label (issue_id, label_id) VALUES (?, ?)`, `${r}1`, `l-${r}`);
43 run(`INSERT INTO app_note (id, app_id, repo_id, issue_number, visitor, name, created_at) VALUES (?, 'app', ?, 1, 'u:7', 'n', ?)`, `n-${r}`, r, OLD);
44 for (const n of [2, 3]) {
45 run(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?, 'main', 'h', 'o', 'o', ?)`, `${r}${n}`, n === 3 ? (r === "a" ? OLD : NEW) : null);
46 }
47 run(`INSERT INTO review (id, issue_id, author_id, state, created_at) VALUES (?, ?, 'ua', 'approved', ?)`, `rv-${r}`, `${r}2`, OLD);
48 run(`INSERT INTO usage (run_id, repo_id, issue_id, outcome, at, tokens_in, tokens_out, cost_usd) VALUES (?, ?, ?, 'ok', ?, 10, 5, ?)`, `u-${r}1`, r, `${r}1`, OLD, r === "a" ? 1 : 2);
49 run(`INSERT INTO usage (run_id, repo_id, issue_id, outcome, at, tokens_in, tokens_out, cost_usd) VALUES (?, ?, NULL, 'ok', ?, 1, 1, 9)`, `u-${r}0`, r, OLD);
50 }
51 db.exec("PRAGMA foreign_keys = ON");
52 }
53
54 beforeAll(() => {
55 root = mkdtempSync(join(tmpdir(), "forge-repo-bound-"));
56 app = createApp({ dataDir: root, port: 0 });
57 seed();
58 });
59 afterAll(() => rmSync(root, { recursive: true, force: true }));
60
61 /**
62 * The plan lines of every SELECT `work` runs (only those containing `only`),
63 * parameters bound as text.
64 */
65 function plans(work: () => void, only = ""): string[] {
66 const db = app.db;
67 const original = db.query.bind(db);
68 const sqls: string[] = [];
69 (db as unknown as { query: unknown }).query = (sql: string) => {
70 sqls.push(sql);
71 return original(sql);
72 };
73 try {
74 work();
75 } finally {
76 (db as unknown as { query: unknown }).query = original;
77 }
78 return sqls
79 .filter((sql) => /^\s*SELECT/i.test(sql) && sql.includes(only))
80 .flatMap((sql) => {
81 const n = Math.max(0, ...[...sql.matchAll(/\?(\d+)/g)].map((m) => Number(m[1])));
82 return original(`EXPLAIN QUERY PLAN ${sql}`)
83 .all(...Array.from({ length: n }, () => "x"))
84 .map((row) => (row as { detail: string }).detail);
85 });
86 }
87
88 /** Tables that hold every repository's rows: a scan of one is a read of the forge. */
89 const WIDE = /^SCAN (issue|pull|review|usage|comment|run|event|check_gate|issue_label|app_note)\b(?!.*USING (COVERING )?INDEX)/;
90 const noWideScan = (lines: string[]) => {
91 expect(lines.length).toBeGreaterThan(0);
92 for (const line of lines) expect(line).not.toMatch(WIDE);
93 };
94
95 const pull = { id: "a2", number: 2, state: "open", mergedAt: null };
96
97 test("an issue's spend is its own, read from the usage index", () => {
98 expect(issueUsage(app.db, "a1")).toEqual({ runs: 1, tokensIn: 10, tokensOut: 5, costUsd: 1 });
99 expect(issueUsage(app.db, "b1")?.costUsd).toBe(2);
100 expect(issueUsage(app.db, "nobody")).toBeUndefined();
101 const lines = plans(() => void issueUsage(app.db, "a1"));
102 expect(lines.join("\n")).toMatch(/SEARCH usage USING (COVERING )?INDEX usage_issue/);
103 noWideScan(lines);
104 });
105
106 test("the landing line reads one repository's approvals, by index", () => {
107 expect(landingLine(app.db, pull, "a")?.says).toStartWith("first in the queue");
108 expect(landingLine(app.db, { ...pull, id: "b2" }, "a")).toBeNull();
109 const lines = plans(() => void landingLine(app.db, pull, "a"), "state = 'approved'");
110 expect(lines.join("\n")).toMatch(/SEARCH review USING (COVERING )?INDEX review_approved_at/);
111 noWideScan(lines);
112 });
113
114 test("a repository's waiting from-app issues are its own, bound by index", () => {
115 expect(unansweredFromApp(app.db, "a")?.repoId).toBe("a");
116 expect(unansweredFromApp(app.db, "b")?.repoId).toBe("b");
117 expect(unansweredFromApp(app.db, "none")).toBeUndefined();
118 expect(waitingFromApp(app.db).map((w) => w.repoId).sort()).toEqual(["a", "b"]);
119 const lines = plans(() => void unansweredFromApp(app.db, "a"));
120 expect(lines.join("\n")).toMatch(/SEARCH i USING (COVERING )?INDEX issue_\w+ \(repo_id=\?\)/);
121 noWideScan(lines);
122 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs