Public: anyone can read this; the crew can change it.

P0 INCIDENT: 0.7.277 hung the forge — readIdlePulls scans every pull.pushed event synchronously on each /boards load (8 s); bound the query, cache the panel, guard with a test and a release smoke #6894

Parked opened this in answer to
Closed without merging

scout/forge:scout/6840-p0-incident-0-7-277-hung-the-forge-readi → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
11 files changed +243 −23
scripts/smoke.ts +1
@@ -72,6 +72,7 @@ if (import.meta.main) {
7272 const failures = await smoke(base, process.env["FORGE_TOKEN"] ?? "", {
7373 version,
7474 waitMs: seconds * 1000,
75 settleMs: Number(process.env["SMOKE_SETTLE_S"] ?? 60) * 1000,
7576 ...(appHost ? { appHost } : {}),
7677 // The forge tells only its probe its version (`src/kernel/probe.ts`).
7778 ...(process.env["FORGE_PROBE"] ? { probe: { [PROBE_HEADER]: process.env["FORGE_PROBE"] } } : {}),
src/core/migrations/20261027110000-pull-push-index.ts added +7
@@ -0,0 +1,7 @@
1 // The idle-pull reader asks, per open pull, for its newest `pull.pushed`
2 // event (`flow/idle_pulls.ts`). Without this, that asks every pushed event in
3 // the forge, once per pull, on a loop that cannot yield (#6840). Additive.
4 export default `
5 CREATE INDEX IF NOT EXISTS event_repo_kind_number_at
6 ON event(repo_id, kind, json_extract(payload, '$.number'), created_at);
7 `;
src/flow/idle_pulls.ts +28 −15
@@ -6,6 +6,8 @@
66
77 import type { Database } from "bun:sqlite";
88 import { closesFromBody } from "../core/pulls.ts";
9 import { writeStamp } from "../core/write_clock.ts";
10 import { Stamped } from "../kernel/stamped.ts";
911
1012 const HOUR_MS = 3_600_000;
1113 const IDLE_HOURS = 72;
@@ -20,27 +22,27 @@ export interface IdlePull {
2022 readonly hours: number;
2123 }
2224
23 /** `publicOnly` reads public repositories alone, as `readOldestApproved` does. */
24 export function readIdlePulls(db: Database, now = new Date(), publicOnly = false): IdlePull[] {
25 const cutoff = new Date(now.getTime() - IDLE_HOURS * HOUR_MS).toISOString();
26 const rows = db
27 .query<{ number: number; owner: string; name: string; moved_at: string; body: string }, [number, string, number]>(
28 `SELECT i.number AS number, ro.username AS owner, r.name AS name, m.at AS moved_at, i.body AS body
25 /** The reader's query; exported so a test can read its plan. */
26 export const IDLE_SQL = `SELECT i.number AS number, ro.username AS owner, r.name AS name, COALESCE((SELECT MAX(e.created_at) FROM event e
27 WHERE e.repo_id = i.repo_id AND e.kind = 'pull.pushed'
28 AND json_extract(e.payload, '$.number') = +i.number), i.created_at) AS moved_at, i.body AS body
2929 FROM pull p
3030 JOIN issue i ON i.id = p.issue_id
3131 JOIN repo r ON r.id = i.repo_id
3232 JOIN user ro ON ro.id = r.owner_id
3333 JOIN user u ON u.id = i.author_id AND u.is_agent = 1
34 JOIN (SELECT p2.issue_id AS issue_id,
35 COALESCE((SELECT MAX(e.created_at) FROM event e
36 WHERE e.kind = 'pull.pushed' AND e.repo_id = i2.repo_id
37 AND json_extract(e.payload, '$.number') = i2.number), i2.created_at) AS at
38 FROM pull p2 JOIN issue i2 ON i2.id = p2.issue_id) m ON m.issue_id = i.id
3934 WHERE (?1 = 0 OR r.visibility = 'public') AND i.state = 'open' AND p.merged_at IS NULL
40 AND m.at <= ?2
41 ORDER BY m.at ASC, i.number ASC
42 LIMIT ?3`,
43 )
35 AND COALESCE((SELECT MAX(e.created_at) FROM event e
36 WHERE e.repo_id = i.repo_id AND e.kind = 'pull.pushed'
37 AND json_extract(e.payload, '$.number') = +i.number), i.created_at) <= ?2
38 ORDER BY moved_at ASC, i.number ASC
39 LIMIT ?3`;
40
41 /** `publicOnly` reads public repositories alone, as `readOldestApproved` does. */
42 export function readIdlePulls(db: Database, now = new Date(), publicOnly = false): IdlePull[] {
43 const cutoff = new Date(now.getTime() - IDLE_HOURS * HOUR_MS).toISOString();
44 const rows = db
45 .query<{ number: number; owner: string; name: string; moved_at: string; body: string }, [number, string, number]>(IDLE_SQL)
4446 .all(publicOnly ? 1 : 0, cutoff, IDLE_LIMIT);
4547 return rows.map((r) => ({
4648 pull: r.number,
@@ -50,3 +52,14 @@ export function readIdlePulls(db: Database, now = new Date(), publicOnly = false
5052 hours: Math.max(0, (now.getTime() - Date.parse(r.moved_at)) / HOUR_MS),
5153 }));
5254 }
55
56 const caches = new WeakMap<Database, Stamped<IdlePull[]>>();
57
58 /** `readIdlePulls`, remembered while the write clock holds, 30 s at most: a page asks this, never the reader (#6840). */
59 export function cachedIdlePulls(db: Database, now = new Date(), publicOnly = false): IdlePull[] {
60 let cache = caches.get(db);
61 if (cache === undefined) caches.set(db, (cache = new Stamped<IdlePull[]>({ maxEntries: 2, maxAgeMs: 30_000 })));
62 const stamp = writeStamp(db);
63 if (stamp === undefined) return readIdlePulls(db, now, publicOnly);
64 return cache.get(publicOnly ? "public" : "all", stamp, () => readIdlePulls(db, now, publicOnly));
65 }
src/release/prove.ts +3 −1
@@ -72,6 +72,8 @@ export interface ProveOptions {
7272 readonly now?: () => number;
7373 readonly sleep?: (ms: number) => Promise<void>;
7474 readonly giveUpMs?: number;
75 /** How long `/healthz` is watched after the pages answer; 60 s unless given. */
76 readonly settleMs?: number;
7577 }
7678
7779 /**
@@ -104,7 +106,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
104106 if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
105107 await sleep(POLL_MS);
106108 }
107 const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true, ...(o.appHost ? { appHost: o.appHost } : {}) });
109 const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true, settleMs: o.settleMs ?? 60_000, now, sleep, ...(o.appHost ? { appHost: o.appHost } : {}) });
108110 return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
109111 } catch (error) {
110112 return finish(`the smoke checks could not run: ${String(error)}`);
src/release/smoke.ts +51
@@ -61,8 +61,19 @@ export interface Options {
6161 anonymousBoard?: boolean;
6262 /** An app's URL on this forge: `GET /` there must never be a forge page (#6726). */
6363 appHost?: string;
64 /** After the pages, poll `/healthz` for this long: a release that reports live and then hangs fails here (#6840). */
65 settleMs?: number;
66 /** The clock and the pause the settle uses; real ones unless a test gives its own. */
67 now?: () => number;
68 sleep?: (ms: number) => Promise<void>;
6469 }
6570
71 /** The heaviest pages: the first to block the loop if a read on them is not bounded. A page that answers slower than `SLOW_PAGE_MS` fails. */
72 export const HEAVY_PAGES = ["/boards", "/explore", "/flow"];
73 export const SLOW_PAGE_MS = 2_000;
74 /** A `/healthz` that takes longer than this while settling is a hung server. */
75 export const SLOW_HEALTH_MS = 250;
76
6677 /** The redirect statuses a sign-in answers with. */
6778 const REDIRECTS = new Set([302, 303, 307]);
6879
@@ -160,6 +171,46 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr
160171 if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
161172 }
162173
174 const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
175 for (const path of HEAVY_PAGES) {
176 const t = Date.now();
177 const page = await get(path, "desktop", auth);
178 if (!page) continue;
179 const ms = Date.now() - t;
180 const signIn = !token && isSignInRedirect(page.url, page.res);
181 // A missing page is not this check's to fail; a hang or an error is.
182 if (page.res.status >= 500) fail(page.url, "desktop", page.res, "expected no server error");
183 else if (page.res.status !== 404 && !signIn && page.res.status !== 200) fail(page.url, "desktop", page.res, "expected 200 (or a redirect to sign-in)");
184 else if (ms > SLOW_PAGE_MS) fail(page.url, "desktop", page.res, `took ${ms} ms, over ${SLOW_PAGE_MS} ms`);
185 }
186
187 if (options.settleMs) {
188 const now = options.now ?? Date.now;
189 const sleep = options.sleep ?? Bun.sleep;
190 const url = new URL("/healthz", root).href;
191 const end = now() + options.settleMs;
192 while (now() < end) {
193 const t = now();
194 try {
195 const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000), headers: { "user-agent": AGENTS["desktop"]!, ...(options.probe ?? {}) } });
196 await res.body?.cancel();
197 const took = now() - t;
198 if (res.status !== 200) {
199 fail(url, "desktop", res, "expected 200 while settling");
200 break;
201 }
202 if (took > SLOW_HEALTH_MS) {
203 fail(url, "desktop", res, `/healthz took ${took} ms while settling, over ${SLOW_HEALTH_MS} ms`);
204 break;
205 }
206 } catch (error) {
207 failures.push({ url, agent: "desktop", status: "error", contentType: "", why: `/healthz while settling: ${String(error)}` });
208 break;
209 }
210 await sleep(options.pollMs ?? 1_000);
211 }
212 }
213
163214 const sw = await get("/sw.js", "desktop");
164215 if (sw) {
165216 const type = sw.res.headers.get("content-type") ?? "";
src/tui/screens/flow.ts +2 −2
@@ -19,7 +19,7 @@ import { pullCounts } from "../../crew/hot_files.ts";
1919 import { warmingLine } from "../../crew/hot_history.ts";
2020 import type { HeldRow } from "../../flow/held.ts";
2121 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
22 import { IDLE_LIMIT, readIdlePulls } from "../../flow/idle_pulls.ts";
22 import { IDLE_LIMIT, cachedIdlePulls } from "../../flow/idle_pulls.ts";
2323 import { pileSays, readPile } from "../../flow/pile.ts";
2424 import { readPipeline, STAGE_LABEL, verdict } from "../../flow/pipeline.ts";
2525 import { parkedSays, readParked } from "../../flow/parked.ts";
@@ -119,7 +119,7 @@ const IDLE_SHOWN = 5;
119119
120120 /** A blank line, `idle: N`, then a line per idle crew pull (`flow/idle_pulls.ts`, #6479), oldest first; nothing when none. */
121121 export function idleLines(db: Database, now: Date, publicOnly: boolean, inner: number): string[] {
122 const idle = readIdlePulls(db, now, publicOnly);
122 const idle = cachedIdlePulls(db, now, publicOnly);
123123 if (idle.length === 0) return [];
124124 return [
125125 "",
src/web/pages/flow-panel.ts +2 −2
@@ -36,7 +36,7 @@ import { type QueueRow, redBatchWords } from "../../flow/queue.ts";
3636 import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts";
3737 import { warmingLine } from "../../crew/hot_history.ts";
3838 import { getRepoById } from "../../core/repos.ts";
39 import { IDLE_LIMIT, readIdlePulls, type IdlePull } from "../../flow/idle_pulls.ts";
39 import { IDLE_LIMIT, cachedIdlePulls, type IdlePull } from "../../flow/idle_pulls.ts";
4040 import { readParked } from "../../flow/parked.ts";
4141 import { hoursWords, readLeadTime } from "../../flow/lead_time.ts";
4242 import { buildYieldSays, readBuildYield } from "../../flow/build_yield.ts";
@@ -113,7 +113,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
113113 const latency = readReviewLatency(app.db, new Date(now), agent);
114114 const openers = readOpeners(app.db, new Date(now), agent);
115115 const pile = pileSays(readPile(app.db, reader.kind === "agent"));
116 const idle = readIdlePulls(app.db, new Date(now), reader.kind === "agent");
116 const idle = cachedIdlePulls(app.db, new Date(now), reader.kind === "agent");
117117 const parked = readParked(app.db, new Date(now), reader.kind === "agent");
118118 const newest = parked.newest[0];
119119 const listed = asked(url);
test/boards-healthz-gap.test.ts added +68
@@ -0,0 +1,68 @@
1 /** #6840: loading /boards must never stop /healthz answering for long. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp } from "../src/app.ts";
7 import { createToken } from "../src/core/tokens.ts";
8 import { createUser } from "../src/core/users.ts";
9 import { serve } from "../src/http/server.ts";
10
11 let dir: string;
12 let app: ReturnType<typeof createApp>;
13 let servers: Awaited<ReturnType<typeof serve>>;
14 let token: string;
15
16 beforeAll(async () => {
17 dir = await mkdtemp(join(tmpdir(), "forge-boards-gap-"));
18 app = createApp({ dataDir: dir, port: 0 });
19 servers = await serve(app);
20 const r = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
21 if (!r.ok) throw new Error(r.error);
22 token = createToken(app.db, r.value.id, "t", "admin").token;
23 const db = app.db;
24 db.run("INSERT INTO user (id, username, email, password_hash, is_agent) VALUES ('bot', 'bot', 'bot@x', 'x', 1)");
25 db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES ('r', ?1, 'r', 'public')", [r.value.id]);
26 const at = "2026-01-01T00:00:00.000Z";
27 db.transaction(() => {
28 const issue = db.prepare("INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, 'r', ?2, 'pull', 't', '', 'bot', 'open', ?3)");
29 const pull = db.prepare("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', ?2, '', 'h')");
30 const ev = db.prepare("INSERT INTO event (id, kind, repo_id, payload, created_at, seq) VALUES (?1, 'pull.pushed', 'r', ?2, ?3, ?4)");
31 for (let n = 1; n <= 2000; n++) {
32 issue.run(`p${n}`, n, at);
33 pull.run(`p${n}`, `b${n}`);
34 }
35 for (let e = 0; e < 100_000; e++) ev.run(`e${e}`, `{"number":${(e % 2000) + 1}}`, at, e + 1);
36 })();
37 }, 60_000);
38
39 afterAll(async () => {
40 servers.stop();
41 app.close();
42 await rm(dir, { recursive: true, force: true });
43 });
44
45 test("no /healthz gap over 250 ms while /boards loads", async () => {
46 const headers = { authorization: `Bearer ${token}` };
47 let running = true;
48 let worst = 0;
49 const poll = (async () => {
50 let last = performance.now();
51 while (running) {
52 await (await fetch(servers.url + "/healthz")).text();
53 const t = performance.now();
54 worst = Math.max(worst, t - last);
55 last = t;
56 await Bun.sleep(5);
57 }
58 })();
59 await Bun.sleep(50);
60 for (let i = 0; i < 3; i++) {
61 const res = await fetch(servers.url + "/boards", { headers });
62 expect(res.status).toBe(200);
63 await res.text();
64 }
65 running = false;
66 await poll;
67 expect(worst).toBeLessThan(250);
68 });
test/flow-idle-pulls-scale.test.ts added +59
@@ -0,0 +1,59 @@
1 /** #6840: the idle-pull reader must not scan `event` per pull. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { IDLE_SQL, cachedIdlePulls, readIdlePulls } from "../src/flow/idle_pulls.ts";
8
9 const NOW = new Date("2026-10-26T12:00:00.000Z");
10 let dir: string;
11 let app: App;
12
13 beforeAll(async () => {
14 dir = await mkdtemp(join(tmpdir(), "idle-scale-"));
15 app = createApp({ dataDir: dir, port: 0 });
16 const db = app.db;
17 db.run("INSERT INTO user (id, username, email, password_hash, is_agent) VALUES ('a', 'scout', 'a@x', 'x', 1)");
18 db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES ('r', 'a', 'r', 'public')");
19 const at = new Date(NOW.getTime() - 100 * 3_600_000).toISOString();
20 db.transaction(() => {
21 const issue = db.prepare("INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, 'r', ?2, 'pull', 't', '', 'a', 'open', ?3)");
22 const pull = db.prepare("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', ?2, '', 'h')");
23 const ev = db.prepare("INSERT INTO event (id, kind, repo_id, payload, created_at, seq) VALUES (?1, 'pull.pushed', 'r', ?2, ?3, ?4)");
24 for (let n = 1; n <= 5000; n++) {
25 issue.run(`p${n}`, n, at);
26 pull.run(`p${n}`, `b${n}`);
27 }
28 for (let e = 0; e < 500_000; e++) ev.run(`e${e}`, `{"number":${(e % 5000) + 1}}`, at, e + 1);
29 })();
30 }, 120_000);
31 afterAll(async () => {
32 app.close?.();
33 await rm(dir, { recursive: true, force: true });
34 });
35
36 test("5k pulls and 500k pushes read in under 100 ms, by index", () => {
37 readIdlePulls(app.db, NOW);
38 const t = performance.now();
39 expect(readIdlePulls(app.db, NOW)).toHaveLength(20);
40 expect(performance.now() - t).toBeLessThan(100);
41 });
42
43 test("no plan step scans event for each pull", () => {
44 const steps = app.db
45 .query<{ detail: string }, [number, string, number]>(`EXPLAIN QUERY PLAN ${IDLE_SQL}`)
46 .all(0, "2027", 20)
47 .map((r) => r.detail);
48 const plan = steps.join("\n");
49 expect(plan).toContain("event_repo_kind_number_at");
50 expect(plan).not.toMatch(/SCAN e\b/);
51 for (const s of steps.filter((d) => /SEARCH e /.test(d))) expect(s).toContain("<expr>=?");
52 });
53
54 test("the cached reader answers again without reading", () => {
55 const a = cachedIdlePulls(app.db, NOW);
56 expect(cachedIdlePulls(app.db, NOW)).toBe(a);
57 app.db.run("UPDATE issue SET state = 'closed'");
58 expect(cachedIdlePulls(app.db, NOW)).toEqual([]);
59 });
test/release-prove.test.ts +1 −1
@@ -41,7 +41,7 @@ describe("proving a release", () => {
4141 });
4242 afterEach(() => rmSync(dir, { recursive: true, force: true }));
4343 const run = (version: string, fake: typeof fetch, extra = {}) =>
44 proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, ...extra });
44 proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, settleMs: 0, ...extra });
4545
4646 test("a pass is recorded with the version and raises nothing", async () => {
4747 const { fake } = site("1.2.3");
test/smoke.test.ts +21 −2
@@ -63,7 +63,7 @@ test("/board without a token is a 302 and fails", async () => {
6363
6464 test("the CLI exits non-zero on failure and zero on success", async () => {
6565 const run = async (url: string) =>
66 (await Bun.spawn(["bun", "scripts/smoke.ts", url], { env: { ...process.env, FORGE_TOKEN: "t" }, stdout: "pipe", stderr: "pipe" }).exited);
66 (await Bun.spawn(["bun", "scripts/smoke.ts", url], { env: { ...process.env, FORGE_TOKEN: "t", SMOKE_SETTLE_S: "0" }, stdout: "pipe", stderr: "pipe" }).exited);
6767 expect(await run(stub("text/css"))).toBe(0);
6868 server?.stop(true);
6969 expect(await run(stub("text/html"))).toBe(1);
@@ -103,7 +103,7 @@ test("a Response.error() on a navigation, a non-JS or a missing sw.js fails", as
103103 test("the CLI takes --version and --wait", async () => {
104104 const base = stub("text/css");
105105 const run = (...a: string[]) =>
106 Bun.spawn(["bun", "scripts/smoke.ts", base, ...a], { env: { ...process.env, FORGE_TOKEN: "t" }, stdout: "pipe", stderr: "pipe" }).exited;
106 Bun.spawn(["bun", "scripts/smoke.ts", base, ...a], { env: { ...process.env, FORGE_TOKEN: "t", SMOKE_SETTLE_S: "0" }, stdout: "pipe", stderr: "pipe" }).exited;
107107 expect(await run("--version", "v1")).toBe(0);
108108 expect(await run("--version", "zz", "--wait", "0")).toBe(1);
109109 });
@@ -198,3 +198,22 @@ test("--app-host fails when an app host answers with the forge's page", async ()
198198 appPage.stop(true);
199199 }
200200 });
201
202 test("the heavy pages are loaded, and a /healthz that hangs while settling fails (#6840)", async () => {
203 const seen: string[] = [];
204 let slow = false;
205 let clock = 0;
206 const base = stub("text/css", seen);
207 const fetchStub = (async (url: string | URL | Request, init?: RequestInit) => {
208 const path = new URL(String(url)).pathname;
209 if (path === "/healthz" && slow) clock += 400;
210 return fetch(url, init);
211 }) as typeof fetch;
212 const options = { settleMs: 5_000, pollMs: 1_000, now: () => clock, sleep: async (ms: number) => void (clock += ms) };
213 expect(await smoke(base, "t", { ...options, fetch: fetchStub })).toEqual([]);
214 expect(seen.some((s) => s.startsWith("/boards "))).toBe(true);
215 slow = true;
216 clock = 0;
217 const failures = await smoke(base, "t", { ...options, fetch: fetchStub });
218 expect(failures.some((f) => f.why.includes("while settling"))).toBe(true);
219 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs