Public: anyone can read this; the crew can change it.

P0 INCIDENT: 0.7.277 hung the forge — readIdlePulls scans every pull.pushed event synchronously on each /boards load (8 s); bound the query, cache the panel, guard with a test and a release smoke #6860

Parked opened this in answer to
Closed without merging

mason/forge:mason/6840-p0-incident-0-7-277-hung-the-forge-readi → main 5 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
13 files changed +424 −21
docs/CACHING.md +16
@@ -291,6 +291,21 @@ fixture; all three caches hit):
291291 The anonymous door (1.5 ms, 10 statements), the app shelf, and the
292292 "On you" card. Each costs less than its cache's key would.
293293
294 ## The flow panel (`/boards`)
295
296 ### The idle pulls: per audience (#6840)
297
298 - **What:** `readIdlePulls` (`flow/idle_pulls.ts`) for the `/boards` flow
299 panel and the TUI flow screen, read through `flow/idle_cache.ts`. The flow
300 tick in `main.ts` warms both audiences every minute.
301 - **Key:** `idle:{all|public}:{UTC minute}`; agents read `public` only.
302 - **Invalidated by:** every write, through the write clock.
303 - **Max age:** 30 seconds.
304 - **Measured** (`test/flow-idle-pulls-scale.test.ts`: 5,000 pulls, 500,000
305 events): a miss is about 10 ms, a seek on `event_pull_pushed` per open
306 pull. Before that index it scanned every pushed event per pull and held
307 the event loop for seconds: 0.7.277 went down on it.
308
294309 ## The writes that invalidate
295310
296311 Each row is an action (or, for a push, the post-receive hook, the one
@@ -411,6 +426,7 @@ warm:
411426 | home feed page | viewer, forge head | 30 s |
412427 | home KPI row | forge head, UTC hour; admin-gated | 30 s |
413428 | home pinned panel | board blob, panel, forge head; admin-gated | 30 s |
429 | idle pulls | audience, forge head, UTC minute | 30 s |
414430
415431 ## Raw files at a full commit id (#3381)
416432
src/core/migration-pins/20261027120000-event-pull-pushed.pin added +1
@@ -0,0 +1,1 @@
1 7b96b56a2c22ec44345bbecf79d0c35312de39b76c16f1965772e70ab7d355c6
src/core/migrations/20261027120000-event-pull-pushed.ts added +9
@@ -0,0 +1,9 @@
1 // Each pull's newest push, found by index (#6840): `flow/idle_pulls.ts` asks
2 // for MAX(created_at) of one repository's `pull.pushed` events for one pull
3 // number. Without this it scanned every pushed event forge-wide per pull and
4 // held the event loop for seconds. Partial, so only pushes pay for it.
5 export default `
6 CREATE INDEX IF NOT EXISTS event_pull_pushed
7 ON event(repo_id, json_extract(payload, '$.number'), created_at)
8 WHERE kind = 'pull.pushed';
9 `;
src/flow/idle_cache.ts added +44
@@ -0,0 +1,44 @@
1 /**
2 * What the flow panel and the TUI flow screen remember of the idle pulls
3 * (#6840): `readIdlePulls`, valid while the forge's write clock has not moved
4 * and for 30 seconds at most, like home's figures (`web/studio/home_cache.ts`).
5 * The flow tick in `main.ts` warms it, so a quiet forge's `/boards` reads a
6 * remembered answer; after a write the next read recomputes, which the
7 * `event_pull_pushed` index keeps to milliseconds.
8 */
9 import type { Database } from "bun:sqlite";
10 import { writeStamp } from "../core/write_clock.ts";
11 import { Stamped } from "../kernel/stamped.ts";
12 import { readIdlePulls, type IdlePull } from "./idle_pulls.ts";
13
14 export const IDLE_MAX_AGE_MS = 30_000;
15
16 const caches = new WeakMap<Database, Stamped<IdlePull[]>>();
17
18 /** The idle-pull cache of `db`: for tests, which count its hits. */
19 export function idleCache(db: Database): Stamped<IdlePull[]> {
20 let cache = caches.get(db);
21 if (cache === undefined) {
22 cache = new Stamped<IdlePull[]>({ maxEntries: 4, maxAgeMs: IDLE_MAX_AGE_MS });
23 caches.set(db, cache);
24 }
25 return cache;
26 }
27
28 /**
29 * The idle pulls, remembered per audience and UTC minute, so a caller asking
30 * about another moment never reads this one's answer. `publicOnly` as in
31 * `readIdlePulls`.
32 */
33 export function cachedIdlePulls(db: Database, now: Date, publicOnly: boolean): IdlePull[] {
34 const stamp = writeStamp(db);
35 if (stamp === undefined) return readIdlePulls(db, now, publicOnly);
36 const key = `idle:${publicOnly ? "public" : "all"}:${now.toISOString().slice(0, 16)}`;
37 return idleCache(db).get(key, stamp, () => readIdlePulls(db, now, publicOnly));
38 }
39
40 /** Reads both audiences into the cache: the flow tick's part. */
41 export function warmIdlePulls(db: Database, now = new Date()): void {
42 cachedIdlePulls(db, now, false);
43 cachedIdlePulls(db, now, true);
44 }
src/flow/idle_pulls.ts +18 −15
@@ -1,7 +1,11 @@
11 /**
22 * Idle pulls (#6470): open, unmerged pulls opened by a crew agent whose head
33 * has not moved for 72 h. A pull "moves" at its newest `pull.pushed` event, or
4 * at its opening when it has none. Oldest first, at most 20. Nothing shows it yet.
4 * at its opening when it has none. Oldest first, at most 20.
5 *
6 * Only open, unmerged pulls are read, and each one's newest push is a seek on
7 * `event_pull_pushed` (#6840): a scan of `event` per pull hung the forge.
8 * Pages read it through `idle_cache.ts`, never on the request path.
59 */
610
711 import type { Database } from "bun:sqlite";
@@ -25,20 +29,19 @@ export function readIdlePulls(db: Database, now = new Date(), publicOnly = false
2529 const cutoff = new Date(now.getTime() - IDLE_HOURS * HOUR_MS).toISOString();
2630 const rows = db
2731 .query<{ number: number; owner: string; name: string; moved_at: string; body: string }, [number, string, number]>(
28 `SELECT i.number AS number, ro.username AS owner, r.name AS name, m.at AS moved_at, i.body AS body
29 FROM pull p
30 JOIN issue i ON i.id = p.issue_id
31 JOIN repo r ON r.id = i.repo_id
32 JOIN user ro ON ro.id = r.owner_id
33 JOIN user u ON u.id = i.author_id AND u.is_agent = 1
34 JOIN (SELECT p2.issue_id AS issue_id,
35 COALESCE((SELECT MAX(e.created_at) FROM event e
36 WHERE e.kind = 'pull.pushed' AND e.repo_id = i2.repo_id
37 AND json_extract(e.payload, '$.number') = i2.number), i2.created_at) AS at
38 FROM pull p2 JOIN issue i2 ON i2.id = p2.issue_id) m ON m.issue_id = i.id
39 WHERE (?1 = 0 OR r.visibility = 'public') AND i.state = 'open' AND p.merged_at IS NULL
40 AND m.at <= ?2
41 ORDER BY m.at ASC, i.number ASC
32 `SELECT number, owner, name, moved_at, body FROM (
33 SELECT i.number AS number, ro.username AS owner, r.name AS name, i.body AS body,
34 COALESCE((SELECT MAX(e.created_at) FROM event e INDEXED BY event_pull_pushed
35 WHERE e.kind = 'pull.pushed' AND e.repo_id = i.repo_id
36 AND json_extract(e.payload, '$.number') = +i.number), i.created_at) AS moved_at
37 FROM pull p
38 JOIN issue i ON i.id = p.issue_id
39 JOIN repo r ON r.id = i.repo_id
40 JOIN user ro ON ro.id = r.owner_id
41 JOIN user u ON u.id = i.author_id AND u.is_agent = 1
42 WHERE (?1 = 0 OR r.visibility = 'public') AND i.state = 'open' AND p.merged_at IS NULL)
43 WHERE moved_at <= ?2
44 ORDER BY moved_at ASC, number ASC
4245 LIMIT ?3`,
4346 )
4447 .all(publicOnly ? 1 : 0, cutoff, IDLE_LIMIT);
src/main.ts +3
@@ -55,6 +55,7 @@ import { graphInstagram } from "./crew/instagram.ts";
5555 import { peopleOnCrews } from "./core/agents.ts";
5656 import { startNotifier } from "./crew/notify.ts";
5757 import { tickClog } from "./flow/clog_tick.ts";
58 import { warmIdlePulls } from "./flow/idle_cache.ts";
5859 import { sweepStaleAsks } from "./crew/stale_asks.ts";
5960 import { recordAskLedger } from "./web/studio/asks.ts";
6061 import { startPromptRoutine } from "./crew/prompts.ts";
@@ -552,9 +553,11 @@ const processTick = (): void => {
552553 }
553554 };
554555 // The clog watch (#6524) reads the flow every minute; its state is the database's.
556 // The idle pulls ride it into their cache (#6840), so /boards reads them warm.
555557 const clogTick = (): void => {
556558 try {
557559 tickClog(app.db, app.bus);
560 warmIdlePulls(app.db);
558561 } catch (error) {
559562 app.log.warn("clog tick failed", { error: String(error) });
560563 }
src/release/prove.ts +12 −1
@@ -6,6 +6,8 @@
66 * restart of the same version never runs it again and vitals reads it
77 * without the network. A failure is a `release` card, paged once by the
88 * cause pager, until a later version passes. It gives up after ten minutes.
9 * The smoke loads the heaviest pages and watches `/healthz` for a minute
10 * after (#6840): a version that answers and then hangs is not proven.
911 */
1012
1113 import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
@@ -17,6 +19,8 @@ import { ownProbeHeaders } from "../kernel/probe.ts";
1719 /** How long the public URL has to answer as the new version. */
1820 export const GIVE_UP_MS = 10 * 60_000;
1921 const POLL_MS = 2_000;
22 /** How long `/healthz` must stay fast after the smoke's heavy pages load (#6840). */
23 export const STEADY_MS = 60_000;
2024
2125 export interface Proof {
2226 readonly version: string;
@@ -104,7 +108,14 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
104108 if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
105109 await sleep(POLL_MS);
106110 }
107 const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true, ...(o.appHost ? { appHost: o.appHost } : {}) });
111 const failures = await smoke(o.url, o.token ?? "", {
112 fetch: doFetch,
113 anonymousBoard: true,
114 steadyMs: STEADY_MS,
115 now,
116 sleep,
117 ...(o.appHost ? { appHost: o.appHost } : {}),
118 });
108119 return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
109120 } catch (error) {
110121 return finish(`the smoke checks could not run: ${String(error)}`);
src/release/smoke.ts +54 −1
@@ -4,7 +4,11 @@
44 * must answer 200; the stylesheet linked from `/` must be a non-empty
55 * `text/css` 200; `/sw.js` must be a 200 JavaScript response whose navigation
66 * branch never answers with `Response.error()`. With `version`, `/healthz` is
7 * first polled until `X-Forge-Version` is that version.
7 * first polled until `X-Forge-Version` is that version. With `steadyMs`, the
8 * heaviest pages (`HEAVY`) are loaded and must answer without a 5xx, then
9 * `/healthz` is polled for that long and every answer must be a 200 within
10 * `PROBE_MS`: 0.7.277 went live, answered, and hung at its first `/boards`
11 * (#6840), so a release proves it stays up, not only that it came up.
812 *
913 * The running forge proves each new version with it (`release/prove.ts`), so
1014 * it lives here: the image ships src/ and nothing else (#2652).
@@ -18,6 +22,11 @@ export const AGENTS: Record<string, string> = {
1822 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
1923 };
2024
25 /** The pages that read the most per request, loaded before the steady watch (#6840). */
26 export const HEAVY = ["/boards", "/explore", "/"] as const;
27 /** The longest a `/healthz` may take while steady: the edge's probe gives up at a second. */
28 export const PROBE_MS = 1_000;
29
2130 export interface Failure {
2231 url: string;
2332 agent: string;
@@ -61,6 +70,11 @@ export interface Options {
6170 anonymousBoard?: boolean;
6271 /** An app's URL on this forge: `GET /` there must never be a forge page (#6726). */
6372 appHost?: string;
73 /** Watch `/healthz` this long after the heavy pages load; 0 (the default) skips it. */
74 steadyMs?: number;
75 /** The clock a probe's latency is measured on, and the pause between steady polls: seams for tests. */
76 now?: () => number;
77 sleep?: (ms: number) => Promise<void>;
6478 }
6579
6680 /** The redirect statuses a sign-in answers with. */
@@ -181,5 +195,44 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr
181195 failures.push({ url, agent: "desktop", status: "error", contentType: "", why: String(error) });
182196 }
183197 }
198 if (options.steadyMs) failures.push(...(await steady(root, token, options.steadyMs, options)));
199 return failures;
200 }
201
202 /** The heavy pages, then `/healthz` every `pollMs` for `ms`: the failures, at most one per page and one for the watch. */
203 async function steady(root: URL, token: string, ms: number, options: Options): Promise<Failure[]> {
204 const fetch = options.fetch ?? globalThis.fetch;
205 const now = options.now ?? performance.now.bind(performance);
206 const sleep = options.sleep ?? Bun.sleep;
207 const pollMs = options.pollMs ?? 1_000;
208 const failures: Failure[] = [];
209 const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
210 for (const path of HEAVY) {
211 const url = new URL(path, root).href;
212 try {
213 const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(10_000), headers: { "user-agent": AGENTS["desktop"]!, ...auth } });
214 await res.body?.cancel();
215 if (res.status >= 500) failures.push({ url, agent: "desktop", status: res.status, contentType: res.headers.get("content-type") ?? "", why: "a heavy page answered 5xx" });
216 } catch (error) {
217 failures.push({ url, agent: "desktop", status: "error", contentType: "", why: `a heavy page did not answer: ${String(error)}` });
218 }
219 }
220 const url = new URL("/healthz", root).href;
221 for (let polls = Math.max(1, Math.ceil(ms / pollMs)); polls > 0; polls--) {
222 const started = now();
223 try {
224 const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(PROBE_MS) });
225 await res.body?.cancel();
226 const took = now() - started;
227 if (res.status !== 200 || took > PROBE_MS) {
228 failures.push({ url, agent: "-", status: res.status, contentType: "", why: `after go-live /healthz answered ${res.status} in ${Math.round(took)} ms` });
229 break;
230 }
231 } catch (error) {
232 failures.push({ url, agent: "-", status: "error", contentType: "", why: `after go-live /healthz stopped answering: ${String(error)}` });
233 break;
234 }
235 await sleep(pollMs);
236 }
184237 return failures;
185238 }
src/tui/screens/flow.ts +3 −2
@@ -19,7 +19,8 @@ import { pullCounts } from "../../crew/hot_files.ts";
1919 import { warmingLine } from "../../crew/hot_history.ts";
2020 import type { HeldRow } from "../../flow/held.ts";
2121 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
22 import { IDLE_LIMIT, readIdlePulls } from "../../flow/idle_pulls.ts";
22 import { IDLE_LIMIT } from "../../flow/idle_pulls.ts";
23 import { cachedIdlePulls } from "../../flow/idle_cache.ts";
2324 import { pileSays, readPile } from "../../flow/pile.ts";
2425 import { readPipeline, STAGE_LABEL, verdict } from "../../flow/pipeline.ts";
2526 import { parkedSays, readParked } from "../../flow/parked.ts";
@@ -119,7 +120,7 @@ const IDLE_SHOWN = 5;
119120
120121 /** A blank line, `idle: N`, then a line per idle crew pull (`flow/idle_pulls.ts`, #6479), oldest first; nothing when none. */
121122 export function idleLines(db: Database, now: Date, publicOnly: boolean, inner: number): string[] {
122 const idle = readIdlePulls(db, now, publicOnly);
123 const idle = cachedIdlePulls(db, now, publicOnly);
123124 if (idle.length === 0) return [];
124125 return [
125126 "",
src/web/pages/flow-panel.ts +3 −2
@@ -36,7 +36,8 @@ import { type QueueRow, redBatchWords } from "../../flow/queue.ts";
3636 import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts";
3737 import { warmingLine } from "../../crew/hot_history.ts";
3838 import { getRepoById } from "../../core/repos.ts";
39 import { IDLE_LIMIT, readIdlePulls, type IdlePull } from "../../flow/idle_pulls.ts";
39 import { IDLE_LIMIT, type IdlePull } from "../../flow/idle_pulls.ts";
40 import { cachedIdlePulls } from "../../flow/idle_cache.ts";
4041 import { readParked } from "../../flow/parked.ts";
4142 import { hoursWords, readLeadTime } from "../../flow/lead_time.ts";
4243 import { buildYieldSays, readBuildYield } from "../../flow/build_yield.ts";
@@ -113,7 +114,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
113114 const latency = readReviewLatency(app.db, new Date(now), agent);
114115 const openers = readOpeners(app.db, new Date(now), agent);
115116 const pile = pileSays(readPile(app.db, reader.kind === "agent"));
116 const idle = readIdlePulls(app.db, new Date(now), reader.kind === "agent");
117 const idle = cachedIdlePulls(app.db, new Date(now), reader.kind === "agent");
117118 const parked = readParked(app.db, new Date(now), reader.kind === "agent");
118119 const newest = parked.newest[0];
119120 const listed = asked(url);
test/flow-idle-cache.test.ts added +58
@@ -0,0 +1,58 @@
1 /**
2 * The idle-pull cache (#6840): the flow tick warms it, `/boards` and the TUI
3 * read the warm answer while nothing is written, and a write recomputes.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createUser } from "../src/core/users.ts";
11 import { cachedIdlePulls, idleCache, warmIdlePulls } from "../src/flow/idle_cache.ts";
12
13 const NOW = new Date("2026-10-27T12:00:00.000Z");
14 const ago = (h: number): string => new Date(NOW.getTime() - h * 3_600_000).toISOString();
15
16 let dir: string;
17 let app: App;
18 let agent: string;
19
20 beforeAll(async () => {
21 dir = await mkdtemp(join(tmpdir(), "idle-cache-"));
22 app = createApp({ dataDir: dir, port: 0 });
23 const u = await createUser(app.db, {
24 origin: "seed", trusted: true, username: "scout", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true,
25 });
26 if (!u.ok) throw new Error(u.error);
27 agent = u.value.id;
28 app.db.run("UPDATE user SET is_agent = 1 WHERE id = ?1", [agent]);
29 app.db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES ('pub', ?1, 'pub', 'public')", [agent]);
30 });
31 afterAll(async () => {
32 app.close?.();
33 await rm(dir, { recursive: true, force: true });
34 });
35
36 function openPull(n: number): void {
37 app.db.run(
38 "INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, 'pub', ?2, 'pull', 't', '', ?3, 'open', ?4)",
39 [`p${n}`, n, agent, ago(100)],
40 );
41 app.db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', ?2, '', 'h')", [`p${n}`, `b${n}`]);
42 }
43
44 test("a warmed answer is read again until something is written", () => {
45 openPull(1);
46 warmIdlePulls(app.db, NOW);
47 const cache = idleCache(app.db);
48 const hits = cache.hits;
49 expect(cachedIdlePulls(app.db, NOW, false).map((p) => p.pull)).toEqual([1]);
50 expect(cachedIdlePulls(app.db, NOW, true).map((p) => p.pull)).toEqual([1]);
51 expect(cache.hits).toBe(hits + 2);
52 openPull(2);
53 expect(cachedIdlePulls(app.db, NOW, false).map((p) => p.pull)).toEqual([1, 2]);
54 });
55
56 test("another minute is another answer", () => {
57 expect(cachedIdlePulls(app.db, new Date(NOW.getTime() - 40 * 3_600_000), false)).toEqual([]);
58 });
test/flow-idle-pulls-scale.test.ts added +137
@@ -0,0 +1,137 @@
1 /**
2 * Idle pulls at the forge's scale (#6840): 0.7.277 hung the forge because
3 * `readIdlePulls` scanned every `pull.pushed` event once per pull. Seeded with
4 * 5,000 pulls and 500,000 events, it answers in under 100 ms and its plan
5 * never scans `event`; and loading `/boards` over that data never holds the
6 * event loop long enough for `/healthz` to go quiet for 250 ms.
7 */
8 import { afterAll, beforeAll, expect, test } from "bun:test";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { createToken } from "../src/core/tokens.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { serve } from "../src/http/server.ts";
16 import { readIdlePulls } from "../src/flow/idle_pulls.ts";
17
18 const NOW = new Date("2026-10-27T12:00:00.000Z");
19 const PULLS = 5_000;
20 const EVENTS = 500_000;
21 const REPOS = 10;
22
23 let dir: string;
24 let app: App;
25 let servers: Awaited<ReturnType<typeof serve>>;
26 let token: string;
27
28 beforeAll(async () => {
29 dir = await mkdtemp(join(tmpdir(), "idle-scale-"));
30 app = createApp({ dataDir: dir, port: 0 });
31 const owner = await createUser(app.db, {
32 origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true,
33 });
34 const crew = await createUser(app.db, {
35 origin: "seed", trusted: true, username: "scout", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: false,
36 });
37 if (!owner.ok || !crew.ok) throw new Error("seed users");
38 const db = app.db;
39 db.run("UPDATE user SET is_agent = 1 WHERE id = ?1", [crew.value.id]);
40 db.transaction(() => {
41 for (let r = 0; r < REPOS; r++)
42 db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES (?1, ?2, ?1, ?3)", [`r${r}`, owner.value.id, r % 2 ? "public" : "private"]);
43 // A pull per number, spread over the repositories; one in seven open, the
44 // rest closed or merged, as on a forge that has run a while.
45 db.run(
46 `WITH RECURSIVE n(k) AS (SELECT 1 UNION ALL SELECT k + 1 FROM n WHERE k < ?1)
47 INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at)
48 SELECT 'p' || k, 'r' || (k % ?2), k, 'pull', 't', '', ?3, CASE WHEN k % 7 = 0 THEN 'open' ELSE 'closed' END,
49 strftime('%Y-%m-%dT%H:%M:%fZ', ?4, '-' || (100 + k % 400) || ' hours') FROM n`,
50 [PULLS, REPOS, crew.value.id, NOW.toISOString()],
51 );
52 db.run(
53 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
54 SELECT id, 'main', 'b' || number, '', 'h', CASE WHEN number % 3 = 0 AND state = 'closed' THEN created_at END FROM issue`,
55 );
56 // `seq` given, so the outbox trigger (migration 0069) stands aside.
57 // Half the events are pushes to some pull of some repository, the rest other kinds.
58 db.run(
59 `WITH RECURSIVE n(k) AS (SELECT 1 UNION ALL SELECT k + 1 FROM n WHERE k < ?1)
60 INSERT INTO event (id, seq, kind, repo_id, payload, created_at)
61 SELECT 'e' || k, k, CASE WHEN k % 2 = 0 THEN 'pull.pushed' ELSE 'issue.commented' END, 'r' || (k % ?2),
62 json_object('number', 1 + k % ?3), strftime('%Y-%m-%dT%H:%M:%fZ', ?4, '-' || (k % 600) || ' hours') FROM n`,
63 [EVENTS, REPOS, PULLS, NOW.toISOString()],
64 );
65 })();
66 db.run("ANALYZE");
67 token = createToken(db, owner.value.id, "t", "admin").token;
68 servers = await serve(app);
69 }, 60_000);
70
71 afterAll(async () => {
72 servers.stop();
73 app.close?.();
74 await rm(dir, { recursive: true, force: true });
75 });
76
77 test("readIdlePulls answers 5,000 pulls and 500,000 events in under 100 ms", () => {
78 readIdlePulls(app.db, NOW, false);
79 for (const publicOnly of [false, true]) {
80 const started = performance.now();
81 const rows = readIdlePulls(app.db, NOW, publicOnly);
82 expect(performance.now() - started).toBeLessThan(100);
83 expect(rows.length).toBeGreaterThan(0);
84 }
85 });
86
87 test("each pull's newest push is a seek on event_pull_pushed, never a scan of event", () => {
88 const plan = app.db
89 .query<{ detail: string }, []>(
90 `EXPLAIN QUERY PLAN SELECT MAX(e.created_at) FROM event e
91 WHERE e.kind = 'pull.pushed' AND e.repo_id = 'r1' AND json_extract(e.payload, '$.number') = 11`,
92 )
93 .all()
94 .map((r) => r.detail);
95 expect(plan.join("\n")).toContain("event_pull_pushed");
96 // The whole reader's plan: `event` is only ever searched, by that index.
97 const sql = readerSql();
98 const whole = app.db.query<{ detail: string }, [number, string, number]>(`EXPLAIN QUERY PLAN ${sql}`).all(0, NOW.toISOString(), 20).map((r) => r.detail);
99 const onEvent = whole.filter((d) => /\b(event|e)\b/.test(d) && /^(SCAN|SEARCH)/.test(d));
100 expect(onEvent.length).toBeGreaterThan(0);
101 for (const d of onEvent) expect(d).toMatch(/^SEARCH e USING (COVERING )?INDEX event_pull_pushed/);
102 });
103
104 test("/healthz never goes quiet for 250 ms while /boards loads", async () => {
105 // Server and probe share this event loop: a synchronous read on the
106 // request path shows as a gap between two answered probes.
107 let polling = true;
108 let worst = 0;
109 const poll = (async () => {
110 let last = performance.now();
111 while (polling) {
112 const res = await fetch(`${servers.url}/healthz`);
113 await res.body?.cancel();
114 expect(res.status).toBe(200);
115 const at = performance.now();
116 worst = Math.max(worst, at - last);
117 last = at;
118 await Bun.sleep(10);
119 }
120 })();
121 for (let i = 0; i < 5; i++) {
122 const res = await fetch(`${servers.url}/boards`, { headers: { authorization: `Bearer ${token}` } });
123 expect(res.status).toBe(200);
124 expect(await res.text()).toContain('id="bottleneck"');
125 }
126 polling = false;
127 await poll;
128 expect(worst).toBeLessThan(250);
129 }, 30_000);
130
131 /** The reader's statement, read from its source so the plan tested is the one shipped. */
132 function readerSql(): string {
133 const source = require("node:fs").readFileSync(join(import.meta.dir, "../src/flow/idle_pulls.ts"), "utf8") as string;
134 const sql = /\.query<[^`]*`([^`]*)`/.exec(source)?.[1];
135 if (sql === undefined) throw new Error("no statement in idle_pulls.ts");
136 return sql;
137 }
test/smoke-steady.test.ts added +66
@@ -0,0 +1,66 @@
1 /**
2 * The smoke's steady watch (#6840): the heavy pages load, then `/healthz`
3 * must keep answering 200 within a second. 0.7.277 answered at go-live and
4 * hung at its first `/boards`; this is the check that would have caught it.
5 */
6 import { afterEach, expect, test } from "bun:test";
7 import { HEAVY, smoke } from "../src/release/smoke.ts";
8
9 let server: ReturnType<typeof Bun.serve> | undefined;
10 afterEach(() => server?.stop(true));
11
12 /** A forge that is good until `/boards` loads, then answers per `after`. */
13 function stub(after: { healthzMs?: number; boards?: number } = {}) {
14 const seen: string[] = [];
15 let boarded = false;
16 server = Bun.serve({
17 port: 0,
18 hostname: "127.0.0.1",
19 async fetch(req) {
20 const path = new URL(req.url).pathname;
21 seen.push(path);
22 if (path === "/") return new Response('<link rel="stylesheet" href="/a.css">', { headers: { "content-type": "text/html" } });
23 if (path === "/a.css") return new Response("a{}", { headers: { "content-type": "text/css" } });
24 if (path === "/board") return new Response(null, { status: 302, headers: { location: "/signin" } });
25 if (path === "/sw.js") return new Response('if (r.mode === "navigate") { e.respondWith(fetch(r)); return; }', { headers: { "content-type": "text/javascript" } });
26 if (path === "/boards") {
27 boarded = true;
28 return new Response("boards", { status: after.boards ?? 200 });
29 }
30 if (path === "/healthz") {
31 if (boarded && after.healthzMs) await Bun.sleep(after.healthzMs);
32 return new Response("ok");
33 }
34 return new Response("no", { status: 404 });
35 },
36 });
37 return { url: `http://127.0.0.1:${server.port}`, seen };
38 }
39
40 const steady = { anonymousBoard: true, steadyMs: 50, pollMs: 10 };
41
42 test("a forge that stays up passes, having loaded every heavy page and polled /healthz", async () => {
43 const s = stub();
44 expect(await smoke(s.url, "", steady)).toEqual([]);
45 for (const path of HEAVY) expect(s.seen).toContain(path);
46 const after = s.seen.slice(s.seen.lastIndexOf("/boards"));
47 expect(after.filter((p) => p === "/healthz").length).toBeGreaterThanOrEqual(5);
48 });
49
50 test("a /healthz that goes quiet after /boards fails the smoke", async () => {
51 const failures = await smoke(stub({ healthzMs: 1_500 }).url, "", steady);
52 expect(failures).toHaveLength(1);
53 expect(failures[0]!.url).toEndWith("/healthz");
54 expect(failures[0]!.why).toContain("after go-live /healthz");
55 });
56
57 test("a heavy page answering 5xx fails the smoke, naming it", async () => {
58 const failures = await smoke(stub({ boards: 503 }).url, "", steady);
59 expect(failures.map((f) => [new URL(f.url).pathname, f.status])).toEqual([["/boards", 503]]);
60 });
61
62 test("without steadyMs the smoke never loads a heavy page", async () => {
63 const s = stub();
64 expect(await smoke(s.url, "", { anonymousBoard: true })).toEqual([]);
65 expect(s.seen).not.toContain("/boards");
66 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs