Public: anyone can read this; the crew can change it.
P0 INCIDENT: 0.7.277 hung the forge — readIdlePulls scans every pull.pushed event synchronously on each /boards load (8 s); bound the query, cache the panel, guard with a test and a release smoke #6860
mason/forge:mason/6840-p0-incident-0-7-277-hung-the-forge-readi → main 5 commits fork
13 files
changed +424
−21
docs/CACHING.md +16
| @@ -291,6 +291,21 @@ fixture; all three caches hit): | ||
| 291 | 291 | The anonymous door (1.5 ms, 10 statements), the app shelf, and the |
| 292 | 292 | "On you" card. Each costs less than its cache's key would. |
| 293 | 293 | |
| 294 | ## The flow panel (`/boards`) | |
| 295 | ||
| 296 | ### The idle pulls: per audience (#6840) | |
| 297 | ||
| 298 | - **What:** `readIdlePulls` (`flow/idle_pulls.ts`) for the `/boards` flow | |
| 299 | panel and the TUI flow screen, read through `flow/idle_cache.ts`. The flow | |
| 300 | tick in `main.ts` warms both audiences every minute. | |
| 301 | - **Key:** `idle:{all|public}:{UTC minute}`; agents read `public` only. | |
| 302 | - **Invalidated by:** every write, through the write clock. | |
| 303 | - **Max age:** 30 seconds. | |
| 304 | - **Measured** (`test/flow-idle-pulls-scale.test.ts`: 5,000 pulls, 500,000 | |
| 305 | events): a miss is about 10 ms, a seek on `event_pull_pushed` per open | |
| 306 | pull. Before that index it scanned every pushed event per pull and held | |
| 307 | the event loop for seconds: 0.7.277 went down on it. | |
| 308 | ||
| 294 | 309 | ## The writes that invalidate |
| 295 | 310 | |
| 296 | 311 | Each row is an action (or, for a push, the post-receive hook, the one |
| @@ -411,6 +426,7 @@ warm: | ||
| 411 | 426 | | home feed page | viewer, forge head | 30 s | |
| 412 | 427 | | home KPI row | forge head, UTC hour; admin-gated | 30 s | |
| 413 | 428 | | home pinned panel | board blob, panel, forge head; admin-gated | 30 s | |
| 429 | | idle pulls | audience, forge head, UTC minute | 30 s | | |
| 414 | 430 | |
| 415 | 431 | ## Raw files at a full commit id (#3381) |
| 416 | 432 |
src/core/migration-pins/20261027120000-event-pull-pushed.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 7b96b56a2c22ec44345bbecf79d0c35312de39b76c16f1965772e70ab7d355c6 |
src/core/migrations/20261027120000-event-pull-pushed.ts added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | // Each pull's newest push, found by index (#6840): `flow/idle_pulls.ts` asks | |
| 2 | // for MAX(created_at) of one repository's `pull.pushed` events for one pull | |
| 3 | // number. Without this it scanned every pushed event forge-wide per pull and | |
| 4 | // held the event loop for seconds. Partial, so only pushes pay for it. | |
| 5 | export default ` | |
| 6 | CREATE INDEX IF NOT EXISTS event_pull_pushed | |
| 7 | ON event(repo_id, json_extract(payload, '$.number'), created_at) | |
| 8 | WHERE kind = 'pull.pushed'; | |
| 9 | `; |
src/flow/idle_cache.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | /** | |
| 2 | * What the flow panel and the TUI flow screen remember of the idle pulls | |
| 3 | * (#6840): `readIdlePulls`, valid while the forge's write clock has not moved | |
| 4 | * and for 30 seconds at most, like home's figures (`web/studio/home_cache.ts`). | |
| 5 | * The flow tick in `main.ts` warms it, so a quiet forge's `/boards` reads a | |
| 6 | * remembered answer; after a write the next read recomputes, which the | |
| 7 | * `event_pull_pushed` index keeps to milliseconds. | |
| 8 | */ | |
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { writeStamp } from "../core/write_clock.ts"; | |
| 11 | import { Stamped } from "../kernel/stamped.ts"; | |
| 12 | import { readIdlePulls, type IdlePull } from "./idle_pulls.ts"; | |
| 13 | ||
| 14 | export const IDLE_MAX_AGE_MS = 30_000; | |
| 15 | ||
| 16 | const caches = new WeakMap<Database, Stamped<IdlePull[]>>(); | |
| 17 | ||
| 18 | /** The idle-pull cache of `db`: for tests, which count its hits. */ | |
| 19 | export function idleCache(db: Database): Stamped<IdlePull[]> { | |
| 20 | let cache = caches.get(db); | |
| 21 | if (cache === undefined) { | |
| 22 | cache = new Stamped<IdlePull[]>({ maxEntries: 4, maxAgeMs: IDLE_MAX_AGE_MS }); | |
| 23 | caches.set(db, cache); | |
| 24 | } | |
| 25 | return cache; | |
| 26 | } | |
| 27 | ||
| 28 | /** | |
| 29 | * The idle pulls, remembered per audience and UTC minute, so a caller asking | |
| 30 | * about another moment never reads this one's answer. `publicOnly` as in | |
| 31 | * `readIdlePulls`. | |
| 32 | */ | |
| 33 | export function cachedIdlePulls(db: Database, now: Date, publicOnly: boolean): IdlePull[] { | |
| 34 | const stamp = writeStamp(db); | |
| 35 | if (stamp === undefined) return readIdlePulls(db, now, publicOnly); | |
| 36 | const key = `idle:${publicOnly ? "public" : "all"}:${now.toISOString().slice(0, 16)}`; | |
| 37 | return idleCache(db).get(key, stamp, () => readIdlePulls(db, now, publicOnly)); | |
| 38 | } | |
| 39 | ||
| 40 | /** Reads both audiences into the cache: the flow tick's part. */ | |
| 41 | export function warmIdlePulls(db: Database, now = new Date()): void { | |
| 42 | cachedIdlePulls(db, now, false); | |
| 43 | cachedIdlePulls(db, now, true); | |
| 44 | } |
src/flow/idle_pulls.ts +18 −15
| @@ -1,7 +1,11 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * Idle pulls (#6470): open, unmerged pulls opened by a crew agent whose head |
| 3 | 3 | * has not moved for 72 h. A pull "moves" at its newest `pull.pushed` event, or |
| 4 | * at its opening when it has none. Oldest first, at most 20. Nothing shows it yet. | |
| 4 | * at its opening when it has none. Oldest first, at most 20. | |
| 5 | * | |
| 6 | * Only open, unmerged pulls are read, and each one's newest push is a seek on | |
| 7 | * `event_pull_pushed` (#6840): a scan of `event` per pull hung the forge. | |
| 8 | * Pages read it through `idle_cache.ts`, never on the request path. | |
| 5 | 9 | */ |
| 6 | 10 | |
| 7 | 11 | import type { Database } from "bun:sqlite"; |
| @@ -25,20 +29,19 @@ export function readIdlePulls(db: Database, now = new Date(), publicOnly = false | ||
| 25 | 29 | const cutoff = new Date(now.getTime() - IDLE_HOURS * HOUR_MS).toISOString(); |
| 26 | 30 | const rows = db |
| 27 | 31 | .query<{ number: number; owner: string; name: string; moved_at: string; body: string }, [number, string, number]>( |
| 28 | `SELECT i.number AS number, ro.username AS owner, r.name AS name, m.at AS moved_at, i.body AS body | |
| 29 | FROM pull p | |
| 30 | JOIN issue i ON i.id = p.issue_id | |
| 31 | JOIN repo r ON r.id = i.repo_id | |
| 32 | JOIN user ro ON ro.id = r.owner_id | |
| 33 | JOIN user u ON u.id = i.author_id AND u.is_agent = 1 | |
| 34 | JOIN (SELECT p2.issue_id AS issue_id, | |
| 35 | COALESCE((SELECT MAX(e.created_at) FROM event e | |
| 36 | WHERE e.kind = 'pull.pushed' AND e.repo_id = i2.repo_id | |
| 37 | AND json_extract(e.payload, '$.number') = i2.number), i2.created_at) AS at | |
| 38 | FROM pull p2 JOIN issue i2 ON i2.id = p2.issue_id) m ON m.issue_id = i.id | |
| 39 | WHERE (?1 = 0 OR r.visibility = 'public') AND i.state = 'open' AND p.merged_at IS NULL | |
| 40 | AND m.at <= ?2 | |
| 41 | ORDER BY m.at ASC, i.number ASC | |
| 32 | `SELECT number, owner, name, moved_at, body FROM ( | |
| 33 | SELECT i.number AS number, ro.username AS owner, r.name AS name, i.body AS body, | |
| 34 | COALESCE((SELECT MAX(e.created_at) FROM event e INDEXED BY event_pull_pushed | |
| 35 | WHERE e.kind = 'pull.pushed' AND e.repo_id = i.repo_id | |
| 36 | AND json_extract(e.payload, '$.number') = +i.number), i.created_at) AS moved_at | |
| 37 | FROM pull p | |
| 38 | JOIN issue i ON i.id = p.issue_id | |
| 39 | JOIN repo r ON r.id = i.repo_id | |
| 40 | JOIN user ro ON ro.id = r.owner_id | |
| 41 | JOIN user u ON u.id = i.author_id AND u.is_agent = 1 | |
| 42 | WHERE (?1 = 0 OR r.visibility = 'public') AND i.state = 'open' AND p.merged_at IS NULL) | |
| 43 | WHERE moved_at <= ?2 | |
| 44 | ORDER BY moved_at ASC, number ASC | |
| 42 | 45 | LIMIT ?3`, |
| 43 | 46 | ) |
| 44 | 47 | .all(publicOnly ? 1 : 0, cutoff, IDLE_LIMIT); |
src/main.ts +3
| @@ -55,6 +55,7 @@ import { graphInstagram } from "./crew/instagram.ts"; | ||
| 55 | 55 | import { peopleOnCrews } from "./core/agents.ts"; |
| 56 | 56 | import { startNotifier } from "./crew/notify.ts"; |
| 57 | 57 | import { tickClog } from "./flow/clog_tick.ts"; |
| 58 | import { warmIdlePulls } from "./flow/idle_cache.ts"; | |
| 58 | 59 | import { sweepStaleAsks } from "./crew/stale_asks.ts"; |
| 59 | 60 | import { recordAskLedger } from "./web/studio/asks.ts"; |
| 60 | 61 | import { startPromptRoutine } from "./crew/prompts.ts"; |
| @@ -552,9 +553,11 @@ const processTick = (): void => { | ||
| 552 | 553 | } |
| 553 | 554 | }; |
| 554 | 555 | // The clog watch (#6524) reads the flow every minute; its state is the database's. |
| 556 | // The idle pulls ride it into their cache (#6840), so /boards reads them warm. | |
| 555 | 557 | const clogTick = (): void => { |
| 556 | 558 | try { |
| 557 | 559 | tickClog(app.db, app.bus); |
| 560 | warmIdlePulls(app.db); | |
| 558 | 561 | } catch (error) { |
| 559 | 562 | app.log.warn("clog tick failed", { error: String(error) }); |
| 560 | 563 | } |
src/release/prove.ts +12 −1
| @@ -6,6 +6,8 @@ | ||
| 6 | 6 | * restart of the same version never runs it again and vitals reads it |
| 7 | 7 | * without the network. A failure is a `release` card, paged once by the |
| 8 | 8 | * cause pager, until a later version passes. It gives up after ten minutes. |
| 9 | * The smoke loads the heaviest pages and watches `/healthz` for a minute | |
| 10 | * after (#6840): a version that answers and then hangs is not proven. | |
| 9 | 11 | */ |
| 10 | 12 | |
| 11 | 13 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| @@ -17,6 +19,8 @@ import { ownProbeHeaders } from "../kernel/probe.ts"; | ||
| 17 | 19 | /** How long the public URL has to answer as the new version. */ |
| 18 | 20 | export const GIVE_UP_MS = 10 * 60_000; |
| 19 | 21 | const POLL_MS = 2_000; |
| 22 | /** How long `/healthz` must stay fast after the smoke's heavy pages load (#6840). */ | |
| 23 | export const STEADY_MS = 60_000; | |
| 20 | 24 | |
| 21 | 25 | export interface Proof { |
| 22 | 26 | readonly version: string; |
| @@ -104,7 +108,14 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 104 | 108 | if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`); |
| 105 | 109 | await sleep(POLL_MS); |
| 106 | 110 | } |
| 107 | const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true, ...(o.appHost ? { appHost: o.appHost } : {}) }); | |
| 111 | const failures = await smoke(o.url, o.token ?? "", { | |
| 112 | fetch: doFetch, | |
| 113 | anonymousBoard: true, | |
| 114 | steadyMs: STEADY_MS, | |
| 115 | now, | |
| 116 | sleep, | |
| 117 | ...(o.appHost ? { appHost: o.appHost } : {}), | |
| 118 | }); | |
| 108 | 119 | return finish(failures[0] === undefined ? undefined : checkLine(failures[0])); |
| 109 | 120 | } catch (error) { |
| 110 | 121 | return finish(`the smoke checks could not run: ${String(error)}`); |
src/release/smoke.ts +54 −1
| @@ -4,7 +4,11 @@ | ||
| 4 | 4 | * must answer 200; the stylesheet linked from `/` must be a non-empty |
| 5 | 5 | * `text/css` 200; `/sw.js` must be a 200 JavaScript response whose navigation |
| 6 | 6 | * branch never answers with `Response.error()`. With `version`, `/healthz` is |
| 7 | * first polled until `X-Forge-Version` is that version. | |
| 7 | * first polled until `X-Forge-Version` is that version. With `steadyMs`, the | |
| 8 | * heaviest pages (`HEAVY`) are loaded and must answer without a 5xx, then | |
| 9 | * `/healthz` is polled for that long and every answer must be a 200 within | |
| 10 | * `PROBE_MS`: 0.7.277 went live, answered, and hung at its first `/boards` | |
| 11 | * (#6840), so a release proves it stays up, not only that it came up. | |
| 8 | 12 | * |
| 9 | 13 | * The running forge proves each new version with it (`release/prove.ts`), so |
| 10 | 14 | * it lives here: the image ships src/ and nothing else (#2652). |
| @@ -18,6 +22,11 @@ export const AGENTS: Record<string, string> = { | ||
| 18 | 22 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", |
| 19 | 23 | }; |
| 20 | 24 | |
| 25 | /** The pages that read the most per request, loaded before the steady watch (#6840). */ | |
| 26 | export const HEAVY = ["/boards", "/explore", "/"] as const; | |
| 27 | /** The longest a `/healthz` may take while steady: the edge's probe gives up at a second. */ | |
| 28 | export const PROBE_MS = 1_000; | |
| 29 | ||
| 21 | 30 | export interface Failure { |
| 22 | 31 | url: string; |
| 23 | 32 | agent: string; |
| @@ -61,6 +70,11 @@ export interface Options { | ||
| 61 | 70 | anonymousBoard?: boolean; |
| 62 | 71 | /** An app's URL on this forge: `GET /` there must never be a forge page (#6726). */ |
| 63 | 72 | appHost?: string; |
| 73 | /** Watch `/healthz` this long after the heavy pages load; 0 (the default) skips it. */ | |
| 74 | steadyMs?: number; | |
| 75 | /** The clock a probe's latency is measured on, and the pause between steady polls: seams for tests. */ | |
| 76 | now?: () => number; | |
| 77 | sleep?: (ms: number) => Promise<void>; | |
| 64 | 78 | } |
| 65 | 79 | |
| 66 | 80 | /** The redirect statuses a sign-in answers with. */ |
| @@ -181,5 +195,44 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr | ||
| 181 | 195 | failures.push({ url, agent: "desktop", status: "error", contentType: "", why: String(error) }); |
| 182 | 196 | } |
| 183 | 197 | } |
| 198 | if (options.steadyMs) failures.push(...(await steady(root, token, options.steadyMs, options))); | |
| 199 | return failures; | |
| 200 | } | |
| 201 | ||
| 202 | /** The heavy pages, then `/healthz` every `pollMs` for `ms`: the failures, at most one per page and one for the watch. */ | |
| 203 | async function steady(root: URL, token: string, ms: number, options: Options): Promise<Failure[]> { | |
| 204 | const fetch = options.fetch ?? globalThis.fetch; | |
| 205 | const now = options.now ?? performance.now.bind(performance); | |
| 206 | const sleep = options.sleep ?? Bun.sleep; | |
| 207 | const pollMs = options.pollMs ?? 1_000; | |
| 208 | const failures: Failure[] = []; | |
| 209 | const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {}; | |
| 210 | for (const path of HEAVY) { | |
| 211 | const url = new URL(path, root).href; | |
| 212 | try { | |
| 213 | const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(10_000), headers: { "user-agent": AGENTS["desktop"]!, ...auth } }); | |
| 214 | await res.body?.cancel(); | |
| 215 | if (res.status >= 500) failures.push({ url, agent: "desktop", status: res.status, contentType: res.headers.get("content-type") ?? "", why: "a heavy page answered 5xx" }); | |
| 216 | } catch (error) { | |
| 217 | failures.push({ url, agent: "desktop", status: "error", contentType: "", why: `a heavy page did not answer: ${String(error)}` }); | |
| 218 | } | |
| 219 | } | |
| 220 | const url = new URL("/healthz", root).href; | |
| 221 | for (let polls = Math.max(1, Math.ceil(ms / pollMs)); polls > 0; polls--) { | |
| 222 | const started = now(); | |
| 223 | try { | |
| 224 | const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(PROBE_MS) }); | |
| 225 | await res.body?.cancel(); | |
| 226 | const took = now() - started; | |
| 227 | if (res.status !== 200 || took > PROBE_MS) { | |
| 228 | failures.push({ url, agent: "-", status: res.status, contentType: "", why: `after go-live /healthz answered ${res.status} in ${Math.round(took)} ms` }); | |
| 229 | break; | |
| 230 | } | |
| 231 | } catch (error) { | |
| 232 | failures.push({ url, agent: "-", status: "error", contentType: "", why: `after go-live /healthz stopped answering: ${String(error)}` }); | |
| 233 | break; | |
| 234 | } | |
| 235 | await sleep(pollMs); | |
| 236 | } | |
| 184 | 237 | return failures; |
| 185 | 238 | } |
src/tui/screens/flow.ts +3 −2
| @@ -19,7 +19,8 @@ import { pullCounts } from "../../crew/hot_files.ts"; | ||
| 19 | 19 | import { warmingLine } from "../../crew/hot_history.ts"; |
| 20 | 20 | import type { HeldRow } from "../../flow/held.ts"; |
| 21 | 21 | import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts"; |
| 22 | import { IDLE_LIMIT, readIdlePulls } from "../../flow/idle_pulls.ts"; | |
| 22 | import { IDLE_LIMIT } from "../../flow/idle_pulls.ts"; | |
| 23 | import { cachedIdlePulls } from "../../flow/idle_cache.ts"; | |
| 23 | 24 | import { pileSays, readPile } from "../../flow/pile.ts"; |
| 24 | 25 | import { readPipeline, STAGE_LABEL, verdict } from "../../flow/pipeline.ts"; |
| 25 | 26 | import { parkedSays, readParked } from "../../flow/parked.ts"; |
| @@ -119,7 +120,7 @@ const IDLE_SHOWN = 5; | ||
| 119 | 120 | |
| 120 | 121 | /** A blank line, `idle: N`, then a line per idle crew pull (`flow/idle_pulls.ts`, #6479), oldest first; nothing when none. */ |
| 121 | 122 | export function idleLines(db: Database, now: Date, publicOnly: boolean, inner: number): string[] { |
| 122 | const idle = readIdlePulls(db, now, publicOnly); | |
| 123 | const idle = cachedIdlePulls(db, now, publicOnly); | |
| 123 | 124 | if (idle.length === 0) return []; |
| 124 | 125 | return [ |
| 125 | 126 | "", |
src/web/pages/flow-panel.ts +3 −2
| @@ -36,7 +36,8 @@ import { type QueueRow, redBatchWords } from "../../flow/queue.ts"; | ||
| 36 | 36 | import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts"; |
| 37 | 37 | import { warmingLine } from "../../crew/hot_history.ts"; |
| 38 | 38 | import { getRepoById } from "../../core/repos.ts"; |
| 39 | import { IDLE_LIMIT, readIdlePulls, type IdlePull } from "../../flow/idle_pulls.ts"; | |
| 39 | import { IDLE_LIMIT, type IdlePull } from "../../flow/idle_pulls.ts"; | |
| 40 | import { cachedIdlePulls } from "../../flow/idle_cache.ts"; | |
| 40 | 41 | import { readParked } from "../../flow/parked.ts"; |
| 41 | 42 | import { hoursWords, readLeadTime } from "../../flow/lead_time.ts"; |
| 42 | 43 | import { buildYieldSays, readBuildYield } from "../../flow/build_yield.ts"; |
| @@ -113,7 +114,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm | ||
| 113 | 114 | const latency = readReviewLatency(app.db, new Date(now), agent); |
| 114 | 115 | const openers = readOpeners(app.db, new Date(now), agent); |
| 115 | 116 | const pile = pileSays(readPile(app.db, reader.kind === "agent")); |
| 116 | const idle = readIdlePulls(app.db, new Date(now), reader.kind === "agent"); | |
| 117 | const idle = cachedIdlePulls(app.db, new Date(now), reader.kind === "agent"); | |
| 117 | 118 | const parked = readParked(app.db, new Date(now), reader.kind === "agent"); |
| 118 | 119 | const newest = parked.newest[0]; |
| 119 | 120 | const listed = asked(url); |
test/flow-idle-cache.test.ts added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | /** | |
| 2 | * The idle-pull cache (#6840): the flow tick warms it, `/boards` and the TUI | |
| 3 | * read the warm answer while nothing is written, and a write recomputes. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { cachedIdlePulls, idleCache, warmIdlePulls } from "../src/flow/idle_cache.ts"; | |
| 12 | ||
| 13 | const NOW = new Date("2026-10-27T12:00:00.000Z"); | |
| 14 | const ago = (h: number): string => new Date(NOW.getTime() - h * 3_600_000).toISOString(); | |
| 15 | ||
| 16 | let dir: string; | |
| 17 | let app: App; | |
| 18 | let agent: string; | |
| 19 | ||
| 20 | beforeAll(async () => { | |
| 21 | dir = await mkdtemp(join(tmpdir(), "idle-cache-")); | |
| 22 | app = createApp({ dataDir: dir, port: 0 }); | |
| 23 | const u = await createUser(app.db, { | |
| 24 | origin: "seed", trusted: true, username: "scout", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true, | |
| 25 | }); | |
| 26 | if (!u.ok) throw new Error(u.error); | |
| 27 | agent = u.value.id; | |
| 28 | app.db.run("UPDATE user SET is_agent = 1 WHERE id = ?1", [agent]); | |
| 29 | app.db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES ('pub', ?1, 'pub', 'public')", [agent]); | |
| 30 | }); | |
| 31 | afterAll(async () => { | |
| 32 | app.close?.(); | |
| 33 | await rm(dir, { recursive: true, force: true }); | |
| 34 | }); | |
| 35 | ||
| 36 | function openPull(n: number): void { | |
| 37 | app.db.run( | |
| 38 | "INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, 'pub', ?2, 'pull', 't', '', ?3, 'open', ?4)", | |
| 39 | [`p${n}`, n, agent, ago(100)], | |
| 40 | ); | |
| 41 | app.db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', ?2, '', 'h')", [`p${n}`, `b${n}`]); | |
| 42 | } | |
| 43 | ||
| 44 | test("a warmed answer is read again until something is written", () => { | |
| 45 | openPull(1); | |
| 46 | warmIdlePulls(app.db, NOW); | |
| 47 | const cache = idleCache(app.db); | |
| 48 | const hits = cache.hits; | |
| 49 | expect(cachedIdlePulls(app.db, NOW, false).map((p) => p.pull)).toEqual([1]); | |
| 50 | expect(cachedIdlePulls(app.db, NOW, true).map((p) => p.pull)).toEqual([1]); | |
| 51 | expect(cache.hits).toBe(hits + 2); | |
| 52 | openPull(2); | |
| 53 | expect(cachedIdlePulls(app.db, NOW, false).map((p) => p.pull)).toEqual([1, 2]); | |
| 54 | }); | |
| 55 | ||
| 56 | test("another minute is another answer", () => { | |
| 57 | expect(cachedIdlePulls(app.db, new Date(NOW.getTime() - 40 * 3_600_000), false)).toEqual([]); | |
| 58 | }); |
test/flow-idle-pulls-scale.test.ts added +137
| @@ -0,0 +1,137 @@ | ||
| 1 | /** | |
| 2 | * Idle pulls at the forge's scale (#6840): 0.7.277 hung the forge because | |
| 3 | * `readIdlePulls` scanned every `pull.pushed` event once per pull. Seeded with | |
| 4 | * 5,000 pulls and 500,000 events, it answers in under 100 ms and its plan | |
| 5 | * never scans `event`; and loading `/boards` over that data never holds the | |
| 6 | * event loop long enough for `/healthz` to go quiet for 250 ms. | |
| 7 | */ | |
| 8 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { createToken } from "../src/core/tokens.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { serve } from "../src/http/server.ts"; | |
| 16 | import { readIdlePulls } from "../src/flow/idle_pulls.ts"; | |
| 17 | ||
| 18 | const NOW = new Date("2026-10-27T12:00:00.000Z"); | |
| 19 | const PULLS = 5_000; | |
| 20 | const EVENTS = 500_000; | |
| 21 | const REPOS = 10; | |
| 22 | ||
| 23 | let dir: string; | |
| 24 | let app: App; | |
| 25 | let servers: Awaited<ReturnType<typeof serve>>; | |
| 26 | let token: string; | |
| 27 | ||
| 28 | beforeAll(async () => { | |
| 29 | dir = await mkdtemp(join(tmpdir(), "idle-scale-")); | |
| 30 | app = createApp({ dataDir: dir, port: 0 }); | |
| 31 | const owner = await createUser(app.db, { | |
| 32 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true, | |
| 33 | }); | |
| 34 | const crew = await createUser(app.db, { | |
| 35 | origin: "seed", trusted: true, username: "scout", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: false, | |
| 36 | }); | |
| 37 | if (!owner.ok || !crew.ok) throw new Error("seed users"); | |
| 38 | const db = app.db; | |
| 39 | db.run("UPDATE user SET is_agent = 1 WHERE id = ?1", [crew.value.id]); | |
| 40 | db.transaction(() => { | |
| 41 | for (let r = 0; r < REPOS; r++) | |
| 42 | db.run("INSERT INTO repo (id, owner_id, name, visibility) VALUES (?1, ?2, ?1, ?3)", [`r${r}`, owner.value.id, r % 2 ? "public" : "private"]); | |
| 43 | // A pull per number, spread over the repositories; one in seven open, the | |
| 44 | // rest closed or merged, as on a forge that has run a while. | |
| 45 | db.run( | |
| 46 | `WITH RECURSIVE n(k) AS (SELECT 1 UNION ALL SELECT k + 1 FROM n WHERE k < ?1) | |
| 47 | INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) | |
| 48 | SELECT 'p' || k, 'r' || (k % ?2), k, 'pull', 't', '', ?3, CASE WHEN k % 7 = 0 THEN 'open' ELSE 'closed' END, | |
| 49 | strftime('%Y-%m-%dT%H:%M:%fZ', ?4, '-' || (100 + k % 400) || ' hours') FROM n`, | |
| 50 | [PULLS, REPOS, crew.value.id, NOW.toISOString()], | |
| 51 | ); | |
| 52 | db.run( | |
| 53 | `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) | |
| 54 | SELECT id, 'main', 'b' || number, '', 'h', CASE WHEN number % 3 = 0 AND state = 'closed' THEN created_at END FROM issue`, | |
| 55 | ); | |
| 56 | // `seq` given, so the outbox trigger (migration 0069) stands aside. | |
| 57 | // Half the events are pushes to some pull of some repository, the rest other kinds. | |
| 58 | db.run( | |
| 59 | `WITH RECURSIVE n(k) AS (SELECT 1 UNION ALL SELECT k + 1 FROM n WHERE k < ?1) | |
| 60 | INSERT INTO event (id, seq, kind, repo_id, payload, created_at) | |
| 61 | SELECT 'e' || k, k, CASE WHEN k % 2 = 0 THEN 'pull.pushed' ELSE 'issue.commented' END, 'r' || (k % ?2), | |
| 62 | json_object('number', 1 + k % ?3), strftime('%Y-%m-%dT%H:%M:%fZ', ?4, '-' || (k % 600) || ' hours') FROM n`, | |
| 63 | [EVENTS, REPOS, PULLS, NOW.toISOString()], | |
| 64 | ); | |
| 65 | })(); | |
| 66 | db.run("ANALYZE"); | |
| 67 | token = createToken(db, owner.value.id, "t", "admin").token; | |
| 68 | servers = await serve(app); | |
| 69 | }, 60_000); | |
| 70 | ||
| 71 | afterAll(async () => { | |
| 72 | servers.stop(); | |
| 73 | app.close?.(); | |
| 74 | await rm(dir, { recursive: true, force: true }); | |
| 75 | }); | |
| 76 | ||
| 77 | test("readIdlePulls answers 5,000 pulls and 500,000 events in under 100 ms", () => { | |
| 78 | readIdlePulls(app.db, NOW, false); | |
| 79 | for (const publicOnly of [false, true]) { | |
| 80 | const started = performance.now(); | |
| 81 | const rows = readIdlePulls(app.db, NOW, publicOnly); | |
| 82 | expect(performance.now() - started).toBeLessThan(100); | |
| 83 | expect(rows.length).toBeGreaterThan(0); | |
| 84 | } | |
| 85 | }); | |
| 86 | ||
| 87 | test("each pull's newest push is a seek on event_pull_pushed, never a scan of event", () => { | |
| 88 | const plan = app.db | |
| 89 | .query<{ detail: string }, []>( | |
| 90 | `EXPLAIN QUERY PLAN SELECT MAX(e.created_at) FROM event e | |
| 91 | WHERE e.kind = 'pull.pushed' AND e.repo_id = 'r1' AND json_extract(e.payload, '$.number') = 11`, | |
| 92 | ) | |
| 93 | .all() | |
| 94 | .map((r) => r.detail); | |
| 95 | expect(plan.join("\n")).toContain("event_pull_pushed"); | |
| 96 | // The whole reader's plan: `event` is only ever searched, by that index. | |
| 97 | const sql = readerSql(); | |
| 98 | const whole = app.db.query<{ detail: string }, [number, string, number]>(`EXPLAIN QUERY PLAN ${sql}`).all(0, NOW.toISOString(), 20).map((r) => r.detail); | |
| 99 | const onEvent = whole.filter((d) => /\b(event|e)\b/.test(d) && /^(SCAN|SEARCH)/.test(d)); | |
| 100 | expect(onEvent.length).toBeGreaterThan(0); | |
| 101 | for (const d of onEvent) expect(d).toMatch(/^SEARCH e USING (COVERING )?INDEX event_pull_pushed/); | |
| 102 | }); | |
| 103 | ||
| 104 | test("/healthz never goes quiet for 250 ms while /boards loads", async () => { | |
| 105 | // Server and probe share this event loop: a synchronous read on the | |
| 106 | // request path shows as a gap between two answered probes. | |
| 107 | let polling = true; | |
| 108 | let worst = 0; | |
| 109 | const poll = (async () => { | |
| 110 | let last = performance.now(); | |
| 111 | while (polling) { | |
| 112 | const res = await fetch(`${servers.url}/healthz`); | |
| 113 | await res.body?.cancel(); | |
| 114 | expect(res.status).toBe(200); | |
| 115 | const at = performance.now(); | |
| 116 | worst = Math.max(worst, at - last); | |
| 117 | last = at; | |
| 118 | await Bun.sleep(10); | |
| 119 | } | |
| 120 | })(); | |
| 121 | for (let i = 0; i < 5; i++) { | |
| 122 | const res = await fetch(`${servers.url}/boards`, { headers: { authorization: `Bearer ${token}` } }); | |
| 123 | expect(res.status).toBe(200); | |
| 124 | expect(await res.text()).toContain('id="bottleneck"'); | |
| 125 | } | |
| 126 | polling = false; | |
| 127 | await poll; | |
| 128 | expect(worst).toBeLessThan(250); | |
| 129 | }, 30_000); | |
| 130 | ||
| 131 | /** The reader's statement, read from its source so the plan tested is the one shipped. */ | |
| 132 | function readerSql(): string { | |
| 133 | const source = require("node:fs").readFileSync(join(import.meta.dir, "../src/flow/idle_pulls.ts"), "utf8") as string; | |
| 134 | const sql = /\.query<[^`]*`([^`]*)`/.exec(source)?.[1]; | |
| 135 | if (sql === undefined) throw new Error("no statement in idle_pulls.ts"); | |
| 136 | return sql; | |
| 137 | } |
test/smoke-steady.test.ts added +66
| @@ -0,0 +1,66 @@ | ||
| 1 | /** | |
| 2 | * The smoke's steady watch (#6840): the heavy pages load, then `/healthz` | |
| 3 | * must keep answering 200 within a second. 0.7.277 answered at go-live and | |
| 4 | * hung at its first `/boards`; this is the check that would have caught it. | |
| 5 | */ | |
| 6 | import { afterEach, expect, test } from "bun:test"; | |
| 7 | import { HEAVY, smoke } from "../src/release/smoke.ts"; | |
| 8 | ||
| 9 | let server: ReturnType<typeof Bun.serve> | undefined; | |
| 10 | afterEach(() => server?.stop(true)); | |
| 11 | ||
| 12 | /** A forge that is good until `/boards` loads, then answers per `after`. */ | |
| 13 | function stub(after: { healthzMs?: number; boards?: number } = {}) { | |
| 14 | const seen: string[] = []; | |
| 15 | let boarded = false; | |
| 16 | server = Bun.serve({ | |
| 17 | port: 0, | |
| 18 | hostname: "127.0.0.1", | |
| 19 | async fetch(req) { | |
| 20 | const path = new URL(req.url).pathname; | |
| 21 | seen.push(path); | |
| 22 | if (path === "/") return new Response('<link rel="stylesheet" href="/a.css">', { headers: { "content-type": "text/html" } }); | |
| 23 | if (path === "/a.css") return new Response("a{}", { headers: { "content-type": "text/css" } }); | |
| 24 | if (path === "/board") return new Response(null, { status: 302, headers: { location: "/signin" } }); | |
| 25 | if (path === "/sw.js") return new Response('if (r.mode === "navigate") { e.respondWith(fetch(r)); return; }', { headers: { "content-type": "text/javascript" } }); | |
| 26 | if (path === "/boards") { | |
| 27 | boarded = true; | |
| 28 | return new Response("boards", { status: after.boards ?? 200 }); | |
| 29 | } | |
| 30 | if (path === "/healthz") { | |
| 31 | if (boarded && after.healthzMs) await Bun.sleep(after.healthzMs); | |
| 32 | return new Response("ok"); | |
| 33 | } | |
| 34 | return new Response("no", { status: 404 }); | |
| 35 | }, | |
| 36 | }); | |
| 37 | return { url: `http://127.0.0.1:${server.port}`, seen }; | |
| 38 | } | |
| 39 | ||
| 40 | const steady = { anonymousBoard: true, steadyMs: 50, pollMs: 10 }; | |
| 41 | ||
| 42 | test("a forge that stays up passes, having loaded every heavy page and polled /healthz", async () => { | |
| 43 | const s = stub(); | |
| 44 | expect(await smoke(s.url, "", steady)).toEqual([]); | |
| 45 | for (const path of HEAVY) expect(s.seen).toContain(path); | |
| 46 | const after = s.seen.slice(s.seen.lastIndexOf("/boards")); | |
| 47 | expect(after.filter((p) => p === "/healthz").length).toBeGreaterThanOrEqual(5); | |
| 48 | }); | |
| 49 | ||
| 50 | test("a /healthz that goes quiet after /boards fails the smoke", async () => { | |
| 51 | const failures = await smoke(stub({ healthzMs: 1_500 }).url, "", steady); | |
| 52 | expect(failures).toHaveLength(1); | |
| 53 | expect(failures[0]!.url).toEndWith("/healthz"); | |
| 54 | expect(failures[0]!.why).toContain("after go-live /healthz"); | |
| 55 | }); | |
| 56 | ||
| 57 | test("a heavy page answering 5xx fails the smoke, naming it", async () => { | |
| 58 | const failures = await smoke(stub({ boards: 503 }).url, "", steady); | |
| 59 | expect(failures.map((f) => [new URL(f.url).pathname, f.status])).toEqual([["/boards", 503]]); | |
| 60 | }); | |
| 61 | ||
| 62 | test("without steadyMs the smoke never loads a heavy page", async () => { | |
| 63 | const s = stub(); | |
| 64 | expect(await smoke(s.url, "", { anonymousBoard: true })).toEqual([]); | |
| 65 | expect(s.seen).not.toContain("/boards"); | |
| 66 | }); |
new above
offline