Public: anyone can read this; the crew can change it.
The page thread stops for half a second about five times a minute with no request running: its heap grows 5 MB a second and is collected in long pauses #13925
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13924-the-page-thread-stops-for-half-a-second → main 8 commits fork 3 threads · 3 unresolved
6 files
changed +287
−2
AGENTS.md +1
| @@ -54,3 +54,4 @@ The check queue hands work to a runner through `src/checks/runner.ts` (#13652, R | ||
| 54 | 54 | An app token's requests go behind people's and repeat reads are 304s (`src/http/app_turn.ts`, `repeat_reads.ts`, `perf/asking.ts`): see `docs/features/app-asking.md`. |
| 55 | 55 | `src/core/reports.ts`: `closedSet`, the ids of repositories whose owner closed reports from outside (#12321), read once per database and kept by `setReportsClosed`, the only writer, so every repository page's head link costs no statement; at most one id per closed repository, gone on restart (the rows are `report_closed`). `src/core/contribute_token.ts`: `openSet`, the ids of repositories that allow outside agents, read once per database and kept by `setContributeOpen`, the only writer, at most one id per allowing repository, gone on restart; a raw SQL write to `contribute_open` bypasses it. The terminal door for reports is `r` on the outside agents screen (`setOutsideReports`, admins). |
| 56 | 56 | `src/crew/beat.ts`: `lastBeat`, when each repository's beat last ran, and `declared`, the `beat` last read from each default branch with the commit it came from (#13709); at most one entry each per repository, gone on restart (a restart beats once at once; the declaration is a file in git). |
| 57 | `src/perf/job_alloc.ts`: `counted`, called by `during` around each job's synchronous slice, adds the JS heap's growth to the job's name (at most 128 names); the minute memory line reads and empties it (`memory allocated by jobs:`, #13924), so a timer that allocates with no request running is named. `bench/idle.ts` is the idle budget on a seeded forge (heap under 20 MiB a minute, no stall over 100 ms). Gone on restart. |
bench/idle.ts added +82
| @@ -0,0 +1,82 @@ | ||
| 1 | /** | |
| 2 | * The idle budget (#13924): boot the forge over a seeded data directory, ask it nothing, and | |
| 3 | * hold its serving thread to a flat heap and no stall over 100 ms. | |
| 4 | * | |
| 5 | * bun run bench/seed.ts --data .bench/data | |
| 6 | * bun run bench/idle.ts --data .bench/data [--secs 120] | |
| 7 | * | |
| 8 | * It reads the forge's own log: each `memory main:` line's heap, each `event loop blocked` line | |
| 9 | * after the boot's first 20 s. Heap growth is the least-squares slope of the heap readings | |
| 10 | * against the time they were logged, per minute, so a collection's sawtooth does not read as | |
| 11 | * growth or hide it; it needs a run long enough for several readings (a minute apart), and with | |
| 12 | * fewer than three it says so and reports none. `memory allocated by jobs:` lines name the timer | |
| 13 | * that did it. Exit 1 over budget. Nothing runs it on a push: it is run by hand, and the budget | |
| 14 | * is enforced only by whoever runs it. | |
| 15 | */ | |
| 16 | ||
| 17 | import { spawn } from "bun"; | |
| 18 | ||
| 19 | const MAX_GROWTH_MIB_PER_MIN = 20; | |
| 20 | const MAX_STALL_MS = 100; | |
| 21 | const BOOT_MS = 20_000; | |
| 22 | ||
| 23 | const arg = (name: string, fallback: string): string => { | |
| 24 | const at = process.argv.indexOf(`--${name}`); | |
| 25 | return at > 0 && process.argv[at + 1] !== undefined ? process.argv[at + 1]! : fallback; | |
| 26 | }; | |
| 27 | ||
| 28 | const data = arg("data", ".bench/data"); | |
| 29 | const secs = Number(arg("secs", "120")); | |
| 30 | const proc = spawn(["bun", "run", "src/main.ts"], { | |
| 31 | env: { ...process.env, FORGE_APP_DATA: data, FORGE_DATA: data, FORGE_ENGINE: "worker", PORT: arg("port", "3917") }, | |
| 32 | stdout: "pipe", | |
| 33 | stderr: "pipe", | |
| 34 | }); | |
| 35 | const lines: string[] = []; | |
| 36 | const started = Date.now(); | |
| 37 | const collect = async (stream: ReadableStream<Uint8Array>): Promise<void> => { | |
| 38 | const decoder = new TextDecoder(); | |
| 39 | let rest = ""; | |
| 40 | for await (const chunk of stream) { | |
| 41 | rest += decoder.decode(chunk); | |
| 42 | const parts = rest.split("\n"); | |
| 43 | rest = parts.pop() ?? ""; | |
| 44 | for (const line of parts) lines.push(`${Date.now() - started}\t${line}`); | |
| 45 | } | |
| 46 | }; | |
| 47 | void collect(proc.stdout); | |
| 48 | void collect(proc.stderr); | |
| 49 | await new Promise((r) => setTimeout(r, secs * 1000)); | |
| 50 | proc.kill(); | |
| 51 | ||
| 52 | const afterTimed = lines | |
| 53 | .map((l) => ({ at: Number(l.split("\t", 1)[0]), text: l.slice(l.indexOf("\t") + 1) })) | |
| 54 | .filter((l) => l.at > BOOT_MS); | |
| 55 | const after = afterTimed.map((l) => l.text); | |
| 56 | const mib = (text: string, re: RegExp): number | undefined => { | |
| 57 | const m = re.exec(text); | |
| 58 | return m === null ? undefined : Number(m[1]); | |
| 59 | }; | |
| 60 | const readings = afterTimed.flatMap((l) => { | |
| 61 | const heap = l.text.includes("memory main:") ? mib(l.text, /heap ([\d.]+) MiB/) : undefined; | |
| 62 | return heap === undefined ? [] : [{ minutes: l.at / 60_000, heap }]; | |
| 63 | }); | |
| 64 | const heaps = readings.map((r) => r.heap); | |
| 65 | const stalls = after.flatMap((l) => (l.includes("event loop blocked") ? [mib(l, /blocked (\d+) ms/)] : [])).filter((n): n is number => n !== undefined); | |
| 66 | /** Least-squares slope of heap (MiB) over time (minutes); undefined with under three readings. */ | |
| 67 | function slope(points: readonly { minutes: number; heap: number }[]): number | undefined { | |
| 68 | if (points.length < 3) return undefined; | |
| 69 | const n = points.length; | |
| 70 | const mx = points.reduce((t, p) => t + p.minutes, 0) / n; | |
| 71 | const my = points.reduce((t, p) => t + p.heap, 0) / n; | |
| 72 | const sxx = points.reduce((t, p) => t + (p.minutes - mx) ** 2, 0); | |
| 73 | if (sxx === 0) return undefined; | |
| 74 | return points.reduce((t, p) => t + (p.minutes - mx) * (p.heap - my), 0) / sxx; | |
| 75 | } | |
| 76 | const growth = slope(readings); | |
| 77 | const allocated = after.filter((l) => l.includes("memory allocated by jobs:")); | |
| 78 | console.log(`idle ${secs} s: heap readings ${heaps.join(", ") || "none"} MiB; growth ${growth === undefined ? "unmeasured (run longer)" : `${growth.toFixed(1)} MiB/min (slope)`}; stalls ${stalls.join(", ") || "none"} ms`); | |
| 79 | for (const l of allocated) console.log(l); | |
| 80 | const bad = (growth ?? 0) > MAX_GROWTH_MIB_PER_MIN || stalls.some((ms) => ms > MAX_STALL_MS); | |
| 81 | if (bad) console.log(`over budget: ${MAX_GROWTH_MIB_PER_MIN} MiB/min and ${MAX_STALL_MS} ms`); | |
| 82 | process.exit(bad ? 1 : 0); |
src/boards/memory.ts +4 −1
| @@ -16,8 +16,9 @@ | ||
| 16 | 16 | import type { Database } from "bun:sqlite"; |
| 17 | 17 | import type { App } from "../app.ts"; |
| 18 | 18 | import { quietly } from "../core/write_clock.ts"; |
| 19 | import { during } from "../perf/jobs.ts"; | |
| 19 | import { allocationsOf, during } from "../perf/jobs.ts"; | |
| 20 | 20 | import { heldLine, heldNow } from "../perf/held.ts"; |
| 21 | import { allocationLine } from "../perf/job_alloc.ts"; | |
| 21 | 22 | import { hearWorkers, memoryLine, memoryWatch, readNow } from "../perf/memory_watch.ts"; |
| 22 | 23 | |
| 23 | 24 | export const MEMORY_EVERY_MS = 60_000; |
| @@ -78,6 +79,8 @@ export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemoryS | ||
| 78 | 79 | memoryWatch.note(reading.rss, reading.heap, now, reading.current); |
| 79 | 80 | app.log.info(memoryLine(reading, process.uptime() * 1000)); |
| 80 | 81 | app.log.info(heldLine(heldNow(app))); |
| 82 | const allocated = allocationLine(allocationsOf(app)); | |
| 83 | if (allocated !== undefined) app.log.info(allocated); | |
| 81 | 84 | } catch (error) { |
| 82 | 85 | app.log.warn("memory sample failed", { error: String(error) }); |
| 83 | 86 | } |
src/perf/job_alloc.ts added +64
| @@ -0,0 +1,64 @@ | ||
| 1 | /** | |
| 2 | * What each background job allocated (#13924): `during` (`jobs.ts`) reads the JS heap's size | |
| 3 | * (`heapSize()`, constant time) on either side of a job's synchronous slice and adds any growth | |
| 4 | * to the job's name. A collection inside a slice shows as no growth, so this is a floor, and | |
| 5 | * what a job allocates after an await is not seen; but a timer that allocates on every tick shows | |
| 6 | * here by name, which is what a heap that climbs with no request running needs. A `during` nested | |
| 7 | * in another counts the same growth under both names, so the totals are not additive. The minute | |
| 8 | * memory line reads and empties it (`allocationLine`). One entry per job name, at most | |
| 9 | * `MAX_NAMES`; gone on restart. | |
| 10 | */ | |
| 11 | ||
| 12 | import { heapSize } from "bun:jsc"; | |
| 13 | ||
| 14 | const MAX_NAMES = 128; | |
| 15 | ||
| 16 | export interface Tally { | |
| 17 | /** Run `fn`, counting the heap it grew by against `name`. */ | |
| 18 | counted<T>(name: string, fn: () => T): T; | |
| 19 | /** The jobs that grew the heap since the last call, most first, and forget them. */ | |
| 20 | take(top?: number): Allocation[]; | |
| 21 | } | |
| 22 | ||
| 23 | export type Allocation = { name: string; bytes: number; runs: number }; | |
| 24 | ||
| 25 | /** One tally per job watch, so a forge's jobs are counted on that forge and not on the process. */ | |
| 26 | export function createTally(size: () => number = heapSize): Tally { | |
| 27 | const seen = new Map<string, { bytes: number; runs: number }>(); | |
| 28 | return { | |
| 29 | counted(name, fn) { | |
| 30 | const before = size(); | |
| 31 | try { | |
| 32 | return fn(); | |
| 33 | } finally { | |
| 34 | const grew = size() - before; | |
| 35 | const at = seen.get(name); | |
| 36 | if (grew > 0 && (at !== undefined || seen.size < MAX_NAMES)) { | |
| 37 | if (at === undefined) seen.set(name, { bytes: grew, runs: 1 }); | |
| 38 | else { | |
| 39 | at.bytes += grew; | |
| 40 | at.runs++; | |
| 41 | } | |
| 42 | } | |
| 43 | } | |
| 44 | }, | |
| 45 | take(top = 5) { | |
| 46 | const out = [...seen] | |
| 47 | .map(([name, n]) => ({ name, ...n })) | |
| 48 | .sort((a, b) => b.bytes - a.bytes) | |
| 49 | .slice(0, top); | |
| 50 | seen.clear(); | |
| 51 | return out; | |
| 52 | }, | |
| 53 | }; | |
| 54 | } | |
| 55 | ||
| 56 | /** | |
| 57 | * `memory allocated by jobs: timers 84.0 MiB in 60 slices, maintain 2.1 MiB in 1 slices`; undefined | |
| 58 | * when nothing grew by 1 MiB. | |
outdated
The comment above shows a different format from this line ( |
||
| 59 | */ | |
| 60 | export function allocationLine(taken: readonly Allocation[]): string | undefined { | |
| 61 | const big = taken.filter((t) => t.bytes >= 1_048_576); | |
| 62 | if (big.length === 0) return undefined; | |
| 63 | return `memory allocated by jobs: ${big.map((t) => `${t.name} ${(t.bytes / 1_048_576).toFixed(1)} MiB in ${t.runs} slices`).join(", ")}`; | |
| 64 | } |
src/perf/jobs.ts +22 −1
| @@ -13,6 +13,7 @@ | ||
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | 15 | import { hostOf, noteChildJob } from "./child_time.ts"; |
| 16 | import { createTally, type Allocation } from "./job_alloc.ts"; | |
| 16 | 17 | |
| 17 | 18 | /** Finished slices kept to match a stall against: a tick a second fills it in about two minutes. */ |
| 18 | 19 | const KEPT = 256; |
| @@ -46,9 +47,14 @@ export interface JobWatch { | ||
| 46 | 47 | * a suspect for a stall nothing measured, never its proven cause (`loop.ts`). |
| 47 | 48 | */ |
| 48 | 49 | awaited(from: number, to: number): string[]; |
| 50 | /** | |
| 51 | * What the jobs allocated since the last call, most first; emptied by reading (`job_alloc.ts`). | |
| 52 | */ | |
| 53 | allocated(): Allocation[]; | |
| 49 | 54 | } |
| 50 | 55 | |
| 51 | 56 | export function createJobWatch(now: () => number = () => performance.now()): JobWatch { |
| 57 | const tally = createTally(); | |
| 52 | 58 | const awaiting = new Map<number, { name: string; start: number }>(); |
| 53 | 59 | const ring: (JobSpan | undefined)[] = new Array(KEPT); |
| 54 | 60 | let next = 0; |
| @@ -67,11 +73,12 @@ export function createJobWatch(now: () => number = () => performance.now()): Job | ||
| 67 | 73 | ...new Set(spans.sort((a, b) => b.end - b.start - (a.end - a.start)).map((s) => s.name)), |
| 68 | 74 | ]; |
| 69 | 75 | return { |
| 76 | allocated: () => tally.take(), | |
| 70 | 77 | during(name, fn) { |
| 71 | 78 | const start = now(); |
| 72 | 79 | let result: ReturnType<typeof fn>; |
| 73 | 80 | try { |
| 74 | result = fn(); | |
| 81 | result = tally.counted(name, fn); | |
| 75 | 82 | } finally { |
| 76 | 83 | keep({ name, start, end: now(), awaiting: false }); |
| 77 | 84 | } |
| @@ -178,3 +185,17 @@ export const during = <T>(name: string, fn: () => T, owner?: string, app?: objec | ||
| 178 | 185 | if (forge !== undefined) noteChildJob(forge, owner, performance.now() - start); |
| 179 | 186 | } |
| 180 | 187 | }; |
| 188 | ||
| 189 | /** What the jobs of the thread and of this forge allocated since the last call, most first. */ | |
| 190 | export function allocationsOf(app: object): Allocation[] { | |
| 191 | const merged = new Map<string, Allocation>(); | |
| 192 | for (const a of [...unowned.allocated(), ...jobWatchOf(app).allocated()]) { | |
| 193 | const at = merged.get(a.name); | |
| 194 | if (at === undefined) merged.set(a.name, { ...a }); | |
| 195 | else { | |
| 196 | at.bytes += a.bytes; | |
| 197 | at.runs += a.runs; | |
| 198 | } | |
| 199 | } | |
| 200 | return [...merged.values()].sort((a, b) => b.bytes - a.bytes).slice(0, 5); | |
| 201 | } |
test/job-alloc.test.ts added +114
| @@ -0,0 +1,114 @@ | ||
| 1 | import { heapSize } from "bun:jsc"; | |
| 2 | import { afterAll, afterEach, beforeAll, expect, spyOn, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { startMemorySampler } from "../src/boards/memory.ts"; | |
| 8 | import { allocationLine, createTally } from "../src/perf/job_alloc.ts"; | |
| 9 | import { allocationsOf, during } from "../src/perf/jobs.ts"; | |
| 10 | ||
| 11 | const MIB = 1024 * 1024; | |
| 12 | const { counted, take: takeAllocations } = createTally(); | |
| 13 | const keep: unknown[] = []; | |
| 14 | /** Grow the JS heap (typed arrays live off it) and keep it, so no collection undoes it. */ | |
| 15 | const grow = (n = 100_000): number => keep.push(Array.from({ length: n }, (_, i) => ({ i, s: `v${i}` }))); | |
| 16 | /** Run `fn` as `name` until the heap was seen to grow across it: a collection can hide one try. */ | |
| 17 | function grown(c: (name: string, fn: () => unknown) => unknown, name: string, fn: () => unknown = grow): void { | |
| 18 | for (let i = 0; i < 40; i++) { | |
| 19 | const before = heapSize(); | |
| 20 | try { | |
| 21 | c(name, fn); | |
| 22 | } catch { | |
outdated
Assert that |
||
| 23 | // the caller's own assertion covers a throw | |
| 24 | } | |
| 25 | if (heapSize() - before > 0) return; | |
| 26 | } | |
| 27 | throw new Error(`the heap never grew across ${name}`); | |
| 28 | } | |
| 29 | ||
| 30 | test("a job that keeps what it allocates is named with its size, and the count is emptied by reading it", () => { | |
| 31 | takeAllocations(); | |
| 32 | for (let i = 0; i < 3; i++) grown(counted, "hungry"); | |
| 33 | counted("quiet", () => 1); | |
| 34 | const taken = takeAllocations(); | |
| 35 | expect(taken[0]?.name).toBe("hungry"); | |
| 36 | expect(taken[0]!.bytes).toBeGreaterThan(0); | |
| 37 | expect(taken[0]!.runs).toBeGreaterThanOrEqual(3); | |
| 38 | expect(allocationLine(taken)).toMatch(/^memory allocated by jobs: hungry \d+\.\d MiB in \d+ slices/); | |
| 39 | expect(takeAllocations()).toEqual([]); | |
| 40 | }); | |
| 41 | ||
| 42 | test("a job that grew nothing leaves no line", () => { | |
| 43 | expect(allocationLine([{ name: "small", bytes: 1000, runs: 1 }])).toBeUndefined(); | |
| 44 | }); | |
| 45 | ||
| 46 | /** A heap that reads `step` bytes higher each time, so every slice grew by exactly `step`. */ | |
| 47 | function steppedHeap(step: number): () => number { | |
| 48 | let heap = 0; | |
| 49 | return () => (heap += step); | |
| 50 | } | |
| 51 | ||
| 52 | test("a throw is still counted and rethrown", () => { | |
| 53 | const tally = createTally(steppedHeap(2 * MIB)); | |
| 54 | expect(() => tally.counted("boom", () => { throw new Error("x"); })).toThrow("x"); | |
| 55 | expect(tally.take()).toEqual([{ name: "boom", bytes: 2 * MIB, runs: 1 }]); | |
| 56 | }); | |
| 57 | ||
| 58 | test("at most 128 names are kept, and a name already kept still counts", () => { | |
| 59 | const tally = createTally(steppedHeap(2 * MIB)); | |
| 60 | for (let i = 0; i < 130; i++) tally.counted(`job-${i}`, () => 1); | |
| 61 | tally.counted("job-0", () => 1); | |
| 62 | tally.counted("job-129", () => 1); | |
| 63 | const taken = tally.take(1000); | |
| 64 | expect(taken.length).toBe(128); | |
| 65 | expect(taken.find((t) => t.name === "job-129")).toBeUndefined(); | |
| 66 | expect(taken.find((t) => t.name === "job-0")?.runs).toBe(2); | |
| 67 | }); | |
| 68 | ||
| 69 | let dataDir: string; | |
| 70 | let app: App; | |
| 71 | beforeAll(async () => { | |
| 72 | dataDir = await mkdtemp(join(tmpdir(), "forge-alloc-")); | |
| 73 | app = createApp({ dataDir, port: 0 }); | |
| 74 | }); | |
| 75 | afterAll(async () => { | |
| 76 | app.close(); | |
| 77 | await rm(dataDir, { recursive: true, force: true }); | |
| 78 | }); | |
| 79 | ||
| 80 | test("allocationsOf merges the unowned and this forge's jobs, most first, five at most", () => { | |
| 81 | allocationsOf(app); | |
| 82 | const duringGrown = (name: string, owner?: object) => grown((n, fn) => during(n, fn as () => number, undefined, owner), name); | |
| 83 | for (let i = 0; i < 5; i++) duringGrown(`own-${i}`, app); | |
| 84 | for (let i = 0; i < 2; i++) duringGrown("shared"); | |
| 85 | for (let i = 0; i < 2; i++) duringGrown("shared", app); | |
| 86 | const got = allocationsOf(app); | |
| 87 | expect(got.length).toBe(5); | |
| 88 | expect(got[0]!.name).toBe("shared"); | |
| 89 | expect(got[0]!.runs).toBeGreaterThanOrEqual(2); | |
| 90 | expect(got.map((g) => g.bytes)).toEqual([...got.map((g) => g.bytes)].sort((a, b) => b - a)); | |
| 91 | expect(allocationsOf(app)).toEqual([]); | |
| 92 | }); | |
| 93 | ||
| 94 | test("the memory sampler logs what the jobs allocated, by name", () => { | |
| 95 | allocationsOf(app); | |
| 96 | const info = spyOn(app.log, "info"); | |
| 97 | const sampler = startMemorySampler(app, 3_600_000); | |
| 98 | try { | |
| 99 | allocationsOf(app); | |
| 100 | info.mockClear(); | |
| 101 | grown((n, fn) => during(n, fn as () => number, undefined, app), "hoarder"); | |
| 102 | sampler.tick(); | |
| 103 | const lines = info.mock.calls.map((c) => String(c[0])); | |
| 104 | expect(lines.some((l) => /^memory allocated by jobs: hoarder \d+\.\d MiB in \d+ slices/.test(l))).toBe(true); | |
| 105 | } finally { | |
| 106 | sampler.stop(); | |
| 107 | info.mockRestore(); | |
| 108 | } | |
| 109 | }); | |
| 110 | ||
| 111 | afterEach(() => { | |
| 112 | keep.length = 0; | |
| 113 | Bun.gc(true); | |
| 114 | }); |
new above
offline
Max minus min over a sawtooth heap is not growth per minute. Use the slope, or the rise of the post-GC floor.