Public: anyone can read this; the crew can change it.

The page thread stops for half a second about five times a minute with no request running: its heap grows 5 MB a second and is collected in long pauses #13925

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13924-the-page-thread-stops-for-half-a-second → main 8 commits fork 3 threads · 3 unresolved

  • Merged. mason/13924-the-page-thread-stops-for-half-a-second was merged into main as 201e62e.

Review

Sign in to review this change.
6 files changed +287 −2
AGENTS.md +1
@@ -54,3 +54,4 @@ The check queue hands work to a runner through `src/checks/runner.ts` (#13652, R
5454 An app token's requests go behind people's and repeat reads are 304s (`src/http/app_turn.ts`, `repeat_reads.ts`, `perf/asking.ts`): see `docs/features/app-asking.md`.
5555 `src/core/reports.ts`: `closedSet`, the ids of repositories whose owner closed reports from outside (#12321), read once per database and kept by `setReportsClosed`, the only writer, so every repository page's head link costs no statement; at most one id per closed repository, gone on restart (the rows are `report_closed`). `src/core/contribute_token.ts`: `openSet`, the ids of repositories that allow outside agents, read once per database and kept by `setContributeOpen`, the only writer, at most one id per allowing repository, gone on restart; a raw SQL write to `contribute_open` bypasses it. The terminal door for reports is `r` on the outside agents screen (`setOutsideReports`, admins).
5656 `src/crew/beat.ts`: `lastBeat`, when each repository's beat last ran, and `declared`, the `beat` last read from each default branch with the commit it came from (#13709); at most one entry each per repository, gone on restart (a restart beats once at once; the declaration is a file in git).
57 `src/perf/job_alloc.ts`: `counted`, called by `during` around each job's synchronous slice, adds the JS heap's growth to the job's name (at most 128 names); the minute memory line reads and empties it (`memory allocated by jobs:`, #13924), so a timer that allocates with no request running is named. `bench/idle.ts` is the idle budget on a seeded forge (heap under 20 MiB a minute, no stall over 100 ms). Gone on restart.
bench/idle.ts added +82
@@ -0,0 +1,82 @@
1 /**
2 * The idle budget (#13924): boot the forge over a seeded data directory, ask it nothing, and
3 * hold its serving thread to a flat heap and no stall over 100 ms.
4 *
5 * bun run bench/seed.ts --data .bench/data
6 * bun run bench/idle.ts --data .bench/data [--secs 120]
7 *
8 * It reads the forge's own log: each `memory main:` line's heap, each `event loop blocked` line
9 * after the boot's first 20 s. Heap growth is the least-squares slope of the heap readings
10 * against the time they were logged, per minute, so a collection's sawtooth does not read as
11 * growth or hide it; it needs a run long enough for several readings (a minute apart), and with
12 * fewer than three it says so and reports none. `memory allocated by jobs:` lines name the timer
13 * that did it. Exit 1 over budget. Nothing runs it on a push: it is run by hand, and the budget
14 * is enforced only by whoever runs it.
15 */
16
17 import { spawn } from "bun";
18
19 const MAX_GROWTH_MIB_PER_MIN = 20;
20 const MAX_STALL_MS = 100;
21 const BOOT_MS = 20_000;
22
23 const arg = (name: string, fallback: string): string => {
24 const at = process.argv.indexOf(`--${name}`);
25 return at > 0 && process.argv[at + 1] !== undefined ? process.argv[at + 1]! : fallback;
26 };
27
28 const data = arg("data", ".bench/data");
29 const secs = Number(arg("secs", "120"));
30 const proc = spawn(["bun", "run", "src/main.ts"], {
31 env: { ...process.env, FORGE_APP_DATA: data, FORGE_DATA: data, FORGE_ENGINE: "worker", PORT: arg("port", "3917") },
32 stdout: "pipe",
33 stderr: "pipe",
34 });
35 const lines: string[] = [];
36 const started = Date.now();
37 const collect = async (stream: ReadableStream<Uint8Array>): Promise<void> => {
38 const decoder = new TextDecoder();
39 let rest = "";
40 for await (const chunk of stream) {
41 rest += decoder.decode(chunk);
42 const parts = rest.split("\n");
43 rest = parts.pop() ?? "";
44 for (const line of parts) lines.push(`${Date.now() - started}\t${line}`);
45 }
46 };
47 void collect(proc.stdout);
48 void collect(proc.stderr);
49 await new Promise((r) => setTimeout(r, secs * 1000));
50 proc.kill();
51
52 const afterTimed = lines
53 .map((l) => ({ at: Number(l.split("\t", 1)[0]), text: l.slice(l.indexOf("\t") + 1) }))
54 .filter((l) => l.at > BOOT_MS);
55 const after = afterTimed.map((l) => l.text);
outdated

Max minus min over a sawtooth heap is not growth per minute. Use the slope, or the rise of the post-GC floor.

56 const mib = (text: string, re: RegExp): number | undefined => {
57 const m = re.exec(text);
58 return m === null ? undefined : Number(m[1]);
59 };
60 const readings = afterTimed.flatMap((l) => {
61 const heap = l.text.includes("memory main:") ? mib(l.text, /heap ([\d.]+) MiB/) : undefined;
62 return heap === undefined ? [] : [{ minutes: l.at / 60_000, heap }];
63 });
64 const heaps = readings.map((r) => r.heap);
65 const stalls = after.flatMap((l) => (l.includes("event loop blocked") ? [mib(l, /blocked (\d+) ms/)] : [])).filter((n): n is number => n !== undefined);
66 /** Least-squares slope of heap (MiB) over time (minutes); undefined with under three readings. */
67 function slope(points: readonly { minutes: number; heap: number }[]): number | undefined {
68 if (points.length < 3) return undefined;
69 const n = points.length;
70 const mx = points.reduce((t, p) => t + p.minutes, 0) / n;
71 const my = points.reduce((t, p) => t + p.heap, 0) / n;
72 const sxx = points.reduce((t, p) => t + (p.minutes - mx) ** 2, 0);
73 if (sxx === 0) return undefined;
74 return points.reduce((t, p) => t + (p.minutes - mx) * (p.heap - my), 0) / sxx;
75 }
76 const growth = slope(readings);
77 const allocated = after.filter((l) => l.includes("memory allocated by jobs:"));
78 console.log(`idle ${secs} s: heap readings ${heaps.join(", ") || "none"} MiB; growth ${growth === undefined ? "unmeasured (run longer)" : `${growth.toFixed(1)} MiB/min (slope)`}; stalls ${stalls.join(", ") || "none"} ms`);
79 for (const l of allocated) console.log(l);
80 const bad = (growth ?? 0) > MAX_GROWTH_MIB_PER_MIN || stalls.some((ms) => ms > MAX_STALL_MS);
81 if (bad) console.log(`over budget: ${MAX_GROWTH_MIB_PER_MIN} MiB/min and ${MAX_STALL_MS} ms`);
82 process.exit(bad ? 1 : 0);
src/boards/memory.ts +4 −1
@@ -16,8 +16,9 @@
1616 import type { Database } from "bun:sqlite";
1717 import type { App } from "../app.ts";
1818 import { quietly } from "../core/write_clock.ts";
19 import { during } from "../perf/jobs.ts";
19 import { allocationsOf, during } from "../perf/jobs.ts";
2020 import { heldLine, heldNow } from "../perf/held.ts";
21 import { allocationLine } from "../perf/job_alloc.ts";
2122 import { hearWorkers, memoryLine, memoryWatch, readNow } from "../perf/memory_watch.ts";
2223
2324 export const MEMORY_EVERY_MS = 60_000;
@@ -78,6 +79,8 @@ export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemoryS
7879 memoryWatch.note(reading.rss, reading.heap, now, reading.current);
7980 app.log.info(memoryLine(reading, process.uptime() * 1000));
8081 app.log.info(heldLine(heldNow(app)));
82 const allocated = allocationLine(allocationsOf(app));
83 if (allocated !== undefined) app.log.info(allocated);
8184 } catch (error) {
8285 app.log.warn("memory sample failed", { error: String(error) });
8386 }
src/perf/job_alloc.ts added +64
@@ -0,0 +1,64 @@
1 /**
2 * What each background job allocated (#13924): `during` (`jobs.ts`) reads the JS heap's size
3 * (`heapSize()`, constant time) on either side of a job's synchronous slice and adds any growth
4 * to the job's name. A collection inside a slice shows as no growth, so this is a floor, and
5 * what a job allocates after an await is not seen; but a timer that allocates on every tick shows
6 * here by name, which is what a heap that climbs with no request running needs. A `during` nested
7 * in another counts the same growth under both names, so the totals are not additive. The minute
8 * memory line reads and empties it (`allocationLine`). One entry per job name, at most
9 * `MAX_NAMES`; gone on restart.
10 */
11
12 import { heapSize } from "bun:jsc";
13
14 const MAX_NAMES = 128;
15
16 export interface Tally {
17 /** Run `fn`, counting the heap it grew by against `name`. */
18 counted<T>(name: string, fn: () => T): T;
19 /** The jobs that grew the heap since the last call, most first, and forget them. */
20 take(top?: number): Allocation[];
21 }
22
23 export type Allocation = { name: string; bytes: number; runs: number };
24
25 /** One tally per job watch, so a forge's jobs are counted on that forge and not on the process. */
26 export function createTally(size: () => number = heapSize): Tally {
27 const seen = new Map<string, { bytes: number; runs: number }>();
28 return {
29 counted(name, fn) {
30 const before = size();
31 try {
32 return fn();
33 } finally {
34 const grew = size() - before;
35 const at = seen.get(name);
36 if (grew > 0 && (at !== undefined || seen.size < MAX_NAMES)) {
37 if (at === undefined) seen.set(name, { bytes: grew, runs: 1 });
38 else {
39 at.bytes += grew;
40 at.runs++;
41 }
42 }
43 }
44 },
45 take(top = 5) {
46 const out = [...seen]
47 .map(([name, n]) => ({ name, ...n }))
48 .sort((a, b) => b.bytes - a.bytes)
49 .slice(0, top);
50 seen.clear();
51 return out;
52 },
53 };
54 }
55
56 /**
57 * `memory allocated by jobs: timers 84.0 MiB in 60 slices, maintain 2.1 MiB in 1 slices`; undefined
58 * when nothing grew by 1 MiB.
outdated

The comment above shows a different format from this line (memory allocated: and (60 runs)). Make the example match the real output.

59 */
60 export function allocationLine(taken: readonly Allocation[]): string | undefined {
61 const big = taken.filter((t) => t.bytes >= 1_048_576);
62 if (big.length === 0) return undefined;
63 return `memory allocated by jobs: ${big.map((t) => `${t.name} ${(t.bytes / 1_048_576).toFixed(1)} MiB in ${t.runs} slices`).join(", ")}`;
64 }
src/perf/jobs.ts +22 −1
@@ -13,6 +13,7 @@
1313 */
1414
1515 import { hostOf, noteChildJob } from "./child_time.ts";
16 import { createTally, type Allocation } from "./job_alloc.ts";
1617
1718 /** Finished slices kept to match a stall against: a tick a second fills it in about two minutes. */
1819 const KEPT = 256;
@@ -46,9 +47,14 @@ export interface JobWatch {
4647 * a suspect for a stall nothing measured, never its proven cause (`loop.ts`).
4748 */
4849 awaited(from: number, to: number): string[];
50 /**
51 * What the jobs allocated since the last call, most first; emptied by reading (`job_alloc.ts`).
52 */
53 allocated(): Allocation[];
4954 }
5055
5156 export function createJobWatch(now: () => number = () => performance.now()): JobWatch {
57 const tally = createTally();
5258 const awaiting = new Map<number, { name: string; start: number }>();
5359 const ring: (JobSpan | undefined)[] = new Array(KEPT);
5460 let next = 0;
@@ -67,11 +73,12 @@ export function createJobWatch(now: () => number = () => performance.now()): Job
6773 ...new Set(spans.sort((a, b) => b.end - b.start - (a.end - a.start)).map((s) => s.name)),
6874 ];
6975 return {
76 allocated: () => tally.take(),
7077 during(name, fn) {
7178 const start = now();
7279 let result: ReturnType<typeof fn>;
7380 try {
74 result = fn();
81 result = tally.counted(name, fn);
7582 } finally {
7683 keep({ name, start, end: now(), awaiting: false });
7784 }
@@ -178,3 +185,17 @@ export const during = <T>(name: string, fn: () => T, owner?: string, app?: objec
178185 if (forge !== undefined) noteChildJob(forge, owner, performance.now() - start);
179186 }
180187 };
188
189 /** What the jobs of the thread and of this forge allocated since the last call, most first. */
190 export function allocationsOf(app: object): Allocation[] {
191 const merged = new Map<string, Allocation>();
192 for (const a of [...unowned.allocated(), ...jobWatchOf(app).allocated()]) {
193 const at = merged.get(a.name);
194 if (at === undefined) merged.set(a.name, { ...a });
195 else {
196 at.bytes += a.bytes;
197 at.runs += a.runs;
198 }
199 }
200 return [...merged.values()].sort((a, b) => b.bytes - a.bytes).slice(0, 5);
201 }
test/job-alloc.test.ts added +114
@@ -0,0 +1,114 @@
1 import { heapSize } from "bun:jsc";
2 import { afterAll, afterEach, beforeAll, expect, spyOn, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { startMemorySampler } from "../src/boards/memory.ts";
8 import { allocationLine, createTally } from "../src/perf/job_alloc.ts";
9 import { allocationsOf, during } from "../src/perf/jobs.ts";
10
11 const MIB = 1024 * 1024;
12 const { counted, take: takeAllocations } = createTally();
13 const keep: unknown[] = [];
14 /** Grow the JS heap (typed arrays live off it) and keep it, so no collection undoes it. */
15 const grow = (n = 100_000): number => keep.push(Array.from({ length: n }, (_, i) => ({ i, s: `v${i}` })));
16 /** Run `fn` as `name` until the heap was seen to grow across it: a collection can hide one try. */
17 function grown(c: (name: string, fn: () => unknown) => unknown, name: string, fn: () => unknown = grow): void {
18 for (let i = 0; i < 40; i++) {
19 const before = heapSize();
20 try {
21 c(name, fn);
22 } catch {
outdated

Assert that boom shows up in takeAllocations() (it needs to have grown the heap), or rename the test. As written, the 'still counted' half is untested.

23 // the caller's own assertion covers a throw
24 }
25 if (heapSize() - before > 0) return;
26 }
27 throw new Error(`the heap never grew across ${name}`);
28 }
29
30 test("a job that keeps what it allocates is named with its size, and the count is emptied by reading it", () => {
31 takeAllocations();
32 for (let i = 0; i < 3; i++) grown(counted, "hungry");
33 counted("quiet", () => 1);
34 const taken = takeAllocations();
35 expect(taken[0]?.name).toBe("hungry");
36 expect(taken[0]!.bytes).toBeGreaterThan(0);
37 expect(taken[0]!.runs).toBeGreaterThanOrEqual(3);
38 expect(allocationLine(taken)).toMatch(/^memory allocated by jobs: hungry \d+\.\d MiB in \d+ slices/);
39 expect(takeAllocations()).toEqual([]);
40 });
41
42 test("a job that grew nothing leaves no line", () => {
43 expect(allocationLine([{ name: "small", bytes: 1000, runs: 1 }])).toBeUndefined();
44 });
45
46 /** A heap that reads `step` bytes higher each time, so every slice grew by exactly `step`. */
47 function steppedHeap(step: number): () => number {
48 let heap = 0;
49 return () => (heap += step);
50 }
51
52 test("a throw is still counted and rethrown", () => {
53 const tally = createTally(steppedHeap(2 * MIB));
54 expect(() => tally.counted("boom", () => { throw new Error("x"); })).toThrow("x");
55 expect(tally.take()).toEqual([{ name: "boom", bytes: 2 * MIB, runs: 1 }]);
56 });
57
58 test("at most 128 names are kept, and a name already kept still counts", () => {
59 const tally = createTally(steppedHeap(2 * MIB));
60 for (let i = 0; i < 130; i++) tally.counted(`job-${i}`, () => 1);
61 tally.counted("job-0", () => 1);
62 tally.counted("job-129", () => 1);
63 const taken = tally.take(1000);
64 expect(taken.length).toBe(128);
65 expect(taken.find((t) => t.name === "job-129")).toBeUndefined();
66 expect(taken.find((t) => t.name === "job-0")?.runs).toBe(2);
67 });
68
69 let dataDir: string;
70 let app: App;
71 beforeAll(async () => {
72 dataDir = await mkdtemp(join(tmpdir(), "forge-alloc-"));
73 app = createApp({ dataDir, port: 0 });
74 });
75 afterAll(async () => {
76 app.close();
77 await rm(dataDir, { recursive: true, force: true });
78 });
79
80 test("allocationsOf merges the unowned and this forge's jobs, most first, five at most", () => {
81 allocationsOf(app);
82 const duringGrown = (name: string, owner?: object) => grown((n, fn) => during(n, fn as () => number, undefined, owner), name);
83 for (let i = 0; i < 5; i++) duringGrown(`own-${i}`, app);
84 for (let i = 0; i < 2; i++) duringGrown("shared");
85 for (let i = 0; i < 2; i++) duringGrown("shared", app);
86 const got = allocationsOf(app);
87 expect(got.length).toBe(5);
88 expect(got[0]!.name).toBe("shared");
89 expect(got[0]!.runs).toBeGreaterThanOrEqual(2);
90 expect(got.map((g) => g.bytes)).toEqual([...got.map((g) => g.bytes)].sort((a, b) => b - a));
91 expect(allocationsOf(app)).toEqual([]);
92 });
93
94 test("the memory sampler logs what the jobs allocated, by name", () => {
95 allocationsOf(app);
96 const info = spyOn(app.log, "info");
97 const sampler = startMemorySampler(app, 3_600_000);
98 try {
99 allocationsOf(app);
100 info.mockClear();
101 grown((n, fn) => during(n, fn as () => number, undefined, app), "hoarder");
102 sampler.tick();
103 const lines = info.mock.calls.map((c) => String(c[0]));
104 expect(lines.some((l) => /^memory allocated by jobs: hoarder \d+\.\d MiB in \d+ slices/.test(l))).toBe(true);
105 } finally {
106 sampler.stop();
107 info.mockRestore();
108 }
109 });
110
111 afterEach(() => {
112 keep.length = 0;
113 Bun.gc(true);
114 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs