Public: anyone can read this; the crew can change it.

The pages worker's memory is bounded: a test renders 2,000 pages and RSS growth stays under a stated ceiling #14511

opened this

Check · Review · Land · Live — No review yet. Next: reviewer, no review yet.

Ready to land

mason/forge:mason/14479-the-pages-worker-s-memory-is-bounded-a-t → main 4 commits fork

Review

Sign in to review this change.
10 files changed +282 −5
AGENTS.md +1
@@ -65,6 +65,7 @@ An app token's requests go behind people's and repeat reads are 304s (`src/http/
6565 `src/spawn/births.ts`: `states`, per forge, the births in flight, the spawns waiting their turn (first in, first out, signed-in people ahead of strangers; at most 4 × `spawn.births_at_once` wait, strangers at most `spawn.births_at_once` of those, the rest are answered at once with their place; each waits 15 s at most, then is answered with its place) and the mean of the births measured (#13822); a birth is counted until its forge answers or 5 minutes pass; at most `spawn.births_at_once` in flight, gone on restart. The limits that bound a surge (`spawn.*`, `forges.wakes_per_forge_hour`) are settings on /admin/limits.
6666 `src/web/studio/waiting_on_me.ts`: `waitingRows`, each viewer's issues and pulls that wait for them, held 10 seconds per viewer (at most 500 viewers, gone on restart) so the board's live lanes share one `decisionsFor` read; the terminal door is `src/tui/screens/waiting_for_me.ts` (palette entry for any person).
6767 `src/crew/beat.ts`: `lastBeat`, when each repository's beat last ran, and `declared`, the `beat` last read from each default branch with the commit it came from (#13709); at most one entry each per repository, gone on restart (a restart beats once at once; the declaration is a file in git).
68 `src/kernel/list_texts.ts`: each connection's list texts (#14479), the statements whose `IN (?1, …, ?n)` list has three or more placeholders, so its length comes from the data: 64 a connection, least recently used out first, kept apart from Bun's 4,096-text cache (`kernel/db.ts`) that holds the fixed texts. `openDb` and the pages, lanes, pipeline and map workers' handles go through `keepFixedTexts`; a new long-lived handle does too. Counted as `list statements kept` on the `memory holds:` line; gone with the handle.
6869 `src/perf/job_alloc.ts`: `counted`, called by `during` around each job's synchronous slice, adds the JS heap's growth to the job's name (at most 128 names); the minute memory line reads and empties it (`memory allocated by jobs:`, #13924), so a timer that allocates with no request running is named. `bench/idle.ts` is the idle budget on a seeded forge (heap under 20 MiB a minute, no stall over 100 ms). Gone on restart.
6970
7071 A pull that changes code under `src/` adds or edits a `test/*.test.ts` too, or says `No test: <why>` in a commit line; `bun scripts/gate.ts` refuses it otherwise ("a behaviour a pull adds has a test that fails without it", #14235).
scripts/gate.ts +1
@@ -231,6 +231,7 @@ export const SLOW: readonly Exclusion[] = [
231231 { file: "test/pull-slices.test.ts", reason: "boots a server, clones and pushes through real git over HTTP, lands three slices; 1.7 s alone, 2.8 s a test under full-suite load", issue: 13223 },
232232 { file: "test/start.test.ts", reason: "pushes to a real main and waits for the deploy, then waits out a start-rate window; 5.4 s a test and 21 s a file under full-suite load", issue: 14211 },
233233 { file: "test/merge-queue-batch.test.ts", reason: "lands a 16-pull batch over real repositories in several rounds; 5.0 s a test under full-suite load", issue: 14211 },
234 { file: "test/pages-memory.test.ts", reason: "renders three pages and reads the pages worker 2,000 times each to bound heap growth (#14479); 17 s and 8 s a test under full-suite load", issue: 14479 },
234235 { file: "test/axe.test.ts", reason: "drives Chromium through axe over each colour scheme; 3.4 s a test and 21 s a file under full-suite load", issue: 14211 },
235236 ];
236237
src/flow/lanes_worker.ts +2 −1
@@ -10,6 +10,7 @@
1010 */
1111
1212 import { Database } from "bun:sqlite";
13 import { keepFixedTexts } from "../kernel/list_texts.ts";
1314 import { compose, readFast, readSlow, SLOW_MS, type LanePicture, type Slow } from "./lanes_read.ts";
1415 import { workerMemory } from "../perf/memory_watch.ts";
1516
@@ -48,7 +49,7 @@ self.onmessage = (e: MessageEvent<LaneJob>) => {
4849 try {
4950 let db = handles.get(path);
5051 if (db === undefined) {
51 db = new Database(path, { readonly: true });
52 db = keepFixedTexts(new Database(path, { readonly: true }));
5253 db.run("PRAGMA busy_timeout = 5000");
5354 handles.set(path, db);
5455 }
src/flow/pages_worker.ts +2 −1
@@ -7,6 +7,7 @@
77 */
88
99 import { Database } from "bun:sqlite";
10 import { keepFixedTexts } from "../kernel/list_texts.ts";
1011 import { readPagePicture, type PageJob, type PagesConfig, type PagesEnv, type Picture } from "./pages_read.ts";
1112 import { workerMemory } from "../perf/memory_watch.ts";
1213
@@ -39,7 +40,7 @@ self.onmessage = (e: MessageEvent<PagesAsk>) => {
3940 let env = envs.get(path);
4041 try {
4142 if (env === undefined) {
42 const db = new Database(path, { readonly: true });
43 const db = keepFixedTexts(new Database(path, { readonly: true }));
4344 db.run("PRAGMA busy_timeout = 5000");
4445 envs.set(path, (env = { db, reposRoot, config, ceilingUsd, inline: false }));
4546 }
src/flow/pipeline_worker.ts +2 −1
@@ -6,6 +6,7 @@
66 */
77
88 import { Database } from "bun:sqlite";
9 import { keepFixedTexts } from "../kernel/list_texts.ts";
910 import { workerMemory } from "../perf/memory_watch.ts";
1011
1112 /** What the worker is asked: the database file, the stage's query and its arguments. */
@@ -35,7 +36,7 @@ self.onmessage = (e: MessageEvent<WaitingJob>) => {
3536 try {
3637 let db = handles.get(path);
3738 if (db === undefined) {
38 db = new Database(path, { readonly: true });
39 db = keepFixedTexts(new Database(path, { readonly: true }));
3940 db.run("PRAGMA busy_timeout = 5000");
4041 handles.set(path, db);
4142 }
src/kernel/db.ts +6 −1
@@ -6,6 +6,7 @@
66 */
77
88 import { Database } from "bun:sqlite";
9 import { keepFixedTexts } from "./list_texts.ts";
910 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
1011 import { dirname, join } from "node:path";
1112 import { createLog } from "./log.ts";
@@ -59,7 +60,9 @@ const MMAP_BYTES = 512 * 1024 * 1024;
5960 * page asks about thirty, so a cached page still spent ~1.4 ms of its ~4 ms
6061 * preparing. The forge's queries are fixed texts (about 2,000 call sites), so
6162 * this holds every one of them; a text past the cap is prepared per call as
62 * before, never wrong. Bun reads it on each `query`, so set once here.
63 * before, never wrong. A text whose IN list grows with the data is not one of
64 * them: `list_texts.ts` keeps those apart, bounded (#14479). Bun reads it on each `query`, so set
65 * once here.
6366 */
6467 export const QUERY_CACHE = 4096;
6568 (Database as unknown as { MAX_QUERY_CACHE_SIZE: number }).MAX_QUERY_CACHE_SIZE = QUERY_CACHE;
@@ -75,6 +78,8 @@ export function openDb(path: string, migrations: readonly Migration[]): Database
7578 if (image !== undefined) writeFileSync(path, image, { flag: "wx", mode: 0o600 });
7679 db = new Database(path, { create: true, strict: true });
7780 }
81 // A text whose IN list grows with the data is kept apart, bounded (#14479).
82 keepFixedTexts(db);
7883 applyPragmas(db, memory ? CACHE_KIB : mainCacheKiB());
7984 // Only files this call made: one that already exists is tightened below if
8085 // too open (`file_modes.ts`; vitals names any it could not). The umask can widen
src/kernel/list_texts.ts added +76
@@ -0,0 +1,76 @@
1 /**
2 * A statement whose text grows with the data is never kept for the life of a connection (#14479).
3 *
4 * `kernel/db.ts` raises Bun's prepared-statement cache to 4,096 texts a connection (#9514),
5 * written for fixed texts. But about 170 call sites build `IN (?1, …, ?n)` with one placeholder
6 * per value, so each new length of a list is a new text: a forge's counts of open pulls, issues
7 * and runs drift all day, and every length met was kept, prepared, until the cache was full. A
8 * prepared statement's compiled program lives in malloc, outside the JS heap, so the collector
9 * never felt it: 4,096 texts under that cap held about 410 MiB of RSS on a bare table (20 and flat
10 * at Bun's default), and the pages worker's long-lived handle, reading the same lists every
11 * minute, grew most.
12 *
13 * `keepFixedTexts` hands a text with an `IN` list of three or more placeholders to a small cache
14 * of its own instead (`LIST_TEXTS_KEPT` a connection, least recently used out first): a list whose
15 * length repeats is still prepared once, and one pushed out is garbage the collector frees. A text
16 * with no such list keeps Bun's cache, as before. A list of one or two placeholders stays there
17 * too: at most two texts a call site, however the data moves.
18 */
19
20 import type { Database, Statement } from "bun:sqlite";
21
22 /** The list texts one connection keeps prepared. */
23 export const LIST_TEXTS_KEPT = 64;
24
25 /** `IN (?, ?, ?…)` or `IN (?1, ?2, ?3…)`: three or more placeholders and nothing else. */
26 const LIST = /\bIN\s*\(\s*\?\d*(?:\s*,\s*\?\d*){2,}\s*\)/i;
27
28 /** Whether `sql` holds an `IN` list long enough to have been built from values. */
29 export const isListText = (sql: string): boolean => LIST.test(sql);
30
31 /** The list texts kept on this thread, across its connections. */
32 let keptHere = 0;
33
34 /**
35 * How many list texts this thread keeps prepared: `perf/held.ts` names it on the `memory holds:`
36 * line.
37 */
38 export const listTextsKeptHere = (): number => keptHere;
39
40 /** Each wrapped connection's list texts, oldest first. */
41 const listsOf = new WeakMap<Database, Map<string, Statement>>();
42
43 /** Route `db.query` so a list text is kept in a bounded cache of its own; returns `db`. */
44 export function keepFixedTexts(db: Database): Database {
45 if (listsOf.has(db)) return db;
46 const handle = db as unknown as { query: (sql: string) => Statement; close: (...a: unknown[]) => void };
47 const cached = handle.query.bind(db);
48 const close = handle.close.bind(db);
49 // Bound now, so a later wrapper of `db.prepare` (a statement counter) sees one call, not two.
50 const prepare = db.prepare.bind(db);
51 const lists = new Map<string, Statement>();
52 listsOf.set(db, lists);
53 handle.query = (sql: string): Statement => {
54 if (typeof sql !== "string" || !isListText(sql)) return cached(sql);
55 const known = lists.get(sql);
56 if (known !== undefined) {
57 lists.delete(sql);
58 lists.set(sql, known);
59 return known;
60 }
61 const made = prepare(sql);
62 lists.set(sql, made);
63 keptHere++;
64 if (lists.size > LIST_TEXTS_KEPT) {
65 lists.delete(lists.keys().next().value!);
66 keptHere--;
67 }
68 return made;
69 };
70 handle.close = (...a: unknown[]): void => {
71 keptHere -= lists.size;
72 lists.clear();
73 close(...a);
74 };
75 return db;
76 }
src/perf/held.ts +6
@@ -12,6 +12,8 @@
1212 * getter, so the registry is as large as the names in the source, never as the work done.
1313 */
1414
15 import { listTextsKeptHere } from "../kernel/list_texts.ts";
16
1517 export interface HeldCount {
1618 readonly name: string;
1719 /** Entries now; -1 when the getter threw. */
@@ -63,6 +65,10 @@ export function heldOf(app: object): Held {
6365 */
6466 export const threadHeld: Held = createHeld();
6567
68 // The kernel names nothing outside itself (test/kernel-boundary.test.ts), so its count is named
69 // here.
70 threadHeld.add("list statements kept", listTextsKeptHere);
71
6672 /** A forge's own holders and the thread's, as one reading: largest first. */
6773 export function heldNow(app: object): HeldCount[] {
6874 return [...heldOf(app).read(), ...threadHeld.read()].sort((a, b) => b.size - a.size || (a.name < b.name ? -1 : 1));
src/web/map_worker.ts +2 −1
@@ -9,6 +9,7 @@
99 // The first import is the order two modules that read each other need (`breaker` and `guard`): this worker starts from the map, not from `main`.
1010 import "../crew/breaker.ts";
1111 import { Database } from "bun:sqlite";
12 import { keepFixedTexts } from "../kernel/list_texts.ts";
1213 import type { App } from "../app.ts";
1314 import type { PagesConfig } from "../flow/pages_read.ts";
1415 import { readMap, type Learned, type MapJob, type MapReading } from "./map_read.ts";
@@ -44,7 +45,7 @@ self.onmessage = (e: MessageEvent<MapAsk>) => {
4445 try {
4546 let app = apps.get(path);
4647 if (app === undefined) {
47 const db = new Database(path, { readonly: true });
48 const db = keepFixedTexts(new Database(path, { readonly: true }));
4849 db.run("PRAGMA busy_timeout = 5000");
4950 apps.set(path, (app = { db, reposRoot, config } as unknown as App));
5051 }
test/pages-memory.test.ts added +184
@@ -0,0 +1,184 @@
1 /**
2 * The pages worker's memory is bounded (#14479). What it kept: every length of an `IN (?1, …, ?n)`
3 * list was a new SQL text, and each connection kept 4,096 of them prepared (`kernel/db.ts`), their
4 * compiled programs in malloc where the collector never looked. `kernel/list_texts.ts` keeps those
5 * texts apart, a few dozen a connection.
6 *
7 * The readings run here on the calling thread with `inline: false`: the worker's own code
8 * (`readPagePicture` over a read-only handle opened as `pages_worker.ts` opens it), the parts it
9 * keeps included, with none of a worker's noise in the figures. Heap is read after `Bun.gc(true)`,
10 * never RSS: Bun's RSS moves by tens of MiB on its own.
11 */
12 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
13 import { Database } from "bun:sqlite";
14 import { heapStats } from "bun:jsc";
15 import { mkdtemp, rm } from "node:fs/promises";
16 import { tmpdir } from "node:os";
17 import { join } from "node:path";
18 import { snappyFixture, type SnappyFixture } from "../bench/snappy.ts";
19 import { createApp, type App } from "../src/app.ts";
20 import { createIssue } from "../src/core/issues.ts";
21 import { getRepo, type Repo } from "../src/core/repos.ts";
22 import { listReviewsFor } from "../src/core/reviews.ts";
23 import { learnedPulls } from "../src/crew/hot_files.ts";
24 import { pagesConfig, readPagePicture, type PageJob, type PagesEnv } from "../src/flow/pages_read.ts";
25 import { isListText, keepFixedTexts, LIST_TEXTS_KEPT, listTextsKeptHere } from "../src/kernel/list_texts.ts";
26 import { seedHomeFlow, type HomeFlowSeed } from "./helpers/home-flow-seed.ts";
27
28 const MIB = 1024 * 1024;
29 /**
30 * The heap 2,000 renders of the three pages may grow by: the pages remember a few dozen regions, no
31 * more.
32 */
33 const RENDER_CEILING_MIB = 24;
34 /**
35 * The heap 2,000 of the worker's readings may grow by while the forge gains 500 issues and pulls.
36 */
37 const READING_CEILING_MIB = 24;
38 /**
39 * Prepared statements alive past what was alive before: the list texts kept, and the fixed ones met
40 * once.
41 */
42 const STATEMENTS_CEILING = 2 * LIST_TEXTS_KEPT + 200;
43
44 const statements = (): number => heapStats().objectTypeCounts["SQLStatement"] ?? 0;
45 function settled(): { heap: number; statements: number } {
46 Bun.gc(true);
47 return { heap: process.memoryUsage().heapUsed, statements: statements() };
48 }
49
50 describe("a list text is kept apart, bounded", () => {
51 test("only an IN list of three or more placeholders is a list text", () => {
52 expect(isListText("SELECT * FROM pull WHERE issue_id IN (?1,?2,?3)")).toBe(true);
53 expect(isListText("SELECT 1 FROM t WHERE a NOT IN (?, ?, ?, ?)")).toBe(true);
54 expect(isListText("SELECT 1 FROM t WHERE a in ( ?4 , ?5 , ?6 )")).toBe(true);
55 expect(isListText("SELECT 1 FROM t WHERE a IN (?1, ?2) AND b = ?3")).toBe(false);
56 expect(isListText("SELECT 1 FROM t WHERE state IN ('open', 'closed', 'merged')")).toBe(false);
57 expect(isListText("SELECT 1 FROM t WHERE a = ?1")).toBe(false);
58 });
59
60 test("400 lengths of one list keep a few dozen statements, and the same text answers the same one", async () => {
61 const dataDir = await mkdtemp(join(tmpdir(), "forge-list-texts-"));
62 const app = createApp({ dataDir, port: 0 });
63 const db = keepFixedTexts(new Database(app.db.filename, { readonly: true }));
64 const keptBefore = listTextsKeptHere();
65 try {
66 const before = settled();
67 for (let round = 0; round < 2; round++)
68 for (let n = 1; n <= 400; n++) listReviewsFor(db, Array.from({ length: n }, (_, i) => `id${i}`));
69 const after = settled();
70 expect(listTextsKeptHere() - keptBefore).toBe(LIST_TEXTS_KEPT);
71 // Before #14479 every one of the 400 texts was still prepared here, twice over on a rerun.
72 expect(after.statements - before.statements).toBeLessThan(LIST_TEXTS_KEPT + 20);
73 const text = "SELECT id FROM issue WHERE id IN (?1,?2,?3)";
74 expect(db.query(text)).toBe(db.query(text));
75 const fixed = "SELECT id FROM issue WHERE id = ?1";
76 expect(db.query(fixed)).toBe(db.query(fixed));
77 expect(db.query<{ n: number }, []>("SELECT 7 AS n WHERE 7 IN (5,6,7)").get()?.n).toBe(7);
78 } finally {
79 db.close();
80 // A closed connection's list texts leave the thread's count.
81 expect(listTextsKeptHere()).toBe(keptBefore);
82 app.close();
83 await rm(dataDir, { recursive: true, force: true });
84 }
85 }, 60_000);
86
87 test("every connection the forge opens is kept so", async () => {
88 const dataDir = await mkdtemp(join(tmpdir(), "forge-list-texts-"));
89 const app = createApp({ dataDir, port: 0 });
90 const keptBefore = listTextsKeptHere();
91 try {
92 listReviewsFor(app.db, ["a", "b", "c"]);
93 expect(listTextsKeptHere() - keptBefore).toBe(1);
94 } finally {
95 app.close();
96 await rm(dataDir, { recursive: true, force: true });
97 }
98 });
99 });
100
101 describe("the pages worker's readings, 2,000 times while the forge grows", () => {
102 let dataDir: string;
103 let app: App;
104 let seed: HomeFlowSeed;
105 beforeAll(async () => {
106 dataDir = await mkdtemp(join(tmpdir(), "forge-pages-memory-"));
107 app = createApp({ dataDir, port: 0 });
108 seed = await seedHomeFlow(app);
109 });
110 afterAll(async () => {
111 app.close();
112 await rm(dataDir, { recursive: true, force: true });
113 });
114
115 test(`grow the heap by under ${READING_CEILING_MIB} MiB and keep the prepared statements bounded`, async () => {
116 // Opened as `pages_worker.ts` opens its handle.
117 const db = keepFixedTexts(new Database(app.db.filename, { readonly: true }));
118 const env: PagesEnv = { db, reposRoot: app.reposRoot, config: pagesConfig(app.config), ceilingUsd: null, inline: false };
119 const keptBefore = listTextsKeptHere();
120 const repos: Repo[] = Object.values(seed.repos).map((r) => getRepo(app.db, seed.owner.username, r.name)!);
121 let written = 0;
122 // A new issue or open pull before each round: every list the readings build is one longer.
123 const write = (): void => {
124 const repo = repos[written % repos.length]!;
125 const pull = written++ % 2 === 0;
126 const made = createIssue(app.db, { repoId: repo.id, authorId: seed.owner.id, title: `Grown ${written}`, kind: pull ? "pull" : "issue" });
127 if (made.ok && pull)
128 app.db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', ?2, 'base', ?3)", [made.value.id, `grown-${written}`, `head-${written}`]);
129 };
130 const jobs = (now: number, i: number): PageJob[] => [
131 { kind: "boards", viewer: "owner", now, boardsSince: 0, out: null },
132 { kind: "boards", viewer: "agent", now, boardsSince: 0, out: null },
133 { kind: "flow", userId: seed.owner.id, now },
134 { kind: "app", repo: repos[i % repos.length]!, scope: { sealed: true }, ownerFacts: true, money: true, now },
135 ];
136 let now = Date.now();
137 const rounds = async (n: number): Promise<void> => {
138 for (let i = 0; i < n; i++) {
139 write();
140 // Past every part's keep time: each reading reads again, as the keeper's would.
141 now += 11 * 60_000;
142 for (const job of jobs(now, i)) await readPagePicture(env, job, learnedPulls());
143 }
144 };
145 try {
146 await rounds(25);
147 const before = settled();
148 // 500 rounds of four readings: 2,000.
149 await rounds(500);
150 const after = settled();
151 expect((after.heap - before.heap) / MIB).toBeLessThan(READING_CEILING_MIB);
152 expect(after.statements - before.statements).toBeLessThan(STATEMENTS_CEILING);
153 // This reading handle's and the writing one's: each keeps its own, bounded.
154 expect(listTextsKeptHere() - keptBefore).toBeLessThanOrEqual(2 * LIST_TEXTS_KEPT);
155 } finally {
156 db.close();
157 }
158 }, 120_000);
159 });
160
161 describe("/, /explore and a repository page, rendered 2,000 times", () => {
162 let f: SnappyFixture;
163 beforeAll(async () => {
164 f = await snappyFixture();
165 }, 120_000);
166 afterAll(async () => {
167 await f.close();
168 });
169
170 test(`grow the heap by under ${RENDER_CEILING_MIB} MiB`, async () => {
171 const paths = ["/", "/explore", `/${f.owner.username}/${f.repos[0]!.name}`];
172 const render = async (i: number): Promise<void> => {
173 const res = await fetch(`${f.base}${paths[i % paths.length]}`, i % 2 === 0 ? { headers: { cookie: f.ownerCookie } } : {});
174 await res.arrayBuffer();
175 expect(res.status).toBe(200);
176 };
177 for (let i = 0; i < 60; i++) await render(i);
178 const before = settled();
179 for (let i = 0; i < 2_000; i++) await render(i);
180 const after = settled();
181 expect((after.heap - before.heap) / MIB).toBeLessThan(RENDER_CEILING_MIB);
182 expect(after.statements - before.statements).toBeLessThan(STATEMENTS_CEILING);
183 }, 120_000);
184 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs