Public: anyone can read this; the crew can change it.

Sentinel: memory-growth was degraded 877 minutes today: The forge's own memory is 2911 MiB, over twice its first-hour 1420 MiB; worker pages's malloc grew most (1338 MiB to 35… #14446

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14445-sentinel-memory-growth-was-degraded-877 → main 3 commits fork 4 threads · 4 unresolved

  • Merged. mason/14445-sentinel-memory-growth-was-degraded-877 was merged into main as 1b66953.

Review

Sign in to review this change.
2 files changed +102 −4
src/flow/pages.ts +41 −4
@@ -90,12 +90,42 @@ const laneOf = (job: PageJob): Lane => (job.kind === "board" || job.kind === "qu
9090 const boardsSinceOf = (db: Database, job: PageJob): number => (job.kind === "boards" || job.kind === "board" ? (changed.get(db) ?? 0) : 0);
9191 const workers = new Map<Lane, Worker>();
9292 let askId = 0;
93 const open = new Map<number, { lane: Lane; resolve: (p: Picture) => void; reject: (e: Error) => void }>();
93 const open = new Map<number, { lane: Lane; worker: Worker; resolve: (p: Picture) => void; reject: (e: Error) => void }>();
9494
95 /**
96 * A worker is retired after this many readings: the keeper reads the pictures again all day, and
97 * a long-lived worker's malloc grew from 1338 MiB to 3503 MiB (#14445); a fresh one gives it back.
98 */
99 export const RETIRE_AFTER = 500;
100 /** The one seam for a test: the limit in force, and how many workers were started and ended. */
101 export const pagesWorkers = { retireAfter: RETIRE_AFTER, started: 0, ended: 0 };
102 const asked = new WeakMap<Worker, number>();
103
outdated

Test-only mutable exports in production code. Narrow this to one seam, or observe retirement another way.

104 function stop(w: Worker): void {
105 pagesWorkers.ended++;
106 w.terminate();
107 }
outdated

A retired worker leaves workers, so endWorker/failLane cannot reach it. If it errors, its asks are rejected and removed and nothing ever terminates it. Terminate retiring workers on failure too.

108
109 /**
110 * The one place a worker is retired. A spent worker is no longer given asks; it ends when its last
111 * ask is answered (or, if it fails first, when `failLane` ends it).
112 */
113 function retireIfSpent(lane: Lane, w: Worker): void {
114 if (workers.get(lane) === w && (asked.get(w) ?? 0) >= pagesWorkers.retireAfter) workers.delete(lane);
115 if (workers.get(lane) === w) return;
116 for (const ask of open.values()) if (ask.worker === w) return;
117 stop(w);
118 }
119
120 /**
121 * Every ask a lane holds fails, and the workers that held them end: none is left running
122 * unreachable.
123 */
95124 function failLane(lane: Lane, why: string): void {
96125 for (const [id, ask] of open) {
97126 if (ask.lane !== lane) continue;
98127 open.delete(id);
128 if (workers.get(lane) !== ask.worker) stop(ask.worker);
99129 ask.reject(new Error(why));
100130 }
101131 }
@@ -107,6 +137,7 @@ function workerOf(lane: Lane): Worker {
107137 w.onmessage = (e: MessageEvent<PagesAnswer>) => {
108138 const ask = open.get(e.data.id);
109139 open.delete(e.data.id);
140 retireIfSpent(lane, w);
110141 if (ask === undefined) return;
111142 if (e.data.picture !== undefined) ask.resolve(e.data.picture);
112143 else ask.reject(new Error(e.data.error ?? "the pages worker answered nothing"));
@@ -114,15 +145,18 @@ function workerOf(lane: Lane): Worker {
114145 w.onerror = (e) => {
115146 if (workers.get(lane) === w) workers.delete(lane);
116147 failLane(lane, String(e.message ?? "the pages worker failed"));
148 stop(w);
117149 };
118150 w.unref();
119151 workers.set(lane, w);
152 pagesWorkers.started++;
120153 return w;
121154 }
122155
123156 function endWorker(lane: Lane): void {

Non-blocking: endWorker stops w here, then failLane stops the same worker again per open ask. Guard stop() so each worker is ended and counted once.

124 workers.get(lane)?.terminate();
157 const w = workers.get(lane);
125158 workers.delete(lane);
159 if (w !== undefined) stop(w);
126160 failLane(lane, "the pages worker was ended");
127161 }
128162
@@ -139,9 +173,11 @@ function viaWorker(app: App, job: PageJob): Promise<Picture> {
139173 return new Promise((resolve, reject) => {
140174 const id = ++askId;
141175 const lane = laneOf(job);
142 open.set(id, { lane, resolve, reject });
143176 try {
144 workerOf(lane).postMessage({
177 const w = workerOf(lane);
178 open.set(id, { lane, worker: w, resolve, reject });
179 asked.set(w, (asked.get(w) ?? 0) + 1);
180 w.postMessage({
145181 id,
146182 path: app.db.filename,
147183 reposRoot: app.reposRoot,
@@ -151,6 +187,7 @@ function viaWorker(app: App, job: PageJob): Promise<Picture> {
151187 // Only the ready lists read which files each open pull touches.
152188 learned: job.kind === "boards" ? learnedPulls() : [],
153189 } satisfies PagesAsk);
190 retireIfSpent(lane, w);
154191 } catch (error) {
155192 open.delete(id);
156193 reject(error instanceof Error ? error : new Error(String(error)));
test/pages-worker-retire.test.ts added +61
@@ -0,0 +1,61 @@
1 /**
2 * The pages worker is retired after a number of readings so its malloc is given back (#14445):
3 * the next ask starts a fresh worker and still answers; a retired worker that is failed is ended
4 * too.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { flowPicture, isFailed, pagesWorkers, RETIRE_AFTER } from "../src/flow/pages.ts";
12 import { seedHomeFlow, type HomeFlowSeed } from "./helpers/home-flow-seed.ts";
13
14 let dataDir: string;
15 let app: App;
16 let seed: HomeFlowSeed;
17 const env = process.env["NODE_ENV"];
18
19 beforeAll(async () => {
20 dataDir = await mkdtemp(join(tmpdir(), "forge-pages-retire-"));
21 app = createApp({ dataDir, port: 0 });
22 seed = await seedHomeFlow(app);
23 // The pages are read by the worker only outside `bun test`'s inline path.
24 process.env["NODE_ENV"] = "production";
25 });
26
27 afterAll(async () => {
28 pagesWorkers.retireAfter = RETIRE_AFTER;
29 process.env["NODE_ENV"] = env ?? "test";
30 app.close();
31 await rm(dataDir, { recursive: true, force: true });
32 });
33
34 test("a worker that has read its share ends, and the next reading starts a new one that answers", async () => {
35 pagesWorkers.retireAfter = 2;
36 const started = pagesWorkers.started;
37 const ended = pagesWorkers.ended;
38 for (let i = 0; i < 5; i++) {
39 const read = await flowPicture(app, seed.owner.id, 20_000, Date.now() + i * 3_600_000);
40 expect(read !== undefined && !isFailed(read)).toBe(true);
41 }
42 // Five readings at two a worker: three workers, and the two spent ones ended.
43 expect(pagesWorkers.started - started).toBe(3);
44 expect(pagesWorkers.ended - ended).toBe(2);
45 }, 60_000);
46
47 test("a retired worker still holding an ask when its lane is ended is ended too", async () => {
48 pagesWorkers.retireAfter = 1;
49 const started = pagesWorkers.started;
50 const ended = pagesWorkers.ended;
51 const now = Date.now() + 10 * 3_600_000;
52 // The ask goes to a worker that is retired at once (its one reading), and is still open...
53 const slow = flowPicture(app, seed.owner.id, 20_000, now);
54 // ...when the same picture is asked for past the give-up time: the lane's workers are ended.
55 const later = flowPicture(app, seed.owner.id, 20_000, now + 60_000);
56 await Promise.all([slow, later]);
57 // Every worker started was ended, the failed retired one included: none is left running.
58 expect(pagesWorkers.started - started).toBeGreaterThanOrEqual(1);
59 // (The worker the first test left in the lane is ended here too.)
60 expect(pagesWorkers.ended - ended).toBeGreaterThanOrEqual(pagesWorkers.started - started);

Non-blocking: with exact ended counting, assert equality instead of >=, so a double stop cannot hide a leak.

61 }, 60_000);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs