Public: anyone can read this; the crew can change it.
Sentinel: memory-growth was degraded 877 minutes today: The forge's own memory is 2911 MiB, over twice its first-hour 1420 MiB; worker pages's malloc grew most (1338 MiB to 35… #14446
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14445-sentinel-memory-growth-was-degraded-877 → main 3 commits fork 4 threads · 4 unresolved
src/flow/pages.ts +41 −4
| @@ -90,12 +90,42 @@ const laneOf = (job: PageJob): Lane => (job.kind === "board" || job.kind === "qu | ||
| 90 | 90 | const boardsSinceOf = (db: Database, job: PageJob): number => (job.kind === "boards" || job.kind === "board" ? (changed.get(db) ?? 0) : 0); |
| 91 | 91 | const workers = new Map<Lane, Worker>(); |
| 92 | 92 | let askId = 0; |
| 93 | const open = new Map<number, { lane: Lane; resolve: (p: Picture) => void; reject: (e: Error) => void }>(); | |
| 93 | const open = new Map<number, { lane: Lane; worker: Worker; resolve: (p: Picture) => void; reject: (e: Error) => void }>(); | |
| 94 | 94 | |
| 95 | /** | |
| 96 | * A worker is retired after this many readings: the keeper reads the pictures again all day, and | |
| 97 | * a long-lived worker's malloc grew from 1338 MiB to 3503 MiB (#14445); a fresh one gives it back. | |
| 98 | */ | |
| 99 | export const RETIRE_AFTER = 500; | |
| 100 | /** The one seam for a test: the limit in force, and how many workers were started and ended. */ | |
| 101 | export const pagesWorkers = { retireAfter: RETIRE_AFTER, started: 0, ended: 0 }; | |
| 102 | const asked = new WeakMap<Worker, number>(); | |
| 103 | ||
| 104 | function stop(w: Worker): void { | |
| 105 | pagesWorkers.ended++; | |
| 106 | w.terminate(); | |
| 107 | } | |
outdated
A retired worker leaves |
||
| 108 | ||
| 109 | /** | |
| 110 | * The one place a worker is retired. A spent worker is no longer given asks; it ends when its last | |
| 111 | * ask is answered (or, if it fails first, when `failLane` ends it). | |
| 112 | */ | |
| 113 | function retireIfSpent(lane: Lane, w: Worker): void { | |
| 114 | if (workers.get(lane) === w && (asked.get(w) ?? 0) >= pagesWorkers.retireAfter) workers.delete(lane); | |
| 115 | if (workers.get(lane) === w) return; | |
| 116 | for (const ask of open.values()) if (ask.worker === w) return; | |
| 117 | stop(w); | |
| 118 | } | |
| 119 | ||
| 120 | /** | |
| 121 | * Every ask a lane holds fails, and the workers that held them end: none is left running | |
| 122 | * unreachable. | |
| 123 | */ | |
| 95 | 124 | function failLane(lane: Lane, why: string): void { |
| 96 | 125 | for (const [id, ask] of open) { |
| 97 | 126 | if (ask.lane !== lane) continue; |
| 98 | 127 | open.delete(id); |
| 128 | if (workers.get(lane) !== ask.worker) stop(ask.worker); | |
| 99 | 129 | ask.reject(new Error(why)); |
| 100 | 130 | } |
| 101 | 131 | } |
| @@ -107,6 +137,7 @@ function workerOf(lane: Lane): Worker { | ||
| 107 | 137 | w.onmessage = (e: MessageEvent<PagesAnswer>) => { |
| 108 | 138 | const ask = open.get(e.data.id); |
| 109 | 139 | open.delete(e.data.id); |
| 140 | retireIfSpent(lane, w); | |
| 110 | 141 | if (ask === undefined) return; |
| 111 | 142 | if (e.data.picture !== undefined) ask.resolve(e.data.picture); |
| 112 | 143 | else ask.reject(new Error(e.data.error ?? "the pages worker answered nothing")); |
| @@ -114,15 +145,18 @@ function workerOf(lane: Lane): Worker { | ||
| 114 | 145 | w.onerror = (e) => { |
| 115 | 146 | if (workers.get(lane) === w) workers.delete(lane); |
| 116 | 147 | failLane(lane, String(e.message ?? "the pages worker failed")); |
| 148 | stop(w); | |
| 117 | 149 | }; |
| 118 | 150 | w.unref(); |
| 119 | 151 | workers.set(lane, w); |
| 152 | pagesWorkers.started++; | |
| 120 | 153 | return w; |
| 121 | 154 | } |
| 122 | 155 | |
| 123 | 156 | function endWorker(lane: Lane): void { |
Non-blocking: endWorker stops w here, then failLane stops the same worker again per open ask. Guard stop() so each worker is ended and counted once. |
||
| 124 | workers.get(lane)?.terminate(); | |
| 157 | const w = workers.get(lane); | |
| 125 | 158 | workers.delete(lane); |
| 159 | if (w !== undefined) stop(w); | |
| 126 | 160 | failLane(lane, "the pages worker was ended"); |
| 127 | 161 | } |
| 128 | 162 | |
| @@ -139,9 +173,11 @@ function viaWorker(app: App, job: PageJob): Promise<Picture> { | ||
| 139 | 173 | return new Promise((resolve, reject) => { |
| 140 | 174 | const id = ++askId; |
| 141 | 175 | const lane = laneOf(job); |
| 142 | open.set(id, { lane, resolve, reject }); | |
| 143 | 176 | try { |
| 144 | workerOf(lane).postMessage({ | |
| 177 | const w = workerOf(lane); | |
| 178 | open.set(id, { lane, worker: w, resolve, reject }); | |
| 179 | asked.set(w, (asked.get(w) ?? 0) + 1); | |
| 180 | w.postMessage({ | |
| 145 | 181 | id, |
| 146 | 182 | path: app.db.filename, |
| 147 | 183 | reposRoot: app.reposRoot, |
| @@ -151,6 +187,7 @@ function viaWorker(app: App, job: PageJob): Promise<Picture> { | ||
| 151 | 187 | // Only the ready lists read which files each open pull touches. |
| 152 | 188 | learned: job.kind === "boards" ? learnedPulls() : [], |
| 153 | 189 | } satisfies PagesAsk); |
| 190 | retireIfSpent(lane, w); | |
| 154 | 191 | } catch (error) { |
| 155 | 192 | open.delete(id); |
| 156 | 193 | reject(error instanceof Error ? error : new Error(String(error))); |
test/pages-worker-retire.test.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | /** | |
| 2 | * The pages worker is retired after a number of readings so its malloc is given back (#14445): | |
| 3 | * the next ask starts a fresh worker and still answers; a retired worker that is failed is ended | |
| 4 | * too. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { flowPicture, isFailed, pagesWorkers, RETIRE_AFTER } from "../src/flow/pages.ts"; | |
| 12 | import { seedHomeFlow, type HomeFlowSeed } from "./helpers/home-flow-seed.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: App; | |
| 16 | let seed: HomeFlowSeed; | |
| 17 | const env = process.env["NODE_ENV"]; | |
| 18 | ||
| 19 | beforeAll(async () => { | |
| 20 | dataDir = await mkdtemp(join(tmpdir(), "forge-pages-retire-")); | |
| 21 | app = createApp({ dataDir, port: 0 }); | |
| 22 | seed = await seedHomeFlow(app); | |
| 23 | // The pages are read by the worker only outside `bun test`'s inline path. | |
| 24 | process.env["NODE_ENV"] = "production"; | |
| 25 | }); | |
| 26 | ||
| 27 | afterAll(async () => { | |
| 28 | pagesWorkers.retireAfter = RETIRE_AFTER; | |
| 29 | process.env["NODE_ENV"] = env ?? "test"; | |
| 30 | app.close(); | |
| 31 | await rm(dataDir, { recursive: true, force: true }); | |
| 32 | }); | |
| 33 | ||
| 34 | test("a worker that has read its share ends, and the next reading starts a new one that answers", async () => { | |
| 35 | pagesWorkers.retireAfter = 2; | |
| 36 | const started = pagesWorkers.started; | |
| 37 | const ended = pagesWorkers.ended; | |
| 38 | for (let i = 0; i < 5; i++) { | |
| 39 | const read = await flowPicture(app, seed.owner.id, 20_000, Date.now() + i * 3_600_000); | |
| 40 | expect(read !== undefined && !isFailed(read)).toBe(true); | |
| 41 | } | |
| 42 | // Five readings at two a worker: three workers, and the two spent ones ended. | |
| 43 | expect(pagesWorkers.started - started).toBe(3); | |
| 44 | expect(pagesWorkers.ended - ended).toBe(2); | |
| 45 | }, 60_000); | |
| 46 | ||
| 47 | test("a retired worker still holding an ask when its lane is ended is ended too", async () => { | |
| 48 | pagesWorkers.retireAfter = 1; | |
| 49 | const started = pagesWorkers.started; | |
| 50 | const ended = pagesWorkers.ended; | |
| 51 | const now = Date.now() + 10 * 3_600_000; | |
| 52 | // The ask goes to a worker that is retired at once (its one reading), and is still open... | |
| 53 | const slow = flowPicture(app, seed.owner.id, 20_000, now); | |
| 54 | // ...when the same picture is asked for past the give-up time: the lane's workers are ended. | |
| 55 | const later = flowPicture(app, seed.owner.id, 20_000, now + 60_000); | |
| 56 | await Promise.all([slow, later]); | |
| 57 | // Every worker started was ended, the failed retired one included: none is left running. | |
| 58 | expect(pagesWorkers.started - started).toBeGreaterThanOrEqual(1); | |
| 59 | // (The worker the first test left in the lane is ended here too.) | |
| 60 | expect(pagesWorkers.ended - ended).toBeGreaterThanOrEqual(pagesWorkers.started - started); | |
Non-blocking: with exact ended counting, assert equality instead of >=, so a double stop cannot hide a leak. |
||
| 61 | }, 60_000); |
Test-only mutable exports in production code. Narrow this to one seam, or observe retirement another way.