Public: anyone can read this; the crew can change it.
Sentinel: memory-growth was degraded 837 minutes today: The forge's own memory is 3299 MiB, over twice its first-hour 1420 MiB; worker raster's malloc grew most (974 MiB to 32… #14433
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14432-sentinel-memory-growth-was-degraded-837 → main 4 commits fork
2 files
changed +64
−6
src/web/og/raster.ts +43 −5
| @@ -99,9 +99,30 @@ export interface RasterAnswer { | ||
| 99 | 99 | /** A render not answered in this long is given up: the worker ends and the next card starts one. */ |
| 100 | 100 | const GIVE_UP_MS = 30_000; |
| 101 | 101 | |
| 102 | /** | |
| 103 | * A worker is retired after this many renders. WebAssembly linear memory only grows, never | |
| 104 | * shrinks, and a long-lived rasterizer held 974 MiB then 3.2 GiB of the forge's RSS (#14432); | |
| 105 | * a fresh worker gives it all back. | |
| 106 | */ | |
| 107 | export const RETIRE_AFTER = 200; | |
| 108 | /** The limit in force; a test lowers it so a retirement needs few renders. */ | |
| 109 | export const retirement = { after: RETIRE_AFTER }; | |
| 110 | ||
| 102 | 111 | let worker: Worker | undefined; |
| 112 | /** Workers started in this process (a test reads it to see a retirement). */ | |
| 113 | export const workersStarted = { count: 0 }; | |
| 114 | /** Renders asked of the current worker. */ | |
| 115 | let sent = 0; | |
| 103 | 116 | let askId = 0; |
| 104 | const open = new Map<number, { resolve: (png: Uint8Array) => void; reject: (e: Error) => void }>(); | |
| 117 | const open = new Map<number, { worker: Worker; resolve: (png: Uint8Array) => void; reject: (e: Error) => void }>(); | |
| 118 | ||
| 119 | /** Stop using a worker that has drawn enough: it ends once its last ask is answered. */ | |
| 120 | function retireIfSpent(w: Worker): void { | |
| 121 | if (worker === w && sent >= retirement.after) worker = undefined; | |
| 122 | if (worker === w) return; | |
| 123 | for (const ask of open.values()) if (ask.worker === w) return; | |
| 124 | w.terminate(); | |
| 125 | } | |
| 105 | 126 | |
| 106 | 127 | function failAll(why: string): void { |
| 107 | 128 | for (const [id, ask] of open) { |
| @@ -116,6 +137,7 @@ function workerOf(): Worker { | ||
| 116 | 137 | w.onmessage = (e: MessageEvent<RasterAnswer>) => { |
| 117 | 138 | const ask = open.get(e.data.id); |
| 118 | 139 | open.delete(e.data.id); |
| 140 | retireIfSpent(w); | |
| 119 | 141 | if (ask === undefined) return; |
| 120 | 142 | if (e.data.png !== undefined) ask.resolve(e.data.png); |
| 121 | 143 | else ask.reject(new Error(e.data.error ?? "the card worker answered nothing")); |
| @@ -125,6 +147,8 @@ function workerOf(): Worker { | ||
| 125 | 147 | failAll(String(e.message ?? "the card worker failed")); |
| 126 | 148 | }; |
| 127 | 149 | w.unref(); |
| 150 | sent = 0; | |
| 151 | workersStarted.count++; | |
| 128 | 152 | return (worker = w); |
| 129 | 153 | } |
| 130 | 154 | |
| @@ -133,7 +157,17 @@ const byWorker: Drawer = (svg) => { | ||
| 133 | 157 | return new Promise<Uint8Array>((resolve, reject) => { |
| 134 | 158 | const timer = setTimeout(() => reject(new Error("the card worker did not answer")), GIVE_UP_MS); |
| 135 | 159 | timer.unref(); |
| 160 | let to: Worker; | |
| 161 | try { | |
| 162 | to = workerOf(); | |
| 163 | sent++; | |
| 164 | } catch (error) { | |
| 165 | clearTimeout(timer); | |
| 166 | reject(error instanceof Error ? error : new Error(String(error))); | |
| 167 | return; | |
| 168 | } | |
| 136 | 169 | open.set(id, { |
| 170 | worker: to, | |
| 137 | 171 | resolve: (png) => { |
| 138 | 172 | clearTimeout(timer); |
| 139 | 173 | resolve(png); |
| @@ -144,16 +178,20 @@ const byWorker: Drawer = (svg) => { | ||
| 144 | 178 | }, |
| 145 | 179 | }); |
| 146 | 180 | try { |
| 147 | workerOf().postMessage({ id, svg } satisfies RasterAsk); | |
| 181 | to.postMessage({ id, svg } satisfies RasterAsk); | |
| 182 | // The next ask starts a fresh worker; this one ends when its last answer is in. | |
| 183 | if (worker === to && sent >= retirement.after) worker = undefined; | |
| 148 | 184 | } catch (error) { |
| 149 | 185 | clearTimeout(timer); |
| 150 | 186 | reject(error instanceof Error ? error : new Error(String(error))); |
| 151 | 187 | } |
| 152 | 188 | }).finally(() => { |
| 153 | 189 | // Still open here is a render that never answered or could not be asked: its worker is ended. |
| 154 | if (open.delete(id)) { | |
| 155 | worker?.terminate(); | |
| 156 | worker = undefined; | |
| 190 | const ask = open.get(id); | |
| 191 | if (ask !== undefined) { | |
| 192 | open.delete(id); | |
| 193 | ask.worker.terminate(); | |
| 194 | if (worker === ask.worker) worker = undefined; | |
| 157 | 195 | } |
| 158 | 196 | }); |
| 159 | 197 | }; |
test/og-raster-worker.test.ts +21 −1
| @@ -9,7 +9,7 @@ import { tmpdir } from "node:os"; | ||
| 9 | 9 | import { join } from "node:path"; |
| 10 | 10 | import { renderCard, type CardInput } from "../src/web/og/card.ts"; |
| 11 | 11 | import { pngCache } from "../src/web/og/png-cache.ts"; |
| 12 | import { drawer, rasterizer, rasterWorker } from "../src/web/og/raster.ts"; | |
| 12 | import { drawer, RETIRE_AFTER, retirement, rasterizer, rasterWorker, workersStarted } from "../src/web/og/raster.ts"; | |
| 13 | 13 | |
| 14 | 14 | const card: CardInput = { kind: "run", title: "A run that landed a pull", lines: ["alice/notes", "#12 Draw cards off the thread"], stat: "ada · landed a pull" }; |
| 15 | 15 | const dir = await mkdtemp(join(tmpdir(), "og-raster-worker-")); |
| @@ -66,3 +66,23 @@ test("a card already on disk builds no SVG: its pictures are read only when it i | ||
| 66 | 66 | expect(built).toBe(1); |
| 67 | 67 | expect(Buffer.from(again).equals(Buffer.from(first))).toBe(true); |
| 68 | 68 | }); |
| 69 | ||
| 70 | test("a worker that has drawn enough is retired, so its wasm memory is given back (#14432)", async () => { | |
| 71 | expect(RETIRE_AFTER).toBe(200); | |
| 72 | rasterWorker.force = true; | |
| 73 | retirement.after = 5; | |
| 74 | try { | |
| 75 | const draw = await drawer(); | |
| 76 | const svg = renderCard(card); | |
| 77 | await draw(svg); | |
| 78 | const before = workersStarted.count; | |
| 79 | // In flight together across the limit: every ask is still answered. | |
| 80 | const pngs = await Promise.all(Array.from({ length: 12 }, () => draw(svg))); | |
| 81 | expect(pngs.every((p) => p[0] === 0x89)).toBe(true); | |
| 82 | expect(workersStarted.count).toBeGreaterThan(before); | |
| 83 | expect(Array.from((await draw(svg)).slice(1, 4))).toEqual([0x50, 0x4e, 0x47]); | |
| 84 | } finally { | |
| 85 | retirement.after = RETIRE_AFTER; | |
| 86 | rasterWorker.force = false; | |
| 87 | } | |
| 88 | }, 60_000); |
new above
offline