Public: anyone can read this; the crew can change it.

Sentinel: memory-growth was degraded 837 minutes today: The forge's own memory is 3299 MiB, over twice its first-hour 1420 MiB; worker raster's malloc grew most (974 MiB to 32… #14433

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14432-sentinel-memory-growth-was-degraded-837 → main 4 commits fork

  • Merged. mason/14432-sentinel-memory-growth-was-degraded-837 was merged into main as a4150a0.

Review

Sign in to review this change.
2 files changed +64 −6
src/web/og/raster.ts +43 −5
@@ -99,9 +99,30 @@ export interface RasterAnswer {
9999 /** A render not answered in this long is given up: the worker ends and the next card starts one. */
100100 const GIVE_UP_MS = 30_000;
101101
102 /**
103 * A worker is retired after this many renders. WebAssembly linear memory only grows, never
104 * shrinks, and a long-lived rasterizer held 974 MiB then 3.2 GiB of the forge's RSS (#14432);
105 * a fresh worker gives it all back.
106 */
107 export const RETIRE_AFTER = 200;
108 /** The limit in force; a test lowers it so a retirement needs few renders. */
109 export const retirement = { after: RETIRE_AFTER };
110
102111 let worker: Worker | undefined;
112 /** Workers started in this process (a test reads it to see a retirement). */
113 export const workersStarted = { count: 0 };
114 /** Renders asked of the current worker. */
115 let sent = 0;
103116 let askId = 0;
104 const open = new Map<number, { resolve: (png: Uint8Array) => void; reject: (e: Error) => void }>();
117 const open = new Map<number, { worker: Worker; resolve: (png: Uint8Array) => void; reject: (e: Error) => void }>();
118
119 /** Stop using a worker that has drawn enough: it ends once its last ask is answered. */
120 function retireIfSpent(w: Worker): void {
121 if (worker === w && sent >= retirement.after) worker = undefined;
122 if (worker === w) return;
123 for (const ask of open.values()) if (ask.worker === w) return;
124 w.terminate();
125 }
105126
106127 function failAll(why: string): void {
107128 for (const [id, ask] of open) {
@@ -116,6 +137,7 @@ function workerOf(): Worker {
116137 w.onmessage = (e: MessageEvent<RasterAnswer>) => {
117138 const ask = open.get(e.data.id);
118139 open.delete(e.data.id);
140 retireIfSpent(w);
119141 if (ask === undefined) return;
120142 if (e.data.png !== undefined) ask.resolve(e.data.png);
121143 else ask.reject(new Error(e.data.error ?? "the card worker answered nothing"));
@@ -125,6 +147,8 @@ function workerOf(): Worker {
125147 failAll(String(e.message ?? "the card worker failed"));
126148 };
127149 w.unref();
150 sent = 0;
151 workersStarted.count++;
128152 return (worker = w);
129153 }
130154
@@ -133,7 +157,17 @@ const byWorker: Drawer = (svg) => {
133157 return new Promise<Uint8Array>((resolve, reject) => {
134158 const timer = setTimeout(() => reject(new Error("the card worker did not answer")), GIVE_UP_MS);
135159 timer.unref();
160 let to: Worker;
161 try {
162 to = workerOf();
163 sent++;
164 } catch (error) {
165 clearTimeout(timer);
166 reject(error instanceof Error ? error : new Error(String(error)));
167 return;
168 }
136169 open.set(id, {
170 worker: to,
137171 resolve: (png) => {
138172 clearTimeout(timer);
139173 resolve(png);
@@ -144,16 +178,20 @@ const byWorker: Drawer = (svg) => {
144178 },
145179 });
146180 try {
147 workerOf().postMessage({ id, svg } satisfies RasterAsk);
181 to.postMessage({ id, svg } satisfies RasterAsk);
182 // The next ask starts a fresh worker; this one ends when its last answer is in.
183 if (worker === to && sent >= retirement.after) worker = undefined;
148184 } catch (error) {
149185 clearTimeout(timer);
150186 reject(error instanceof Error ? error : new Error(String(error)));
151187 }
152188 }).finally(() => {
153189 // Still open here is a render that never answered or could not be asked: its worker is ended.
154 if (open.delete(id)) {
155 worker?.terminate();
156 worker = undefined;
190 const ask = open.get(id);
191 if (ask !== undefined) {
192 open.delete(id);
193 ask.worker.terminate();
194 if (worker === ask.worker) worker = undefined;
157195 }
158196 });
159197 };
test/og-raster-worker.test.ts +21 −1
@@ -9,7 +9,7 @@ import { tmpdir } from "node:os";
99 import { join } from "node:path";
1010 import { renderCard, type CardInput } from "../src/web/og/card.ts";
1111 import { pngCache } from "../src/web/og/png-cache.ts";
12 import { drawer, rasterizer, rasterWorker } from "../src/web/og/raster.ts";
12 import { drawer, RETIRE_AFTER, retirement, rasterizer, rasterWorker, workersStarted } from "../src/web/og/raster.ts";
1313
1414 const card: CardInput = { kind: "run", title: "A run that landed a pull", lines: ["alice/notes", "#12 Draw cards off the thread"], stat: "ada · landed a pull" };
1515 const dir = await mkdtemp(join(tmpdir(), "og-raster-worker-"));
@@ -66,3 +66,23 @@ test("a card already on disk builds no SVG: its pictures are read only when it i
6666 expect(built).toBe(1);
6767 expect(Buffer.from(again).equals(Buffer.from(first))).toBe(true);
6868 });
69
70 test("a worker that has drawn enough is retired, so its wasm memory is given back (#14432)", async () => {
71 expect(RETIRE_AFTER).toBe(200);
72 rasterWorker.force = true;
73 retirement.after = 5;
74 try {
75 const draw = await drawer();
76 const svg = renderCard(card);
77 await draw(svg);
78 const before = workersStarted.count;
79 // In flight together across the limit: every ask is still answered.
80 const pngs = await Promise.all(Array.from({ length: 12 }, () => draw(svg)));
81 expect(pngs.every((p) => p[0] === 0x89)).toBe(true);
82 expect(workersStarted.count).toBeGreaterThan(before);
83 expect(Array.from((await draw(svg)).slice(1, 4))).toEqual([0x50, 0x4e, 0x47]);
84 } finally {
85 retirement.after = RETIRE_AFTER;
86 rasterWorker.force = false;
87 }
88 }, 60_000);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs