Public: anyone can read this; the crew can change it.

A memory-growth finding carries its own evidence: each VM's heap, the outside-heap figure and the held-by-name counts, at the first hour, at the trip and now #14376

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/14362-a-memory-growth-finding-carries-its-own → main 1 commit fork

  • Merged. torque/14362-a-memory-growth-finding-carries-its-own was merged into main as bd36f5e.

Review

Sign in to review this change.
4 files changed +205 −10
src/boards/memory.ts +5 −2
@@ -76,9 +76,12 @@ export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemoryS
7676 try {
7777 const reading = readNow();
7878 recordMemory(app.db, reading.rss, reading.heap, now);
79 memoryWatch.note(reading.rss, reading.heap, now);
79 // The held counts are read once: the line writes them, the watch keeps them for a finding
80 // (#14362).
81 const held = heldNow(app);
82 memoryWatch.note(reading.rss, reading.heap, now, { commit: reading.commit, held });
8083 app.log.info(memoryLine(reading, process.uptime() * 1000));
81 app.log.info(heldLine(heldNow(app)));
84 app.log.info(heldLine(held));
8285 const allocated = allocationLine(allocationsOf(app));
8386 if (allocated !== undefined) app.log.info(allocated);
8487 } catch (error) {
src/crew/sentinel.ts +12 −1
@@ -56,6 +56,7 @@ import type { FindingKind } from "../hunt/finding.ts";
5656 import { addLabels, kindLabels } from "../hunt/label.ts";
5757 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
5858 import { seenCause } from "./sentinel_seen.ts";
59 import { memoryWatch, type MemoryWatch } from "../perf/memory_watch.ts";
5960
6061 const log = createLog("sentinel");
6162
@@ -411,7 +412,14 @@ export function repeatedErrors(now: number, ledger?: ErrorLedger): Finding[] {
411412 * Each cause whose spans total more than ten minutes in the day ending at
412413 * `now`, quoting its sentence. Shorter totals stay in the history.
413414 */
414 export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] {
415 export function degradedCauses(
416 db: Database,
417 now: number,
418 forgeRepoId?: string,
419 dataDir?: string,
420 ceilingUsd: number | null = null,
421 watch: Pick<MemoryWatch, "evidence"> = memoryWatch,
422 ): Finding[] {
415423 const from = now - DAY_MS;
416424 const byCause = new Map<
417425 string,
@@ -448,6 +456,9 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string,
448456 `total: ${minutes} minutes in the day`,
449457 `href: ${d.href}`,
450458 ...sharedLines(db, cause, now, forgeRepoId, dataDir),
459 // /system is admin people only: the finding carries what grew, so a builder can name it
460 // (#14362).
461 ...(cause === "memory-growth" ? watch.evidence() : []),
451462 ],
452463 pillar: "P0 Trust: the forge says when it is not well, and is well again",
453464 kind: "defect",
src/perf/memory_watch.ts +93 −7
@@ -172,13 +172,50 @@ function mostGrown(owners: Iterable<readonly [string, Owner]>, now: number): Gro
172172 return best;
173173 }
174174
175 /**
176 * What the serving VM's minute tick also hands the watch (#14362): mimalloc's commit for the
177 * process and the held-by-name counts it already read for its line. Both are kept as the
178 * latest and, at the trip, as they stood then; nothing here is read by the watch itself.
179 */
180 export interface NoteExtra {
181 readonly commit?: number;
182 readonly held?: readonly HeldCount[];
183 }
184
185 /** The watch as it stood at the trip: the minute RSS first reached twice its first hour. */
186 interface Trip {
187 readonly at: number;
188 readonly rss: number;
189 readonly commit: number | undefined;
190 readonly outside: number | undefined;
191 /** Each fresh owner's JS heap then, by name. */
192 readonly heaps: ReadonlyMap<string, number>;
193 readonly held: readonly HeldCount[];
194 }
195
196 /**
197 * How many held-by-name counts the evidence names, and how many workers at most (#14362: under 40
198 * lines).
199 */
200 export const EVIDENCE_HELD = 10;
201 export const EVIDENCE_WORKERS = 16;
202
175203 export interface MemoryWatch {
176 /** The serving VM's minute reading: the process's RSS and main's own heap. */
177 note(rss: number, heap: number, now?: number): void;
204 /**
205 * The serving VM's minute reading: the process's RSS and main's own heap, and what `extra` adds.
206 */
207 note(rss: number, heap: number, now?: number, extra?: NoteExtra): void;
178208 /** A worker's minute reading (`hearWorkers` carries it here), against its own first hour. */
179209 noteWorker(r: WorkerReading, now?: number): void;
180210 /** RSS at least twice its first-hour mean, with the part that grew most, or undefined. */
181211 grown(): Grown | undefined;
212 /**
213 * The lines a memory-growth finding quotes (#14362): each reporting VM's JS heap and the
214 * process's RSS outside every heap at the first-hour mean, at the trip and now; mimalloc's
215 * commit; the largest held-by-name counts at the trip and now. Read from what the minute
216 * tick kept: nothing is measured here. Empty before any reading.
217 */
218 evidence(): string[];
182219 /**
183220 * The heap now (read here, or `heapNow` in a test) against the last minute's sample, for a
184221 * stall line.
@@ -193,13 +230,29 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch {
193230 let last: { heap: number; rss: number; at: number } | undefined;
194231 let above: number | undefined;
195232 let latest: number | undefined;
233 let commitSum = 0;
234 let commitCount = 0;
235 let commitNow: number | undefined;
236 let heldNow: readonly HeldCount[] = [];
237 let trip: Trip | undefined;
196238 const main = owner(startedAt);
197239 const workers = new Map<string, Owner>();
240 const grown = (): Grown | undefined => {
241 if (base === undefined || latest === undefined || above === undefined || latest < base * GROWN_RATIO) return undefined;
242 const grew = mostGrown([["main", main], ...workers], main.at);
243 return { baselineBytes: base, rssBytes: latest, since: above, ...(grew === undefined ? {} : { grew }) };
244 };
198245 return {
199 note(rss, heap, now = Date.now()) {
246 note(rss, heap, now = Date.now(), extra = {}) {
200247 if (!Number.isFinite(rss) || rss <= 0) return;
201248 last = { heap, rss, at: now };
202249 latest = rss;
250 commitNow = bytes(extra.commit) ? extra.commit : undefined;
251 if (commitNow !== undefined && now - startedAt < FIRST_HOUR_MS) {
252 commitSum += commitNow;
253 commitCount++;
254 }
255 if (extra.held !== undefined) heldNow = extra.held.slice();
203256 // Every fresh worker's heap is its own VM's, inside the process's RSS: not outside.
204257 let others = 0;
205258 for (const w of workers.values()) if (now - w.at <= WORKER_STALE_MS) others += w.heap;
@@ -216,6 +269,12 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch {
216269 }
217270 if (rss > base * GROWING_RATIO) above ??= now;
218271 else above = undefined;
272 if (above === undefined) trip = undefined;
273 else if (trip === undefined && rss >= base * GROWN_RATIO) {
274 const heaps = new Map<string, number>([["main", heap]]);
275 for (const [name, w] of workers) if (now - w.at <= WORKER_STALE_MS) heaps.set(name, w.heap);
276 trip = { at: now, rss, commit: commitNow, outside: main.latest.outside, heaps, held: heldNow };
277 }
219278 },
220279 noteWorker(r, now = Date.now()) {
221280 let o = workers.get(r.name);
@@ -227,10 +286,37 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch {
227286 o.heap = r.heap;
228287 take(o, { heap: r.heap }, now);
229288 },
230 grown() {
231 if (base === undefined || latest === undefined || above === undefined || latest < base * GROWN_RATIO) return undefined;
232 const grew = mostGrown([["main", main], ...workers], main.at);
233 return { baselineBytes: base, rssBytes: latest, since: above, ...(grew === undefined ? {} : { grew }) };
289 grown,
290 evidence() {
291 if (latest === undefined) return [];
292 const at = main.at;
293 const first = (o: Owner, p: Part): number | undefined =>
294 o.base !== undefined ? o.base[p] : o.count[p] > 0 ? o.sum[p] / o.count[p] : undefined;
295 const fig = (n: number | undefined): string => (n === undefined ? "none" : mib(n));
296 const three = (a: number | undefined, t: number | undefined, n: number | undefined): string =>
297 `first hour ${fig(a)}, trip ${trip === undefined ? "none" : fig(t)}, now ${fig(n)}`;
298 const out = [
299 `memory rss: first hour ${fig(base ?? (count > 0 ? sum / count : undefined))}, trip ${trip === undefined ? "none" : `${mib(trip.rss)} at ${new Date(trip.at).toISOString()}`}, now ${mib(latest)}`,
300 ];
301 const g = grown();
302 if (g?.grew !== undefined) out.push(`memory grew most: ${grewWords(g.grew)}, ${mib(g.grew.fromBytes)} to ${mib(g.grew.toBytes)}`);
303 out.push(`memory heap main: ${three(first(main, "heap"), trip?.heaps.get("main"), main.latest.heap)}`);
304 const fresh = [...workers].filter(([, w]) => at - w.at <= WORKER_STALE_MS).sort(([a], [b]) => (a < b ? -1 : 1));
305 for (const [name, w] of fresh.slice(0, EVIDENCE_WORKERS))
306 out.push(`memory heap worker ${name}: ${three(first(w, "heap"), trip?.heaps.get(name), w.latest.heap)}`);
307 if (fresh.length > EVIDENCE_WORKERS) out.push(`memory heap: ${fresh.length - EVIDENCE_WORKERS} more workers not listed`);
308 out.push(`memory outside every heap (process): ${three(first(main, "outside"), trip?.outside, main.latest.outside)}`);
309 out.push(`memory mimalloc committed (process): ${three(commitCount > 0 ? commitSum / commitCount : undefined, trip?.commit, commitNow)}`);
310 const then = new Map((trip?.held ?? []).map((c) => [c.name, c.size]));
311 const named = heldNow.filter((c) => c.size !== 0).slice(0, EVIDENCE_HELD);
312 if (named.length === 0) out.push("memory held: nothing held by name now");
313 for (const c of named) {
314 const was = then.get(c.name);
315 const count = (n: number): string => (n < 0 ? "unread" : String(n));
316 const growth = was === undefined || was < 0 || c.size < 0 ? "" : ` (${c.size - was >= 0 ? "+" : ""}${c.size - was})`;
317 out.push(`memory held ${c.name}: trip ${trip === undefined || was === undefined ? "none" : count(was)}, now ${count(c.size)}${growth}`);
318 }
319 return out;
234320 },
235321 heapAcross(now = Date.now(), heapNow = heapSize()) {
236322 if (last === undefined) return `heap ${mib(heapNow)}`;
test/sentinel-memory-evidence.test.ts added +95
@@ -0,0 +1,95 @@
1 /**
2 * A memory-growth finding carries its own evidence (#14362): /system is admin people only,
3 * so the issue a builder is handed must name what grew, with its figures, from the watch.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import "./helpers/globals.ts";
10 import { createApp, type App } from "../src/app.ts";
11 import { degradedCauses } from "../src/crew/sentinel.ts";
12 import { createMemoryWatch, EVIDENCE_HELD, FIRST_HOUR_MS } from "../src/perf/memory_watch.ts";
13
14 const MIB = 1048576;
15 const MIN = 60_000;
16 let dir: string;
17 let app: App;
18
19 beforeAll(async () => {
20 dir = await mkdtemp(join(tmpdir(), "forge-sentinel-memory-"));
21 app = createApp({ dataDir: dir, port: 0 });
22 });
23 afterAll(async () => {
24 app.db.close();
25 await rm(dir, { recursive: true, force: true });
26 });
27
28 /** A watch whose worker `pages` grew from 100 MiB to 2410 MiB, with `sse` held growing. */
29 function grownWatch(start: number) {
30 const w = createMemoryWatch(start);
31 const held = (sse: number) => [
32 { name: "sse", size: sse },
33 { name: "turn", size: 3 },
34 ...Array.from({ length: 12 }, (_, i) => ({ name: `h${i}`, size: 1 })),
35 ];
36 for (let i = 0; i < 12; i++) {
37 w.noteWorker({ name: "pages", heap: 100 * MIB }, start + i * 5 * MIN);
38 w.note(2000 * MIB, 500 * MIB, start + i * 5 * MIN, { commit: 1200 * MIB, held: held(10) });
39 }
40 // The hour is over: the baseline is struck, then the trip (twice the first hour), then now.
41 const trip = start + FIRST_HOUR_MS + MIN;
42 w.noteWorker({ name: "pages", heap: 100 * MIB }, start + FIRST_HOUR_MS);
43 w.note(2000 * MIB, 500 * MIB, start + FIRST_HOUR_MS, { commit: 1200 * MIB, held: held(10) });
44 w.noteWorker({ name: "pages", heap: 2000 * MIB }, trip);
45 w.note(4100 * MIB, 500 * MIB, trip, { commit: 3000 * MIB, held: held(400) });
46 w.noteWorker({ name: "pages", heap: 2410 * MIB }, trip + MIN);
47 w.note(4200 * MIB, 500 * MIB, trip + MIN, { commit: 3100 * MIB, held: held(900) });
48 return { w, trip };
49 }
50
51 test("a memory-growth finding names the part that grew, each VM's heap and the held counts, at the first hour, the trip and now", () => {
52 const now = Date.now();
53 const start = now - 3 * FIRST_HOUR_MS;
54 const { w, trip } = grownWatch(start);
55 app.db
56 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('memory-growth', ?1, '/system', ?2, NULL)")
57 .run("The forge's own memory is 4200 MiB.", now - 30 * MIN);
58
59 const found = degradedCauses(app.db, now, undefined, dir, null, w).find((f) => f.key === "degraded:memory-growth")!;
60 expect(found).toBeDefined();
61 const lines = found.evidence;
62 const text = lines.join("\n");
63 expect(text).toContain("memory grew most: worker pages's JS heap, 100 MiB to 2410 MiB");
64 expect(text).toContain(`memory rss: first hour 2000 MiB, trip 4100 MiB at ${new Date(trip).toISOString()}, now 4200 MiB`);
65 expect(text).toContain("memory heap worker pages: first hour 100 MiB, trip 2000 MiB, now 2410 MiB");
66 expect(text).toContain("memory heap main: first hour 500 MiB, trip 500 MiB, now 500 MiB");
67 // Outside every heap: RSS less main's and the fresh worker's heap.
68 expect(text).toContain("memory outside every heap (process): first hour 1400 MiB, trip 1600 MiB, now 1290 MiB");
69 expect(text).toContain("memory mimalloc committed (process): first hour 1200 MiB, trip 3000 MiB, now 3100 MiB");
70 expect(text).toContain("memory held sse: trip 400, now 900 (+500)");
71 expect(lines.filter((l) => l.startsWith("memory held ")).length).toBe(EVIDENCE_HELD);
72 expect(lines.length).toBeLessThan(40);
73 // Nothing in it is a path.
74 expect(text).not.toContain(dir);
75 });
76
77 test("another cause's finding carries no memory lines", () => {
78 const now = Date.now() + 2 * 86_400_000;
79 const { w } = grownWatch(now - 3 * FIRST_HOUR_MS);
80 app.db
81 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('disk', 'Disk is low.', '/system', ?1, NULL)")
82 .run(now - 30 * MIN);
83 const found = degradedCauses(app.db, now, undefined, dir, null, w).find((f) => f.key === "degraded:disk")!;
84 expect(found.evidence.some((l) => l.startsWith("memory "))).toBe(false);
85 });
86
87 test("a watch that never tripped says so, and lists no trip figures", () => {
88 const start = Date.now() - 2 * FIRST_HOUR_MS;
89 const w = createMemoryWatch(start);
90 for (let i = 0; i < 12; i++) w.note(1000 * MIB, 200 * MIB, start + i * 5 * MIN, { commit: 300 * MIB, held: [{ name: "sse", size: 4 }] });
91 const lines = w.evidence();
92 expect(lines[0]).toBe("memory rss: first hour 1000 MiB, trip none, now 1000 MiB");
93 expect(lines).toContain("memory held sse: trip none, now 4");
94 expect(createMemoryWatch().evidence()).toEqual([]);
95 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs