Public: anyone can read this; the crew can change it.
A memory-growth finding carries its own evidence: each VM's heap, the outside-heap figure and the held-by-name counts, at the first hour, at the trip and now #14376
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14362-a-memory-growth-finding-carries-its-own → main 1 commit fork
4 files
changed +205
−10
src/boards/memory.ts +5 −2
| @@ -76,9 +76,12 @@ export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemoryS | ||
| 76 | 76 | try { |
| 77 | 77 | const reading = readNow(); |
| 78 | 78 | recordMemory(app.db, reading.rss, reading.heap, now); |
| 79 | memoryWatch.note(reading.rss, reading.heap, now); | |
| 79 | // The held counts are read once: the line writes them, the watch keeps them for a finding | |
| 80 | // (#14362). | |
| 81 | const held = heldNow(app); | |
| 82 | memoryWatch.note(reading.rss, reading.heap, now, { commit: reading.commit, held }); | |
| 80 | 83 | app.log.info(memoryLine(reading, process.uptime() * 1000)); |
| 81 | app.log.info(heldLine(heldNow(app))); | |
| 84 | app.log.info(heldLine(held)); | |
| 82 | 85 | const allocated = allocationLine(allocationsOf(app)); |
| 83 | 86 | if (allocated !== undefined) app.log.info(allocated); |
| 84 | 87 | } catch (error) { |
src/crew/sentinel.ts +12 −1
| @@ -56,6 +56,7 @@ import type { FindingKind } from "../hunt/finding.ts"; | ||
| 56 | 56 | import { addLabels, kindLabels } from "../hunt/label.ts"; |
| 57 | 57 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; |
| 58 | 58 | import { seenCause } from "./sentinel_seen.ts"; |
| 59 | import { memoryWatch, type MemoryWatch } from "../perf/memory_watch.ts"; | |
| 59 | 60 | |
| 60 | 61 | const log = createLog("sentinel"); |
| 61 | 62 | |
| @@ -411,7 +412,14 @@ export function repeatedErrors(now: number, ledger?: ErrorLedger): Finding[] { | ||
| 411 | 412 | * Each cause whose spans total more than ten minutes in the day ending at |
| 412 | 413 | * `now`, quoting its sentence. Shorter totals stay in the history. |
| 413 | 414 | */ |
| 414 | export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] { | |
| 415 | export function degradedCauses( | |
| 416 | db: Database, | |
| 417 | now: number, | |
| 418 | forgeRepoId?: string, | |
| 419 | dataDir?: string, | |
| 420 | ceilingUsd: number | null = null, | |
| 421 | watch: Pick<MemoryWatch, "evidence"> = memoryWatch, | |
| 422 | ): Finding[] { | |
| 415 | 423 | const from = now - DAY_MS; |
| 416 | 424 | const byCause = new Map< |
| 417 | 425 | string, |
| @@ -448,6 +456,9 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string, | ||
| 448 | 456 | `total: ${minutes} minutes in the day`, |
| 449 | 457 | `href: ${d.href}`, |
| 450 | 458 | ...sharedLines(db, cause, now, forgeRepoId, dataDir), |
| 459 | // /system is admin people only: the finding carries what grew, so a builder can name it | |
| 460 | // (#14362). | |
| 461 | ...(cause === "memory-growth" ? watch.evidence() : []), | |
| 451 | 462 | ], |
| 452 | 463 | pillar: "P0 Trust: the forge says when it is not well, and is well again", |
| 453 | 464 | kind: "defect", |
src/perf/memory_watch.ts +93 −7
| @@ -172,13 +172,50 @@ function mostGrown(owners: Iterable<readonly [string, Owner]>, now: number): Gro | ||
| 172 | 172 | return best; |
| 173 | 173 | } |
| 174 | 174 | |
| 175 | /** | |
| 176 | * What the serving VM's minute tick also hands the watch (#14362): mimalloc's commit for the | |
| 177 | * process and the held-by-name counts it already read for its line. Both are kept as the | |
| 178 | * latest and, at the trip, as they stood then; nothing here is read by the watch itself. | |
| 179 | */ | |
| 180 | export interface NoteExtra { | |
| 181 | readonly commit?: number; | |
| 182 | readonly held?: readonly HeldCount[]; | |
| 183 | } | |
| 184 | ||
| 185 | /** The watch as it stood at the trip: the minute RSS first reached twice its first hour. */ | |
| 186 | interface Trip { | |
| 187 | readonly at: number; | |
| 188 | readonly rss: number; | |
| 189 | readonly commit: number | undefined; | |
| 190 | readonly outside: number | undefined; | |
| 191 | /** Each fresh owner's JS heap then, by name. */ | |
| 192 | readonly heaps: ReadonlyMap<string, number>; | |
| 193 | readonly held: readonly HeldCount[]; | |
| 194 | } | |
| 195 | ||
| 196 | /** | |
| 197 | * How many held-by-name counts the evidence names, and how many workers at most (#14362: under 40 | |
| 198 | * lines). | |
| 199 | */ | |
| 200 | export const EVIDENCE_HELD = 10; | |
| 201 | export const EVIDENCE_WORKERS = 16; | |
| 202 | ||
| 175 | 203 | export interface MemoryWatch { |
| 176 | /** The serving VM's minute reading: the process's RSS and main's own heap. */ | |
| 177 | note(rss: number, heap: number, now?: number): void; | |
| 204 | /** | |
| 205 | * The serving VM's minute reading: the process's RSS and main's own heap, and what `extra` adds. | |
| 206 | */ | |
| 207 | note(rss: number, heap: number, now?: number, extra?: NoteExtra): void; | |
| 178 | 208 | /** A worker's minute reading (`hearWorkers` carries it here), against its own first hour. */ |
| 179 | 209 | noteWorker(r: WorkerReading, now?: number): void; |
| 180 | 210 | /** RSS at least twice its first-hour mean, with the part that grew most, or undefined. */ |
| 181 | 211 | grown(): Grown | undefined; |
| 212 | /** | |
| 213 | * The lines a memory-growth finding quotes (#14362): each reporting VM's JS heap and the | |
| 214 | * process's RSS outside every heap at the first-hour mean, at the trip and now; mimalloc's | |
| 215 | * commit; the largest held-by-name counts at the trip and now. Read from what the minute | |
| 216 | * tick kept: nothing is measured here. Empty before any reading. | |
| 217 | */ | |
| 218 | evidence(): string[]; | |
| 182 | 219 | /** |
| 183 | 220 | * The heap now (read here, or `heapNow` in a test) against the last minute's sample, for a |
| 184 | 221 | * stall line. |
| @@ -193,13 +230,29 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch { | ||
| 193 | 230 | let last: { heap: number; rss: number; at: number } | undefined; |
| 194 | 231 | let above: number | undefined; |
| 195 | 232 | let latest: number | undefined; |
| 233 | let commitSum = 0; | |
| 234 | let commitCount = 0; | |
| 235 | let commitNow: number | undefined; | |
| 236 | let heldNow: readonly HeldCount[] = []; | |
| 237 | let trip: Trip | undefined; | |
| 196 | 238 | const main = owner(startedAt); |
| 197 | 239 | const workers = new Map<string, Owner>(); |
| 240 | const grown = (): Grown | undefined => { | |
| 241 | if (base === undefined || latest === undefined || above === undefined || latest < base * GROWN_RATIO) return undefined; | |
| 242 | const grew = mostGrown([["main", main], ...workers], main.at); | |
| 243 | return { baselineBytes: base, rssBytes: latest, since: above, ...(grew === undefined ? {} : { grew }) }; | |
| 244 | }; | |
| 198 | 245 | return { |
| 199 | note(rss, heap, now = Date.now()) { | |
| 246 | note(rss, heap, now = Date.now(), extra = {}) { | |
| 200 | 247 | if (!Number.isFinite(rss) || rss <= 0) return; |
| 201 | 248 | last = { heap, rss, at: now }; |
| 202 | 249 | latest = rss; |
| 250 | commitNow = bytes(extra.commit) ? extra.commit : undefined; | |
| 251 | if (commitNow !== undefined && now - startedAt < FIRST_HOUR_MS) { | |
| 252 | commitSum += commitNow; | |
| 253 | commitCount++; | |
| 254 | } | |
| 255 | if (extra.held !== undefined) heldNow = extra.held.slice(); | |
| 203 | 256 | // Every fresh worker's heap is its own VM's, inside the process's RSS: not outside. |
| 204 | 257 | let others = 0; |
| 205 | 258 | for (const w of workers.values()) if (now - w.at <= WORKER_STALE_MS) others += w.heap; |
| @@ -216,6 +269,12 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch { | ||
| 216 | 269 | } |
| 217 | 270 | if (rss > base * GROWING_RATIO) above ??= now; |
| 218 | 271 | else above = undefined; |
| 272 | if (above === undefined) trip = undefined; | |
| 273 | else if (trip === undefined && rss >= base * GROWN_RATIO) { | |
| 274 | const heaps = new Map<string, number>([["main", heap]]); | |
| 275 | for (const [name, w] of workers) if (now - w.at <= WORKER_STALE_MS) heaps.set(name, w.heap); | |
| 276 | trip = { at: now, rss, commit: commitNow, outside: main.latest.outside, heaps, held: heldNow }; | |
| 277 | } | |
| 219 | 278 | }, |
| 220 | 279 | noteWorker(r, now = Date.now()) { |
| 221 | 280 | let o = workers.get(r.name); |
| @@ -227,10 +286,37 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch { | ||
| 227 | 286 | o.heap = r.heap; |
| 228 | 287 | take(o, { heap: r.heap }, now); |
| 229 | 288 | }, |
| 230 | grown() { | |
| 231 | if (base === undefined || latest === undefined || above === undefined || latest < base * GROWN_RATIO) return undefined; | |
| 232 | const grew = mostGrown([["main", main], ...workers], main.at); | |
| 233 | return { baselineBytes: base, rssBytes: latest, since: above, ...(grew === undefined ? {} : { grew }) }; | |
| 289 | grown, | |
| 290 | evidence() { | |
| 291 | if (latest === undefined) return []; | |
| 292 | const at = main.at; | |
| 293 | const first = (o: Owner, p: Part): number | undefined => | |
| 294 | o.base !== undefined ? o.base[p] : o.count[p] > 0 ? o.sum[p] / o.count[p] : undefined; | |
| 295 | const fig = (n: number | undefined): string => (n === undefined ? "none" : mib(n)); | |
| 296 | const three = (a: number | undefined, t: number | undefined, n: number | undefined): string => | |
| 297 | `first hour ${fig(a)}, trip ${trip === undefined ? "none" : fig(t)}, now ${fig(n)}`; | |
| 298 | const out = [ | |
| 299 | `memory rss: first hour ${fig(base ?? (count > 0 ? sum / count : undefined))}, trip ${trip === undefined ? "none" : `${mib(trip.rss)} at ${new Date(trip.at).toISOString()}`}, now ${mib(latest)}`, | |
| 300 | ]; | |
| 301 | const g = grown(); | |
| 302 | if (g?.grew !== undefined) out.push(`memory grew most: ${grewWords(g.grew)}, ${mib(g.grew.fromBytes)} to ${mib(g.grew.toBytes)}`); | |
| 303 | out.push(`memory heap main: ${three(first(main, "heap"), trip?.heaps.get("main"), main.latest.heap)}`); | |
| 304 | const fresh = [...workers].filter(([, w]) => at - w.at <= WORKER_STALE_MS).sort(([a], [b]) => (a < b ? -1 : 1)); | |
| 305 | for (const [name, w] of fresh.slice(0, EVIDENCE_WORKERS)) | |
| 306 | out.push(`memory heap worker ${name}: ${three(first(w, "heap"), trip?.heaps.get(name), w.latest.heap)}`); | |
| 307 | if (fresh.length > EVIDENCE_WORKERS) out.push(`memory heap: ${fresh.length - EVIDENCE_WORKERS} more workers not listed`); | |
| 308 | out.push(`memory outside every heap (process): ${three(first(main, "outside"), trip?.outside, main.latest.outside)}`); | |
| 309 | out.push(`memory mimalloc committed (process): ${three(commitCount > 0 ? commitSum / commitCount : undefined, trip?.commit, commitNow)}`); | |
| 310 | const then = new Map((trip?.held ?? []).map((c) => [c.name, c.size])); | |
| 311 | const named = heldNow.filter((c) => c.size !== 0).slice(0, EVIDENCE_HELD); | |
| 312 | if (named.length === 0) out.push("memory held: nothing held by name now"); | |
| 313 | for (const c of named) { | |
| 314 | const was = then.get(c.name); | |
| 315 | const count = (n: number): string => (n < 0 ? "unread" : String(n)); | |
| 316 | const growth = was === undefined || was < 0 || c.size < 0 ? "" : ` (${c.size - was >= 0 ? "+" : ""}${c.size - was})`; | |
| 317 | out.push(`memory held ${c.name}: trip ${trip === undefined || was === undefined ? "none" : count(was)}, now ${count(c.size)}${growth}`); | |
| 318 | } | |
| 319 | return out; | |
| 234 | 320 | }, |
| 235 | 321 | heapAcross(now = Date.now(), heapNow = heapSize()) { |
| 236 | 322 | if (last === undefined) return `heap ${mib(heapNow)}`; |
test/sentinel-memory-evidence.test.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | /** | |
| 2 | * A memory-growth finding carries its own evidence (#14362): /system is admin people only, | |
| 3 | * so the issue a builder is handed must name what grew, with its figures, from the watch. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import "./helpers/globals.ts"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { degradedCauses } from "../src/crew/sentinel.ts"; | |
| 12 | import { createMemoryWatch, EVIDENCE_HELD, FIRST_HOUR_MS } from "../src/perf/memory_watch.ts"; | |
| 13 | ||
| 14 | const MIB = 1048576; | |
| 15 | const MIN = 60_000; | |
| 16 | let dir: string; | |
| 17 | let app: App; | |
| 18 | ||
| 19 | beforeAll(async () => { | |
| 20 | dir = await mkdtemp(join(tmpdir(), "forge-sentinel-memory-")); | |
| 21 | app = createApp({ dataDir: dir, port: 0 }); | |
| 22 | }); | |
| 23 | afterAll(async () => { | |
| 24 | app.db.close(); | |
| 25 | await rm(dir, { recursive: true, force: true }); | |
| 26 | }); | |
| 27 | ||
| 28 | /** A watch whose worker `pages` grew from 100 MiB to 2410 MiB, with `sse` held growing. */ | |
| 29 | function grownWatch(start: number) { | |
| 30 | const w = createMemoryWatch(start); | |
| 31 | const held = (sse: number) => [ | |
| 32 | { name: "sse", size: sse }, | |
| 33 | { name: "turn", size: 3 }, | |
| 34 | ...Array.from({ length: 12 }, (_, i) => ({ name: `h${i}`, size: 1 })), | |
| 35 | ]; | |
| 36 | for (let i = 0; i < 12; i++) { | |
| 37 | w.noteWorker({ name: "pages", heap: 100 * MIB }, start + i * 5 * MIN); | |
| 38 | w.note(2000 * MIB, 500 * MIB, start + i * 5 * MIN, { commit: 1200 * MIB, held: held(10) }); | |
| 39 | } | |
| 40 | // The hour is over: the baseline is struck, then the trip (twice the first hour), then now. | |
| 41 | const trip = start + FIRST_HOUR_MS + MIN; | |
| 42 | w.noteWorker({ name: "pages", heap: 100 * MIB }, start + FIRST_HOUR_MS); | |
| 43 | w.note(2000 * MIB, 500 * MIB, start + FIRST_HOUR_MS, { commit: 1200 * MIB, held: held(10) }); | |
| 44 | w.noteWorker({ name: "pages", heap: 2000 * MIB }, trip); | |
| 45 | w.note(4100 * MIB, 500 * MIB, trip, { commit: 3000 * MIB, held: held(400) }); | |
| 46 | w.noteWorker({ name: "pages", heap: 2410 * MIB }, trip + MIN); | |
| 47 | w.note(4200 * MIB, 500 * MIB, trip + MIN, { commit: 3100 * MIB, held: held(900) }); | |
| 48 | return { w, trip }; | |
| 49 | } | |
| 50 | ||
| 51 | test("a memory-growth finding names the part that grew, each VM's heap and the held counts, at the first hour, the trip and now", () => { | |
| 52 | const now = Date.now(); | |
| 53 | const start = now - 3 * FIRST_HOUR_MS; | |
| 54 | const { w, trip } = grownWatch(start); | |
| 55 | app.db | |
| 56 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('memory-growth', ?1, '/system', ?2, NULL)") | |
| 57 | .run("The forge's own memory is 4200 MiB.", now - 30 * MIN); | |
| 58 | ||
| 59 | const found = degradedCauses(app.db, now, undefined, dir, null, w).find((f) => f.key === "degraded:memory-growth")!; | |
| 60 | expect(found).toBeDefined(); | |
| 61 | const lines = found.evidence; | |
| 62 | const text = lines.join("\n"); | |
| 63 | expect(text).toContain("memory grew most: worker pages's JS heap, 100 MiB to 2410 MiB"); | |
| 64 | expect(text).toContain(`memory rss: first hour 2000 MiB, trip 4100 MiB at ${new Date(trip).toISOString()}, now 4200 MiB`); | |
| 65 | expect(text).toContain("memory heap worker pages: first hour 100 MiB, trip 2000 MiB, now 2410 MiB"); | |
| 66 | expect(text).toContain("memory heap main: first hour 500 MiB, trip 500 MiB, now 500 MiB"); | |
| 67 | // Outside every heap: RSS less main's and the fresh worker's heap. | |
| 68 | expect(text).toContain("memory outside every heap (process): first hour 1400 MiB, trip 1600 MiB, now 1290 MiB"); | |
| 69 | expect(text).toContain("memory mimalloc committed (process): first hour 1200 MiB, trip 3000 MiB, now 3100 MiB"); | |
| 70 | expect(text).toContain("memory held sse: trip 400, now 900 (+500)"); | |
| 71 | expect(lines.filter((l) => l.startsWith("memory held ")).length).toBe(EVIDENCE_HELD); | |
| 72 | expect(lines.length).toBeLessThan(40); | |
| 73 | // Nothing in it is a path. | |
| 74 | expect(text).not.toContain(dir); | |
| 75 | }); | |
| 76 | ||
| 77 | test("another cause's finding carries no memory lines", () => { | |
| 78 | const now = Date.now() + 2 * 86_400_000; | |
| 79 | const { w } = grownWatch(now - 3 * FIRST_HOUR_MS); | |
| 80 | app.db | |
| 81 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('disk', 'Disk is low.', '/system', ?1, NULL)") | |
| 82 | .run(now - 30 * MIN); | |
| 83 | const found = degradedCauses(app.db, now, undefined, dir, null, w).find((f) => f.key === "degraded:disk")!; | |
| 84 | expect(found.evidence.some((l) => l.startsWith("memory "))).toBe(false); | |
| 85 | }); | |
| 86 | ||
| 87 | test("a watch that never tripped says so, and lists no trip figures", () => { | |
| 88 | const start = Date.now() - 2 * FIRST_HOUR_MS; | |
| 89 | const w = createMemoryWatch(start); | |
| 90 | for (let i = 0; i < 12; i++) w.note(1000 * MIB, 200 * MIB, start + i * 5 * MIN, { commit: 300 * MIB, held: [{ name: "sse", size: 4 }] }); | |
| 91 | const lines = w.evidence(); | |
| 92 | expect(lines[0]).toBe("memory rss: first hour 1000 MiB, trip none, now 1000 MiB"); | |
| 93 | expect(lines).toContain("memory held sse: trip none, now 4"); | |
| 94 | expect(createMemoryWatch().evidence()).toEqual([]); | |
| 95 | }); |
new above
offline