Public: anyone can read this; the crew can change it.
Sentinel: memory-growth was degraded 854 minutes today: The forge's own memory is 3749 MiB, over twice its first-hour 1448 MiB and past the 2176 MiB more its database caches m… #14746
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14745-sentinel-memory-growth-was-degraded-854 → main 1 commit fork
2 files
changed +23
−6
src/perf/memory_watch.ts +9 −3
| @@ -33,8 +33,8 @@ | ||
| 33 | 33 | * and its map as it is read (`kernel/db_ceiling.ts`), about 2.1 GiB on production's database, |
| 34 | 34 | * most of it after the first hour. Each VM reads its own handles' ceiling with its reading (a |
| 35 | 35 | * worker's crosses on the channel), and `grown` speaks only past twice the first hour *and* |
| 36 | * past the first hour plus every fresh VM's ceiling. A leak still raises the need; a warm cache | |
| 37 | * at its ceiling, which levels off where a leak keeps climbing, does not. | |
| 36 | * past the first hour plus every fresh VM's ceiling plus `BURST_SLACK`. A leak still raises the | |
| 37 | * need; a warm cache at its ceiling, which levels off where a leak keeps climbing, does not. | |
| 38 | 38 | */ |
| 39 | 39 | import { heapSize, memoryUsage as jscMemory } from "bun:jsc"; |
| 40 | 40 | import { createLog } from "../kernel/log.ts"; |
| @@ -48,6 +48,12 @@ export const MIN_SAMPLES = 10; | ||
| 48 | 48 | export const GROWN_RATIO = 2; |
| 49 | 49 | /** RSS above this share of the baseline counts as growing, for the time it started. */ |
| 50 | 50 | const GROWING_RATIO = 1.25; |
| 51 | /** | |
| 52 | * What a busy minute may hold past the baseline and the caches' ceiling without being growth: | |
| 53 | * requests in flight, buffers not yet returned, a collector not yet run. The 2026-11-20 trip was | |
| 54 | * 125 MiB past the line with 29 requests in flight, and RSS fell back by 1.3 GiB with 11. | |
| 55 | */ | |
| 56 | export const BURST_SLACK = 256 * MIB; | |
| 51 | 57 | |
| 52 | 58 | const mib = (bytes: number): string => `${(bytes / MIB).toFixed(bytes < 100 * MIB ? 1 : 0)} MiB`; |
| 53 | 59 | |
| @@ -269,7 +275,7 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch { | ||
| 269 | 275 | let ceiling = main.ceiling; |
| 270 | 276 | for (const w of workers.values()) if (main.at - w.at <= WORKER_STALE_MS) ceiling += w.ceiling; |
| 271 | 277 | // A cache filling to its ceiling is the design, not a leak: only what is past it is growth. |
| 272 | if (latest <= base + ceiling) return undefined; | |
| 278 | if (latest <= base + ceiling + BURST_SLACK) return undefined; | |
| 273 | 279 | const grew = mostGrown([["main", main], ...workers], main.at); |
| 274 | 280 | return { |
| 275 | 281 | baselineBytes: base, rssBytes: latest, since: above, |
test/memory-watch.test.ts +14 −3
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | import { expect, test } from "bun:test"; |
| 2 | 2 | import { |
| 3 | 3 | createMemoryWatch, grownSentence, hearWorkers, memoryLine, tellReading, workerMemoryLine, workerReadingOf, |
| 4 | FIRST_HOUR_MS, MAX_OWNERS, MEMORY_CHANNEL, WORKER_STALE_MS, type WorkerReading, | |
| 4 | BURST_SLACK, FIRST_HOUR_MS, MAX_OWNERS, MEMORY_CHANNEL, WORKER_STALE_MS, type WorkerReading, | |
| 5 | 5 | } from "../src/perf/memory_watch.ts"; |
| 6 | 6 | import { readNow } from "../src/perf/memory_watch.ts"; |
| 7 | 7 | |
| @@ -241,16 +241,27 @@ test("a database cache filling to its ceiling is not growth: production's 3290 M | ||
| 241 | 241 | expect(w.grown()).toBeUndefined(); |
| 242 | 242 | }); |
| 243 | 243 | |
| 244 | test("a burst within the slack past the line is not growth; past the slack it is", () => { | |
| 245 | const w = createMemoryWatch(T0); | |
| 246 | warmHour(w); | |
| 247 | const at = HOUR + 60_000; | |
| 248 | w.noteWorker({ name: "engine", heap: 200 * MIB, ceiling: ENGINE }, at); | |
| 249 | w.note(3720 * MIB, 400 * MIB, at, { ceiling: SERVING }); // 124 MiB past 1420 + 2176 | |
| 250 | expect(w.grown()).toBeUndefined(); | |
| 251 | w.note(3600 * MIB + BURST_SLACK + MIB, 400 * MIB, at + 60_000, { ceiling: SERVING }); | |
| 252 | expect(w.grown()).toBeDefined(); | |
| 253 | }); | |
| 254 | ||
| 244 | 255 | test("past the first hour plus every database's ceiling, the need speaks and names the ceiling", () => { |
| 245 | 256 | const w = createMemoryWatch(T0); |
| 246 | 257 | warmHour(w); |
| 247 | 258 | const at = HOUR + 60_000; |
| 248 | 259 | w.noteWorker({ name: "engine", heap: 200 * MIB, ceiling: ENGINE }, at); |
| 249 | w.note(3700 * MIB, 400 * MIB, at, { ceiling: SERVING }); | |
| 260 | w.note(3900 * MIB, 400 * MIB, at, { ceiling: SERVING }); | |
| 250 | 261 | const g = w.grown()!; |
| 251 | 262 | expect(g.ceilingBytes).toBe(2176 * MIB); |
| 252 | 263 | expect(grownSentence(g)).toStartWith( |
| 253 | "The forge's own memory is 3700 MiB, over twice its first-hour 1420 MiB and past the 2176 MiB more its database caches may fill; ", | |
| 264 | "The forge's own memory is 3900 MiB, over twice its first-hour 1420 MiB and past the 2176 MiB more its database caches may fill; ", | |
| 254 | 265 | ); |
| 255 | 266 | }); |
| 256 | 267 |
new above
offline