Public: anyone can read this; the crew can change it.

Sentinel: memory-growth was degraded 854 minutes today: The forge's own memory is 3749 MiB, over twice its first-hour 1448 MiB and past the 2176 MiB more its database caches m… #14746

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14745-sentinel-memory-growth-was-degraded-854 → main 1 commit fork

  • Merged. mason/14745-sentinel-memory-growth-was-degraded-854 was merged into main as 593fb67.

Review

Sign in to review this change.
2 files changed +23 −6
src/perf/memory_watch.ts +9 −3
@@ -33,8 +33,8 @@
3333 * and its map as it is read (`kernel/db_ceiling.ts`), about 2.1 GiB on production's database,
3434 * most of it after the first hour. Each VM reads its own handles' ceiling with its reading (a
3535 * worker's crosses on the channel), and `grown` speaks only past twice the first hour *and*
36 * past the first hour plus every fresh VM's ceiling. A leak still raises the need; a warm cache
37 * at its ceiling, which levels off where a leak keeps climbing, does not.
36 * past the first hour plus every fresh VM's ceiling plus `BURST_SLACK`. A leak still raises the
37 * need; a warm cache at its ceiling, which levels off where a leak keeps climbing, does not.
3838 */
3939 import { heapSize, memoryUsage as jscMemory } from "bun:jsc";
4040 import { createLog } from "../kernel/log.ts";
@@ -48,6 +48,12 @@ export const MIN_SAMPLES = 10;
4848 export const GROWN_RATIO = 2;
4949 /** RSS above this share of the baseline counts as growing, for the time it started. */
5050 const GROWING_RATIO = 1.25;
51 /**
52 * What a busy minute may hold past the baseline and the caches' ceiling without being growth:
53 * requests in flight, buffers not yet returned, a collector not yet run. The 2026-11-20 trip was
54 * 125 MiB past the line with 29 requests in flight, and RSS fell back by 1.3 GiB with 11.
55 */
56 export const BURST_SLACK = 256 * MIB;
5157
5258 const mib = (bytes: number): string => `${(bytes / MIB).toFixed(bytes < 100 * MIB ? 1 : 0)} MiB`;
5359
@@ -269,7 +275,7 @@ export function createMemoryWatch(startedAt: number = Date.now()): MemoryWatch {
269275 let ceiling = main.ceiling;
270276 for (const w of workers.values()) if (main.at - w.at <= WORKER_STALE_MS) ceiling += w.ceiling;
271277 // A cache filling to its ceiling is the design, not a leak: only what is past it is growth.
272 if (latest <= base + ceiling) return undefined;
278 if (latest <= base + ceiling + BURST_SLACK) return undefined;
273279 const grew = mostGrown([["main", main], ...workers], main.at);
274280 return {
275281 baselineBytes: base, rssBytes: latest, since: above,
test/memory-watch.test.ts +14 −3
@@ -1,7 +1,7 @@
11 import { expect, test } from "bun:test";
22 import {
33 createMemoryWatch, grownSentence, hearWorkers, memoryLine, tellReading, workerMemoryLine, workerReadingOf,
4 FIRST_HOUR_MS, MAX_OWNERS, MEMORY_CHANNEL, WORKER_STALE_MS, type WorkerReading,
4 BURST_SLACK, FIRST_HOUR_MS, MAX_OWNERS, MEMORY_CHANNEL, WORKER_STALE_MS, type WorkerReading,
55 } from "../src/perf/memory_watch.ts";
66 import { readNow } from "../src/perf/memory_watch.ts";
77
@@ -241,16 +241,27 @@ test("a database cache filling to its ceiling is not growth: production's 3290 M
241241 expect(w.grown()).toBeUndefined();
242242 });
243243
244 test("a burst within the slack past the line is not growth; past the slack it is", () => {
245 const w = createMemoryWatch(T0);
246 warmHour(w);
247 const at = HOUR + 60_000;
248 w.noteWorker({ name: "engine", heap: 200 * MIB, ceiling: ENGINE }, at);
249 w.note(3720 * MIB, 400 * MIB, at, { ceiling: SERVING }); // 124 MiB past 1420 + 2176
250 expect(w.grown()).toBeUndefined();
251 w.note(3600 * MIB + BURST_SLACK + MIB, 400 * MIB, at + 60_000, { ceiling: SERVING });
252 expect(w.grown()).toBeDefined();
253 });
254
244255 test("past the first hour plus every database's ceiling, the need speaks and names the ceiling", () => {
245256 const w = createMemoryWatch(T0);
246257 warmHour(w);
247258 const at = HOUR + 60_000;
248259 w.noteWorker({ name: "engine", heap: 200 * MIB, ceiling: ENGINE }, at);
249 w.note(3700 * MIB, 400 * MIB, at, { ceiling: SERVING });
260 w.note(3900 * MIB, 400 * MIB, at, { ceiling: SERVING });
250261 const g = w.grown()!;
251262 expect(g.ceilingBytes).toBe(2176 * MIB);
252263 expect(grownSentence(g)).toStartWith(
253 "The forge's own memory is 3700 MiB, over twice its first-hour 1420 MiB and past the 2176 MiB more its database caches may fill; ",
264 "The forge's own memory is 3900 MiB, over twice its first-hour 1420 MiB and past the 2176 MiB more its database caches may fill; ",
254265 );
255266 });
256267

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs