Public: anyone can read this; the crew can change it.

Sentinel: memory-growth was degraded 631 minutes today: The forge's own memory is 4544 MiB, over twice its first-hour 1448 MiB and past the 2176 MiB more its database caches m… #14896

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14891-sentinel-memory-growth-was-degraded-631 → main 9 commits fork

  • Merged. mason/14891-sentinel-memory-growth-was-degraded-631 was merged into main as 4ced5ba.

Review

Sign in to review this change.
9 files changed +156 −27
docs/features/memory-growth.md +1 −1
@@ -39,6 +39,6 @@ Found on a copy of production's data with no network, production's own work on (
3939 - **The plan and lane caches kept answers nobody could ask for again** (`src/web/map_cache.ts`, `src/view/cache.ts`). A plan was keyed by its stamp, which moves with every run that starts or spends, and a lane by its repository's latest event; an entry past its minute was only dropped when the same key was read again or the count (64 plans, 128 lanes) pushed it out. A plan holds its repository's whole graph: the copy held ten graphs of the busiest repository (4,593 issues each) after an hour of page views, and the cache had room for 64. Now there is one plan per repository, spend visibility and day, and one lane per repository, scope and day, each replaced when its stamp moves; and every view cache drops what is past its time on the first write after `SWEEP_MS` (30 s).
4040 - **A hosted forge that slept left its request counter and route timing behind** (`src/http/server.ts`), one more of each with every wake, each keeping the closed database and a flush that retried against it. They now go when the forge closes, written out first.
4141
42 A new long-lived collection names itself in `heldOf(app)` (a forge's) or `threadHeld` (the thread's), with a bound and a test of its eviction. Named today: `turn`, `requests in flight`, `event streams`, `request counters`, `route timings`, `map plans`, `map lanes`, `map layouts`, and four the crew keeps per pull or per check that nothing sweeps yet (`autonomy conflicted pulls`, `finishing said`, `checks retried`, `app links said`): each is a short string per entry, and its line says when one is worth a sweep.
42 A new long-lived collection names itself in `heldOf(app)` (a forge's) or `threadHeld` (the thread's), with a bound and a test of its eviction. Named today: `turn`, `requests in flight`, `event streams`, `request counters`, `route timings`, `map plans`, `map lanes`, `map layouts`, `finishing said` (the newest 512 pulls, `src/crew/said_once.ts`), and three the crew keeps per pull or per check that nothing sweeps yet (`autonomy conflicted pulls`, `checks retried`, `app links said`): each is a short string per entry, and its line says when one is worth a sweep.
4343
4444 Tests: `test/app-turn.test.ts`, `test/cache-holds.test.ts`, `test/forge-sleep.test.ts`, `test/held.test.ts`.
src/crew/finishing.ts +11 −8
@@ -62,6 +62,7 @@ import { interruptionOf } from "./interrupted.ts";
6262 import { pickBuilder } from "./route.ts";
6363 import { NO_BUILDER_ON_CREW, WAITS_FOR_BUILDER_SLOT } from "./words.ts";
6464 import { threadHeld } from "../perf/held.ts";
65 import { createSaidOnce } from "./said_once.ts";
6566
6667 const log = createLog("crew");
6768
@@ -343,9 +344,13 @@ function handOver(app: App, repo: Repo, pull: Pull, issue: Issue, files: readonl
343344 return { pull: pull.number, did: "handed to a person", to: by.username };
344345 }
345346
346 /** The last why-not said for a pull, so a stall is said once until it changes. */
347 const said = new Map<string, string>();
348 threadHeld.add("finishing said", () => said.size);
347 /**
348 * The last why-not said for a pull, so a stall is said once until it changes; the newest
349 * `SAID_MAX`.
350 */
351 export const SAID_MAX = 512;
352 const said = createSaidOnce(SAID_MAX);
353 threadHeld.add("finishing said", said.size);
349354
350355 /** What each repository's pass last said it read, so its entry is said when that changes. */
351356 const entered = new Map<string, string>();
@@ -360,8 +365,7 @@ function enter(repo: Repo, what: string): void {
360365 function drop(repo: Repo, pull: Pull, why: string): void {
361366 const key = `${repo.id}:${pull.number}`;
362367 const line = `${pull.headOid}:${why}`;
363 if (said.get(key) === line) return;
364 said.set(key, line);
368 if (!said.fresh(key, line)) return;
365369 log.info("finishing: drop", { pull: pull.number, why, repo: `${repo.ownerName}/${repo.name}` });
366370 }
367371
@@ -369,7 +373,7 @@ function say(repo: Repo, pull: Pull, move: FinishMove): void {
369373 const where = `${repo.ownerName}/${repo.name}`;
370374 const key = `${repo.id}:${pull.number}`;
371375 if (move.did === "queued update") {
372 said.delete(key);
376 said.forget(key);
373377 log.info("approved pull conflicts: brought up to date first", {
374378 repo: where,
375379 pull: pull.number,
@@ -379,7 +383,6 @@ function say(repo: Repo, pull: Pull, move: FinishMove): void {
379383 }
380384 if (move.did !== "not moved") return;
381385 const line = `${pull.headOid}:${move.why}`;
382 if (said.get(key) === line) return;
383 said.set(key, line);
386 if (!said.fresh(key, line)) return;
384387 log.info("finishing: skip", { pull: pull.number, why: move.why, repo: where });
385388 }
src/crew/said_once.ts added +31
@@ -0,0 +1,31 @@
1 /**
2 * Said once: the last line logged for each key, so a repeat is said once until it
3 * changes. Bounded in entries (#14891): past `max`, the key said longest ago goes, and at
4 * worst its line is said once more.
5 */
6
7 export interface SaidOnce {
8 /** True when `line` is new for `key`; it is then remembered as the newest. */
9 fresh(key: string, line: string): boolean;
10 /** Forget `key`, so its next line is said whatever it is. */
11 forget(key: string): void;
12 /** How many keys are remembered now. */
13 size(): number;
14 }
15
16 export function createSaidOnce(max: number): SaidOnce {
17 const said = new Map<string, string>();
18 return {
19 fresh(key, line) {
20 if (said.get(key) === line) return false;
21 said.delete(key);
22 said.set(key, line);
23 if (said.size > max) said.delete(said.keys().next().value as string);
24 return true;
25 },
26 forget(key) {
27 said.delete(key);
28 },
29 size: () => said.size,
30 };
31 }
src/story/wave_last.ts +3 −3
@@ -11,7 +11,7 @@ import { canRead, type Repo } from "../core/repos.ts";
1111 import type { User } from "../core/users.ts";
1212 import { repoDir } from "../git/repos.ts";
1313 import { movedForViewer } from "./wave_moved.ts";
14 import { ViewCache } from "../view/cache.ts";
14 import { approxBytes, ViewCache } from "../view/cache.ts";
1515 import { repoHeads } from "../web/map_cache.ts";
1616 import { pullRefs, readWaves, wavePath, type WaveEntry } from "./waves.ts";
1717
@@ -31,7 +31,7 @@ export interface LastWave {
3131 * the branch is an event, so a new wave reads fresh; the TTL covers the rest.
3232 * Nothing about a viewer is in here, only what the files say.
3333 */
34 const waveFiles = new ViewCache({ maxEntries: 64 });
34 const waveFiles = new ViewCache({ maxEntries: 64, maxBytes: 8 * 1024 * 1024 });
3535 const WAVES_TTL_MS = 60_000;
3636
3737 /** Milestone number to the newest wave that moved it. */
@@ -54,7 +54,7 @@ export async function lastWaves(input: {
5454 repo.defaultBranch,
5555 )
5656 ).slice(0, WAVES_READ);
57 waveFiles.set(key, entries, WAVES_TTL_MS);
57 waveFiles.set(key, entries, WAVES_TTL_MS, approxBytes(entries));
5858 }
5959 const out = new Map<number, LastWave>();
6060 let plan: Plan | undefined;
src/view/cache.ts
Binary file not shown.
src/web/map_cache.ts +17 −14
@@ -11,7 +11,7 @@
1111
1212 import type { Database } from "bun:sqlite";
1313 import { threadHeld } from "../perf/held.ts";
14 import { ViewCache } from "../view/cache.ts";
14 import { approxBytes, ViewCache } from "../view/cache.ts";
1515 import type { Graph } from "../core/dependencies.ts";
1616
1717 /** A lane is rebuilt at least this often: covers inputs that move without an event. */
@@ -19,8 +19,12 @@ export const LANE_TTL_MS = 60_000;
1919 /** An unused layout goes after an hour; the cap is what bounds memory. */
2020 const LAYOUT_TTL_MS = 60 * 60_000;
2121
22 const lanes = new ViewCache({ maxEntries: 128 });
23 const layouts = new ViewCache({ maxEntries: 256 });
22 /** Bytes each map cache may hold: the entry counts alone never bounded memory (#14891). */
23 const MAP_CACHE_BYTES = 16 * 1024 * 1024;
24
25 const lanes = new ViewCache({ maxEntries: 128, maxBytes: MAP_CACHE_BYTES });
26 const layouts = new ViewCache({ maxEntries: 256, maxBytes: MAP_CACHE_BYTES });
27
2428 threadHeld.add("map lanes", () => lanes.stats().entries);
2529 threadHeld.add("map layouts", () => layouts.stats().entries);
2630
@@ -75,11 +79,12 @@ export function cachedLane<T>(
7579 const kept = lanes.get<{ head: number; value: T }>(k);
7680 if (kept !== undefined && kept.head === key.head) return kept.value;
7781 const value = build();
78 lanes.set(k, { head: key.head, value }, LANE_TTL_MS);
82 const entry = { head: key.head, value };
83 lanes.set(k, entry, LANE_TTL_MS, approxBytes(entry));
7984 return value;
8085 }
8186
82 const plans = new ViewCache({ maxEntries: 64 });
87 const plans = new ViewCache({ maxEntries: 64, maxBytes: MAP_CACHE_BYTES });
8388 threadHeld.add("map plans", () => plans.stats().entries);
8489 let planBuilds = 0;
8590
@@ -111,7 +116,8 @@ export function cachedPlan<T>(
111116 if (kept !== undefined && kept.stamp === key.stamp) return kept.value;
112117 planBuilds++;
113118 const value = build();
114 plans.set(k, { stamp: key.stamp, value }, LANE_TTL_MS);
119 const entry = { stamp: key.stamp, value };
120 plans.set(k, entry, LANE_TTL_MS, approxBytes(entry));
115121 return value;
116122 }
117123
@@ -149,14 +155,11 @@ export function cachedLayout<
149155 } as unknown as L;
150156 }
151157 const drawn = draw(graph);
152 layouts.set(
153 key,
154 {
155 ...drawn,
156 nodes: drawn.nodes.map(({ issue, ...rest }) => ({ ...rest, id: issue.id })),
157 },
158 LAYOUT_TTL_MS,
159 );
158 const entry = {
159 ...drawn,
160 nodes: drawn.nodes.map(({ issue, ...rest }) => ({ ...rest, id: issue.id })),
161 };
162 layouts.set(key, entry, LAYOUT_TTL_MS, approxBytes(entry));
160163 return drawn;
161164 }
162165
test/map-cache.test.ts +47
@@ -301,3 +301,50 @@ describe("/map's Server-Timing names its phases", () => {
301301 });
302302 }
303303 });
304
305 describe("the map caches are bounded in bytes (#14891)", () => {
306 test("a value is weighed by its text, and one that cannot be written is refused as huge", async () => {
307 const { approxBytes } = await import("../src/view/cache.ts");
308 expect(approxBytes({ a: "xyz" })).toBe(JSON.stringify({ a: "xyz" }).length * 2);
309 const loop: Record<string, unknown> = {};
310 loop.self = loop;
311 expect(approxBytes(loop)).toBe(Number.MAX_SAFE_INTEGER);
312 });
313 });
314
315 describe("a lane past the byte cap is not kept (#14891)", () => {
316 test("an oversized lane is built and served, but the cache holds nothing of it", async () => {
317 const { cachedLane, mapCacheStats } = await import("../src/web/map_cache.ts");
318 clearMapCache();
319 const huge = "x".repeat(9 * 1024 * 1024); // 18 MiB as text, over the 16 MiB cap
320 const key = { repoId: "r-huge", head: 1, scope: "none", now: new Date() };
321 expect(cachedLane(key, () => huge)).toBe(huge);
322 expect(mapCacheStats().lanes.entries).toBe(0);
323 expect(cachedLane(key, () => "small")).toBe("small");
324 expect(mapCacheStats().lanes.entries).toBe(1);
325 clearMapCache();
326 });
327 });
328
329 describe("the lanes are evicted oldest first by bytes, not count alone (#14891)", () => {
330 test("three 6 MiB lanes, each under the 16 MiB cap, leave the oldest out of the cache", async () => {
331 const { cachedLane, mapCacheStats } = await import("../src/web/map_cache.ts");
332 clearMapCache();
333 const now = new Date();
334 const lane = (repoId: string) => ({ repoId, head: 1, scope: "none", now });
335 const big = (id: string) => id.slice(-1).repeat(3 * 1024 * 1024); // 6 MiB as text
336 let built = 0;
337 for (const id of ["r-a", "r-b", "r-c"]) cachedLane(lane(id), () => (built++, big(id)));
338 // Two fit under 16 MiB, the third pushes the oldest out.
339 expect(mapCacheStats().lanes.entries).toBe(2);
340 expect(mapCacheStats().lanes.bytes).toBeLessThanOrEqual(16 * 1024 * 1024);
341 // The newest two are kept: asked again, nothing is built.
342 cachedLane(lane("r-b"), () => (built++, "rebuilt"));
343 cachedLane(lane("r-c"), () => (built++, "rebuilt"));
344 expect(built).toBe(3);
345 // The oldest is gone: asked again, it is built anew.
346 expect(cachedLane(lane("r-a"), () => (built++, "rebuilt"))).toBe("rebuilt");
347 expect(built).toBe(4);
348 clearMapCache();
349 });
350 });
test/module-state.baseline.txt −1
@@ -68,7 +68,6 @@ src/crew/dispatch.ts:retriedChecks
6868 src/crew/dispatch.ts:waitingOnMain
6969 src/crew/finishing.ts:entered
7070 src/crew/finishing.ts:read
71 src/crew/finishing.ts:said
7271 src/crew/governed.ts:seen
7372 src/crew/handover.ts:handing
7473 src/crew/hot_files.ts:standing
test/said-once.test.ts added +46
@@ -0,0 +1,46 @@
1 /**
2 * Finishing says each pull's stall once until it changes, and remembers only the newest
3 * `SAID_MAX` pulls (#14891): past that, the pull said longest ago is forgotten and said
4 * again at worst.
5 */
6 import { describe, expect, test } from "bun:test";
7 import { SAID_MAX } from "../src/crew/finishing.ts";
8 import { createSaidOnce } from "../src/crew/said_once.ts";
9 import { threadHeld } from "../src/perf/held.ts";
10
11 describe("said once (#14891)", () => {
12 test("a repeated line is said once, a changed one again, a forgotten one again", () => {
13 const said = createSaidOnce(4);
14 expect(said.fresh("r:1", "h1:why")).toBe(true);
15 expect(said.fresh("r:1", "h1:why")).toBe(false);
16 expect(said.fresh("r:1", "h2:why")).toBe(true);
17 said.forget("r:1");
18 expect(said.fresh("r:1", "h2:why")).toBe(true);
19 });
20
21 test("more than SAID_MAX pulls keep SAID_MAX, and the oldest is the one said again", () => {
22 expect(SAID_MAX).toBe(512);
23 const said = createSaidOnce(SAID_MAX);
24 for (let n = 1; n <= SAID_MAX + 1; n++) expect(said.fresh(`r:${n}`, "h:stalled")).toBe(true);
25 expect(said.size()).toBe(SAID_MAX);
26 // Pull 2 is still remembered; pull 1, the oldest, was dropped and is said again.
27 expect(said.fresh("r:2", "h:stalled")).toBe(false);
28 expect(said.fresh(`r:${SAID_MAX + 1}`, "h:stalled")).toBe(false);
29 expect(said.fresh("r:1", "h:stalled")).toBe(true);
30 expect(said.size()).toBe(SAID_MAX);
31 });
32
33 test("a changed line makes its pull the newest, so it outlives older ones", () => {
34 const said = createSaidOnce(2);
35 said.fresh("a", "1");
36 said.fresh("b", "1");
37 said.fresh("a", "2"); // a is now newest
38 said.fresh("c", "1"); // b goes
39 expect(said.fresh("a", "2")).toBe(false);
40 expect(said.fresh("b", "1")).toBe(true);
41 });
42
43 test("finishing's count is named in the thread's held list", () => {
44 expect(threadHeld.read().find((h) => h.name === "finishing said")?.size).toBeGreaterThanOrEqual(0);
45 });
46 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs