Public: anyone can read this; the crew can change it.
Sentinel: memory-growth was degraded 631 minutes today: The forge's own memory is 4544 MiB, over twice its first-hour 1448 MiB and past the 2176 MiB more its database caches m… #14896
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14891-sentinel-memory-growth-was-degraded-631 → main 9 commits fork
9 files
changed +156
−27
docs/features/memory-growth.md +1 −1
| @@ -39,6 +39,6 @@ Found on a copy of production's data with no network, production's own work on ( | ||
| 39 | 39 | - **The plan and lane caches kept answers nobody could ask for again** (`src/web/map_cache.ts`, `src/view/cache.ts`). A plan was keyed by its stamp, which moves with every run that starts or spends, and a lane by its repository's latest event; an entry past its minute was only dropped when the same key was read again or the count (64 plans, 128 lanes) pushed it out. A plan holds its repository's whole graph: the copy held ten graphs of the busiest repository (4,593 issues each) after an hour of page views, and the cache had room for 64. Now there is one plan per repository, spend visibility and day, and one lane per repository, scope and day, each replaced when its stamp moves; and every view cache drops what is past its time on the first write after `SWEEP_MS` (30 s). |
| 40 | 40 | - **A hosted forge that slept left its request counter and route timing behind** (`src/http/server.ts`), one more of each with every wake, each keeping the closed database and a flush that retried against it. They now go when the forge closes, written out first. |
| 41 | 41 | |
| 42 | A new long-lived collection names itself in `heldOf(app)` (a forge's) or `threadHeld` (the thread's), with a bound and a test of its eviction. Named today: `turn`, `requests in flight`, `event streams`, `request counters`, `route timings`, `map plans`, `map lanes`, `map layouts`, and four the crew keeps per pull or per check that nothing sweeps yet (`autonomy conflicted pulls`, `finishing said`, `checks retried`, `app links said`): each is a short string per entry, and its line says when one is worth a sweep. | |
| 42 | A new long-lived collection names itself in `heldOf(app)` (a forge's) or `threadHeld` (the thread's), with a bound and a test of its eviction. Named today: `turn`, `requests in flight`, `event streams`, `request counters`, `route timings`, `map plans`, `map lanes`, `map layouts`, `finishing said` (the newest 512 pulls, `src/crew/said_once.ts`), and three the crew keeps per pull or per check that nothing sweeps yet (`autonomy conflicted pulls`, `checks retried`, `app links said`): each is a short string per entry, and its line says when one is worth a sweep. | |
| 43 | 43 | |
| 44 | 44 | Tests: `test/app-turn.test.ts`, `test/cache-holds.test.ts`, `test/forge-sleep.test.ts`, `test/held.test.ts`. |
src/crew/finishing.ts +11 −8
| @@ -62,6 +62,7 @@ import { interruptionOf } from "./interrupted.ts"; | ||
| 62 | 62 | import { pickBuilder } from "./route.ts"; |
| 63 | 63 | import { NO_BUILDER_ON_CREW, WAITS_FOR_BUILDER_SLOT } from "./words.ts"; |
| 64 | 64 | import { threadHeld } from "../perf/held.ts"; |
| 65 | import { createSaidOnce } from "./said_once.ts"; | |
| 65 | 66 | |
| 66 | 67 | const log = createLog("crew"); |
| 67 | 68 | |
| @@ -343,9 +344,13 @@ function handOver(app: App, repo: Repo, pull: Pull, issue: Issue, files: readonl | ||
| 343 | 344 | return { pull: pull.number, did: "handed to a person", to: by.username }; |
| 344 | 345 | } |
| 345 | 346 | |
| 346 | /** The last why-not said for a pull, so a stall is said once until it changes. */ | |
| 347 | const said = new Map<string, string>(); | |
| 348 | threadHeld.add("finishing said", () => said.size); | |
| 347 | /** | |
| 348 | * The last why-not said for a pull, so a stall is said once until it changes; the newest | |
| 349 | * `SAID_MAX`. | |
| 350 | */ | |
| 351 | export const SAID_MAX = 512; | |
| 352 | const said = createSaidOnce(SAID_MAX); | |
| 353 | threadHeld.add("finishing said", said.size); | |
| 349 | 354 | |
| 350 | 355 | /** What each repository's pass last said it read, so its entry is said when that changes. */ |
| 351 | 356 | const entered = new Map<string, string>(); |
| @@ -360,8 +365,7 @@ function enter(repo: Repo, what: string): void { | ||
| 360 | 365 | function drop(repo: Repo, pull: Pull, why: string): void { |
| 361 | 366 | const key = `${repo.id}:${pull.number}`; |
| 362 | 367 | const line = `${pull.headOid}:${why}`; |
| 363 | if (said.get(key) === line) return; | |
| 364 | said.set(key, line); | |
| 368 | if (!said.fresh(key, line)) return; | |
| 365 | 369 | log.info("finishing: drop", { pull: pull.number, why, repo: `${repo.ownerName}/${repo.name}` }); |
| 366 | 370 | } |
| 367 | 371 | |
| @@ -369,7 +373,7 @@ function say(repo: Repo, pull: Pull, move: FinishMove): void { | ||
| 369 | 373 | const where = `${repo.ownerName}/${repo.name}`; |
| 370 | 374 | const key = `${repo.id}:${pull.number}`; |
| 371 | 375 | if (move.did === "queued update") { |
| 372 | said.delete(key); | |
| 376 | said.forget(key); | |
| 373 | 377 | log.info("approved pull conflicts: brought up to date first", { |
| 374 | 378 | repo: where, |
| 375 | 379 | pull: pull.number, |
| @@ -379,7 +383,6 @@ function say(repo: Repo, pull: Pull, move: FinishMove): void { | ||
| 379 | 383 | } |
| 380 | 384 | if (move.did !== "not moved") return; |
| 381 | 385 | const line = `${pull.headOid}:${move.why}`; |
| 382 | if (said.get(key) === line) return; | |
| 383 | said.set(key, line); | |
| 386 | if (!said.fresh(key, line)) return; | |
| 384 | 387 | log.info("finishing: skip", { pull: pull.number, why: move.why, repo: where }); |
| 385 | 388 | } |
src/crew/said_once.ts added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | /** | |
| 2 | * Said once: the last line logged for each key, so a repeat is said once until it | |
| 3 | * changes. Bounded in entries (#14891): past `max`, the key said longest ago goes, and at | |
| 4 | * worst its line is said once more. | |
| 5 | */ | |
| 6 | ||
| 7 | export interface SaidOnce { | |
| 8 | /** True when `line` is new for `key`; it is then remembered as the newest. */ | |
| 9 | fresh(key: string, line: string): boolean; | |
| 10 | /** Forget `key`, so its next line is said whatever it is. */ | |
| 11 | forget(key: string): void; | |
| 12 | /** How many keys are remembered now. */ | |
| 13 | size(): number; | |
| 14 | } | |
| 15 | ||
| 16 | export function createSaidOnce(max: number): SaidOnce { | |
| 17 | const said = new Map<string, string>(); | |
| 18 | return { | |
| 19 | fresh(key, line) { | |
| 20 | if (said.get(key) === line) return false; | |
| 21 | said.delete(key); | |
| 22 | said.set(key, line); | |
| 23 | if (said.size > max) said.delete(said.keys().next().value as string); | |
| 24 | return true; | |
| 25 | }, | |
| 26 | forget(key) { | |
| 27 | said.delete(key); | |
| 28 | }, | |
| 29 | size: () => said.size, | |
| 30 | }; | |
| 31 | } |
src/story/wave_last.ts +3 −3
| @@ -11,7 +11,7 @@ import { canRead, type Repo } from "../core/repos.ts"; | ||
| 11 | 11 | import type { User } from "../core/users.ts"; |
| 12 | 12 | import { repoDir } from "../git/repos.ts"; |
| 13 | 13 | import { movedForViewer } from "./wave_moved.ts"; |
| 14 | import { ViewCache } from "../view/cache.ts"; | |
| 14 | import { approxBytes, ViewCache } from "../view/cache.ts"; | |
| 15 | 15 | import { repoHeads } from "../web/map_cache.ts"; |
| 16 | 16 | import { pullRefs, readWaves, wavePath, type WaveEntry } from "./waves.ts"; |
| 17 | 17 | |
| @@ -31,7 +31,7 @@ export interface LastWave { | ||
| 31 | 31 | * the branch is an event, so a new wave reads fresh; the TTL covers the rest. |
| 32 | 32 | * Nothing about a viewer is in here, only what the files say. |
| 33 | 33 | */ |
| 34 | const waveFiles = new ViewCache({ maxEntries: 64 }); | |
| 34 | const waveFiles = new ViewCache({ maxEntries: 64, maxBytes: 8 * 1024 * 1024 }); | |
| 35 | 35 | const WAVES_TTL_MS = 60_000; |
| 36 | 36 | |
| 37 | 37 | /** Milestone number to the newest wave that moved it. */ |
| @@ -54,7 +54,7 @@ export async function lastWaves(input: { | ||
| 54 | 54 | repo.defaultBranch, |
| 55 | 55 | ) |
| 56 | 56 | ).slice(0, WAVES_READ); |
| 57 | waveFiles.set(key, entries, WAVES_TTL_MS); | |
| 57 | waveFiles.set(key, entries, WAVES_TTL_MS, approxBytes(entries)); | |
| 58 | 58 | } |
| 59 | 59 | const out = new Map<number, LastWave>(); |
| 60 | 60 | let plan: Plan | undefined; |
src/view/cache.ts
Binary file not shown.
src/web/map_cache.ts +17 −14
| @@ -11,7 +11,7 @@ | ||
| 11 | 11 | |
| 12 | 12 | import type { Database } from "bun:sqlite"; |
| 13 | 13 | import { threadHeld } from "../perf/held.ts"; |
| 14 | import { ViewCache } from "../view/cache.ts"; | |
| 14 | import { approxBytes, ViewCache } from "../view/cache.ts"; | |
| 15 | 15 | import type { Graph } from "../core/dependencies.ts"; |
| 16 | 16 | |
| 17 | 17 | /** A lane is rebuilt at least this often: covers inputs that move without an event. */ |
| @@ -19,8 +19,12 @@ export const LANE_TTL_MS = 60_000; | ||
| 19 | 19 | /** An unused layout goes after an hour; the cap is what bounds memory. */ |
| 20 | 20 | const LAYOUT_TTL_MS = 60 * 60_000; |
| 21 | 21 | |
| 22 | const lanes = new ViewCache({ maxEntries: 128 }); | |
| 23 | const layouts = new ViewCache({ maxEntries: 256 }); | |
| 22 | /** Bytes each map cache may hold: the entry counts alone never bounded memory (#14891). */ | |
| 23 | const MAP_CACHE_BYTES = 16 * 1024 * 1024; | |
| 24 | ||
| 25 | const lanes = new ViewCache({ maxEntries: 128, maxBytes: MAP_CACHE_BYTES }); | |
| 26 | const layouts = new ViewCache({ maxEntries: 256, maxBytes: MAP_CACHE_BYTES }); | |
| 27 | ||
| 24 | 28 | threadHeld.add("map lanes", () => lanes.stats().entries); |
| 25 | 29 | threadHeld.add("map layouts", () => layouts.stats().entries); |
| 26 | 30 | |
| @@ -75,11 +79,12 @@ export function cachedLane<T>( | ||
| 75 | 79 | const kept = lanes.get<{ head: number; value: T }>(k); |
| 76 | 80 | if (kept !== undefined && kept.head === key.head) return kept.value; |
| 77 | 81 | const value = build(); |
| 78 | lanes.set(k, { head: key.head, value }, LANE_TTL_MS); | |
| 82 | const entry = { head: key.head, value }; | |
| 83 | lanes.set(k, entry, LANE_TTL_MS, approxBytes(entry)); | |
| 79 | 84 | return value; |
| 80 | 85 | } |
| 81 | 86 | |
| 82 | const plans = new ViewCache({ maxEntries: 64 }); | |
| 87 | const plans = new ViewCache({ maxEntries: 64, maxBytes: MAP_CACHE_BYTES }); | |
| 83 | 88 | threadHeld.add("map plans", () => plans.stats().entries); |
| 84 | 89 | let planBuilds = 0; |
| 85 | 90 | |
| @@ -111,7 +116,8 @@ export function cachedPlan<T>( | ||
| 111 | 116 | if (kept !== undefined && kept.stamp === key.stamp) return kept.value; |
| 112 | 117 | planBuilds++; |
| 113 | 118 | const value = build(); |
| 114 | plans.set(k, { stamp: key.stamp, value }, LANE_TTL_MS); | |
| 119 | const entry = { stamp: key.stamp, value }; | |
| 120 | plans.set(k, entry, LANE_TTL_MS, approxBytes(entry)); | |
| 115 | 121 | return value; |
| 116 | 122 | } |
| 117 | 123 | |
| @@ -149,14 +155,11 @@ export function cachedLayout< | ||
| 149 | 155 | } as unknown as L; |
| 150 | 156 | } |
| 151 | 157 | const drawn = draw(graph); |
| 152 | layouts.set( | |
| 153 | key, | |
| 154 | { | |
| 155 | ...drawn, | |
| 156 | nodes: drawn.nodes.map(({ issue, ...rest }) => ({ ...rest, id: issue.id })), | |
| 157 | }, | |
| 158 | LAYOUT_TTL_MS, | |
| 159 | ); | |
| 158 | const entry = { | |
| 159 | ...drawn, | |
| 160 | nodes: drawn.nodes.map(({ issue, ...rest }) => ({ ...rest, id: issue.id })), | |
| 161 | }; | |
| 162 | layouts.set(key, entry, LAYOUT_TTL_MS, approxBytes(entry)); | |
| 160 | 163 | return drawn; |
| 161 | 164 | } |
| 162 | 165 |
test/map-cache.test.ts +47
| @@ -301,3 +301,50 @@ describe("/map's Server-Timing names its phases", () => { | ||
| 301 | 301 | }); |
| 302 | 302 | } |
| 303 | 303 | }); |
| 304 | ||
| 305 | describe("the map caches are bounded in bytes (#14891)", () => { | |
| 306 | test("a value is weighed by its text, and one that cannot be written is refused as huge", async () => { | |
| 307 | const { approxBytes } = await import("../src/view/cache.ts"); | |
| 308 | expect(approxBytes({ a: "xyz" })).toBe(JSON.stringify({ a: "xyz" }).length * 2); | |
| 309 | const loop: Record<string, unknown> = {}; | |
| 310 | loop.self = loop; | |
| 311 | expect(approxBytes(loop)).toBe(Number.MAX_SAFE_INTEGER); | |
| 312 | }); | |
| 313 | }); | |
| 314 | ||
| 315 | describe("a lane past the byte cap is not kept (#14891)", () => { | |
| 316 | test("an oversized lane is built and served, but the cache holds nothing of it", async () => { | |
| 317 | const { cachedLane, mapCacheStats } = await import("../src/web/map_cache.ts"); | |
| 318 | clearMapCache(); | |
| 319 | const huge = "x".repeat(9 * 1024 * 1024); // 18 MiB as text, over the 16 MiB cap | |
| 320 | const key = { repoId: "r-huge", head: 1, scope: "none", now: new Date() }; | |
| 321 | expect(cachedLane(key, () => huge)).toBe(huge); | |
| 322 | expect(mapCacheStats().lanes.entries).toBe(0); | |
| 323 | expect(cachedLane(key, () => "small")).toBe("small"); | |
| 324 | expect(mapCacheStats().lanes.entries).toBe(1); | |
| 325 | clearMapCache(); | |
| 326 | }); | |
| 327 | }); | |
| 328 | ||
| 329 | describe("the lanes are evicted oldest first by bytes, not count alone (#14891)", () => { | |
| 330 | test("three 6 MiB lanes, each under the 16 MiB cap, leave the oldest out of the cache", async () => { | |
| 331 | const { cachedLane, mapCacheStats } = await import("../src/web/map_cache.ts"); | |
| 332 | clearMapCache(); | |
| 333 | const now = new Date(); | |
| 334 | const lane = (repoId: string) => ({ repoId, head: 1, scope: "none", now }); | |
| 335 | const big = (id: string) => id.slice(-1).repeat(3 * 1024 * 1024); // 6 MiB as text | |
| 336 | let built = 0; | |
| 337 | for (const id of ["r-a", "r-b", "r-c"]) cachedLane(lane(id), () => (built++, big(id))); | |
| 338 | // Two fit under 16 MiB, the third pushes the oldest out. | |
| 339 | expect(mapCacheStats().lanes.entries).toBe(2); | |
| 340 | expect(mapCacheStats().lanes.bytes).toBeLessThanOrEqual(16 * 1024 * 1024); | |
| 341 | // The newest two are kept: asked again, nothing is built. | |
| 342 | cachedLane(lane("r-b"), () => (built++, "rebuilt")); | |
| 343 | cachedLane(lane("r-c"), () => (built++, "rebuilt")); | |
| 344 | expect(built).toBe(3); | |
| 345 | // The oldest is gone: asked again, it is built anew. | |
| 346 | expect(cachedLane(lane("r-a"), () => (built++, "rebuilt"))).toBe("rebuilt"); | |
| 347 | expect(built).toBe(4); | |
| 348 | clearMapCache(); | |
| 349 | }); | |
| 350 | }); |
test/module-state.baseline.txt −1
| @@ -68,7 +68,6 @@ src/crew/dispatch.ts:retriedChecks | ||
| 68 | 68 | src/crew/dispatch.ts:waitingOnMain |
| 69 | 69 | src/crew/finishing.ts:entered |
| 70 | 70 | src/crew/finishing.ts:read |
| 71 | src/crew/finishing.ts:said | |
| 72 | 71 | src/crew/governed.ts:seen |
| 73 | 72 | src/crew/handover.ts:handing |
| 74 | 73 | src/crew/hot_files.ts:standing |
test/said-once.test.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** | |
| 2 | * Finishing says each pull's stall once until it changes, and remembers only the newest | |
| 3 | * `SAID_MAX` pulls (#14891): past that, the pull said longest ago is forgotten and said | |
| 4 | * again at worst. | |
| 5 | */ | |
| 6 | import { describe, expect, test } from "bun:test"; | |
| 7 | import { SAID_MAX } from "../src/crew/finishing.ts"; | |
| 8 | import { createSaidOnce } from "../src/crew/said_once.ts"; | |
| 9 | import { threadHeld } from "../src/perf/held.ts"; | |
| 10 | ||
| 11 | describe("said once (#14891)", () => { | |
| 12 | test("a repeated line is said once, a changed one again, a forgotten one again", () => { | |
| 13 | const said = createSaidOnce(4); | |
| 14 | expect(said.fresh("r:1", "h1:why")).toBe(true); | |
| 15 | expect(said.fresh("r:1", "h1:why")).toBe(false); | |
| 16 | expect(said.fresh("r:1", "h2:why")).toBe(true); | |
| 17 | said.forget("r:1"); | |
| 18 | expect(said.fresh("r:1", "h2:why")).toBe(true); | |
| 19 | }); | |
| 20 | ||
| 21 | test("more than SAID_MAX pulls keep SAID_MAX, and the oldest is the one said again", () => { | |
| 22 | expect(SAID_MAX).toBe(512); | |
| 23 | const said = createSaidOnce(SAID_MAX); | |
| 24 | for (let n = 1; n <= SAID_MAX + 1; n++) expect(said.fresh(`r:${n}`, "h:stalled")).toBe(true); | |
| 25 | expect(said.size()).toBe(SAID_MAX); | |
| 26 | // Pull 2 is still remembered; pull 1, the oldest, was dropped and is said again. | |
| 27 | expect(said.fresh("r:2", "h:stalled")).toBe(false); | |
| 28 | expect(said.fresh(`r:${SAID_MAX + 1}`, "h:stalled")).toBe(false); | |
| 29 | expect(said.fresh("r:1", "h:stalled")).toBe(true); | |
| 30 | expect(said.size()).toBe(SAID_MAX); | |
| 31 | }); | |
| 32 | ||
| 33 | test("a changed line makes its pull the newest, so it outlives older ones", () => { | |
| 34 | const said = createSaidOnce(2); | |
| 35 | said.fresh("a", "1"); | |
| 36 | said.fresh("b", "1"); | |
| 37 | said.fresh("a", "2"); // a is now newest | |
| 38 | said.fresh("c", "1"); // b goes | |
| 39 | expect(said.fresh("a", "2")).toBe(false); | |
| 40 | expect(said.fresh("b", "1")).toBe(true); | |
| 41 | }); | |
| 42 | ||
| 43 | test("finishing's count is named in the thread's held list", () => { | |
| 44 | expect(threadHeld.read().find((h) => h.name === "finishing said")?.size).toBeGreaterThanOrEqual(0); | |
| 45 | }); | |
| 46 | }); |
new above
offline