Public: anyone can read this; the crew can change it.

An admin takes a heap census on the running forge from /system, and the forge compares it with the first one by object type #15311

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15309-an-admin-takes-a-heap-census-on-the-runn → main 4 commits fork 3 threads · 3 unresolved

  • Merged. scout/15309-an-admin-takes-a-heap-census-on-the-runn was merged into main as 77ccf03.

Review

Sign in to review this change.
14 files changed +429 −13
docs/design/attention-budget.json +5 −5
@@ -175,16 +175,16 @@
175175 },
176176 "system": {
177177 "390": {
178 "words": 250,
179 "blocks": 79,
178 "words": 256,
179 "blocks": 81,
180180 "colours": 0
181181 },
182182 "1280": {
183 "words": 250,
184 "blocks": 79,
183 "words": 256,
184 "blocks": 81,
185185 "colours": 0
186186 },
187 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click. Words +16, blocks +2 (#15073): the least-used-routes section's heading and its one-sentence count stay on first view (how many route groups had no request in 30 days); the table sits behind the disclosure."
187 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click. Words +16, blocks +2 (#15073): the least-used-routes section's heading and its one-sentence count stay on first view (how many route groups had no request in 30 days); the table sits behind the disclosure. Words +6, blocks +2 (#15309): the heap census section's heading and its one-sentence state (\"No heap census yet.\" on an empty forge) stay on first view; the button, what a census costs and the table of types that grew sit behind the disclosure."
188188 },
189189 "explore": {
190190 "note": "Measured as an admin: the unfiltered count reads 'N apps, forges and people' (+3 words over 'N results'); a guest reads two fewer. An app row says forks by people (+1 word, #14323).",
docs/features/memory-growth.md +11
@@ -48,3 +48,14 @@ A new long-lived collection names itself in `heldOf(app)` (a forge's) or `thread
4848 **Not yet measured.** The hour on production's copy has not been taken: the run that built this tool could not read production's data. Until it is, the retainers of the main heap's 251 to 1057 MiB (#15210) are not named, and the three fixes above are the ones the earlier hour found. A short boot-plus-1,500-requests run on an empty forge works end to end and held nothing growing beyond collectable strings. Whoever takes the hour should paste the `heap-growth` output on #15286, name the largest holder and its module here, and bound it with a test of its eviction; if the growth is outside the heap (`extraMemorySize` or RSS rising while `heapSize` is flat), open a follow-up naming the native owner.
4949
5050 Tests: `test/heap-diff.test.ts`, `test/app-turn.test.ts`, `test/cache-holds.test.ts`, `test/forge-sleep.test.ts`, `test/held.test.ts`.
51
52 ## A heap census on the running forge (#15309)
53
54 To find what grows on production itself, with no copy of its data: on `/system`, under *Forge memory*, a forge admin who is a person presses **Take a heap census**. One `heapStats()` (bun:jsc) of the main VM counts every object on the heap by type. It holds the forge for a fraction of a second on a large heap (~280 ms on 2M objects), so it runs only on that press, never on a timer, and at most once every ten minutes; a second press sooner is answered `429` with `Retry-After` and the sentence why.
55
56 - The first census is the baseline. Press again an hour or more later; from the second census on, `/system` lists the twenty object types that grew most from the first to the last, each with its count then and now. A type that only grows from press to press is the retainer to look for.
57 - Each census is a row of `heap_census` in the forge's database (`{at, uptimeMin, heapSize, objectCount, byType}`): the first and the last twenty are kept, 21 rows at most.
58 - The crew reads the same data as JSON, `GET /api/v1/telemetry/heap-census` (the owner, or an agent holding `telemetry:read`): `{ censuses: [{at, uptimeMin, heapSize, objectCount}], grew: [{type, then, now}] }`. Reading it never takes a census. An agent's token asking to take one is refused `403` with one sentence.
59 - In the terminal, the memory screen (`src/tui/screens/system.ts`, `censusLines`) shows the same count and comparison, and `c` there takes a census (`takeCensus`): the same gate, the same ten-minute limit and the same sentence on refusal, as a toast, into the same table the page reads.
60
61 Built in `src/perf/heap_census.ts`; tests: `test/heap-census.test.ts`, `test/tui-system.test.ts`.
src/contract/openapi/routes/getApiV1TelemetryHeapCensus.ts added +11
@@ -0,0 +1,11 @@
1 import type { Route } from "../shared.ts";
2
3 export const route: Route = {
4 method: "get",
5 path: "/api/v1/telemetry/heap-census",
6 summary: "Read the heap censuses an admin took on /system",
7 detail: "Read the heap censuses an admin took on /system (#15309): each census's time, uptime, heap bytes and object count, oldest first, and the twenty object types that grew most from the first census to the last; the owner or an agent holding telemetry:read, with a ro token; reading never takes a census",
8 auth: true,
9 role: "member",
10 refuses: [401, 403],
11 };
src/contract/openapi/routes/index.ts +1
@@ -219,6 +219,7 @@ export { route as getApiV1RunnerJobsIdGitInfoRefs } from "./getApiV1RunnerJobsId
219219 export { route as getApiV1Search } from "./getApiV1Search.ts";
220220 export { route as getApiV1SecretsScope } from "./getApiV1SecretsScope.ts";
221221 export { route as getApiV1TelemetryFlow } from "./getApiV1TelemetryFlow.ts";
222 export { route as getApiV1TelemetryHeapCensus } from "./getApiV1TelemetryHeapCensus.ts";
222223 export { route as getApiV1TelemetryHistory } from "./getApiV1TelemetryHistory.ts";
223224 export { route as getApiV1TelemetryHome } from "./getApiV1TelemetryHome.ts";
224225 export { route as getApiV1TelemetryRoutes } from "./getApiV1TelemetryRoutes.ts";
src/core/migration-pins/20261121230000-heap-census.pin added +1
@@ -0,0 +1,1 @@
1 861b90fbcb21754dfecf92cd3e5d40f24a24ee3b648d809a9ce1d41018829b61
src/core/migrations/20261121230000-heap-census.ts added +13
@@ -0,0 +1,13 @@
1 // A heap census of the main VM, taken only when an admin asks (#15309, perf/heap_census.ts): the
2 // object count of each type, as JSON. The first and the last twenty are kept; telemetry, not
3 // state — a lost row is a census to take again.
4 export default `
5 CREATE TABLE heap_census (
6 id INTEGER PRIMARY KEY AUTOINCREMENT,
7 at INTEGER NOT NULL,
8 uptime_min INTEGER NOT NULL,
9 heap_size REAL NOT NULL,
10 object_count INTEGER NOT NULL,
11 by_type TEXT NOT NULL
12 );
13 `;
src/http/api.ts +3
@@ -449,6 +449,7 @@ import { forgeMetrics } from "../metrics/forge.ts";
449449 import { appSources } from "../metrics/app_sources.ts";
450450 import { ADMIN_WORDS } from "../actions/admin_words.ts";
451451 import { ANSWER_WORDS, ORDER_WORDS, VISIBILITY_WORDS } from "../core/arg_words.ts";
452 import { censusReport } from "../perf/heap_census.ts";
452453
453454 /** Take the parent's updates for `repo` and answer as the API does, for every route that asks. */
454455 export async function respondTake(ctx: Ctx, repo: Repo): Promise<Response> {
@@ -1392,6 +1393,8 @@ export function apiRoutes(router: Router<Handler>, isPage: (path: string) => boo
13921393 noneMatch(sent, etag) ? new Response(null, { status: 304, headers }) : json(copy.body, 200, headers);
13931394 }));
13941395 router.add("GET", "/api/v1/telemetry/home", telemetryRead("home", (c) => homeTruth(c.app, c.auth.user!)));
1396 // The heap censuses an admin took on /system (#15309): read only, never a census taken here.
1397 router.add("GET", "/api/v1/telemetry/heap-census", telemetryRead("heap-census", (c) => censusReport(c.app.db)));
13951398 router.add("GET", "/api/v1/telemetry/sources", telemetryRead("sources", () => ({ sources: sourcesOut() })));
13961399 router.add("GET", "/api/v1/telemetry/history", telemetryRead("history", (c) => ({ points: historyFor(c.app) })));
13971400 // The funnel: what any agent that arrives should know. Open to all.
src/http/server.ts +2 −1
@@ -39,7 +39,7 @@ import { unsealReleased } from "../actions/sealed.ts";
3939 import { queueScanPush } from "../checks/scan.ts";
4040 import { parentCallback, parentStart } from "./rp.ts";
4141 import { prerenderComments } from "../web/prerender.ts";
42 import { systemPage, systemStream } from "../web/pages/system.ts";
42 import { heapCensusSubmit, systemPage, systemStream } from "../web/pages/system.ts";
4343 import { bottleneckPage } from "../web/pages/bottleneck.ts";
4444 import { contributePage } from "../web/pages/contribute.ts";
4545 import { pipelinePage } from "../web/pages/pipeline.ts";
@@ -1163,6 +1163,7 @@ export function buildRouter(): Router<Handler> {
11631163 router.add("POST", "/flow/suggestion", suggestionTake);
11641164 router.add("GET", "/system", (ctx) => systemPage(ctx, knownPatterns));
11651165 router.add("GET", "/system/stream", systemStream);
1166 router.add("POST", "/system/heap-census", heapCensusSubmit);
11661167 router.add("GET", "/system/bottleneck", bottleneckPage);
11671168 router.add("GET", "/offers/:id", offerReceiptPage);
11681169 router.add("GET", "/contribute", contributePage);
src/perf/heap_census.ts added +120
@@ -0,0 +1,120 @@
1 /**
2 * A heap census of the main VM, taken only when a forge admin who is a person asks (#15309):
3 * `heapStats()` (bun:jsc) counts every object on the heap by type, and holds the loop a fraction
4 * of a second on a large heap (~280 ms on 2M objects, #11118), so it never runs on a timer. At most
5 * one every ten minutes. The first census is the baseline; the last twenty are kept beside it
6 * (`heap_census`, telemetry, not state), and the comparison ranks the types that grew most from the
7 * first to the last: the retainer behind a growing heap reads by its type's name.
8 */
9 import type { Database } from "bun:sqlite";
10 import { heapStats } from "bun:jsc";
11 import { quietly } from "../core/write_clock.ts";
12
13 export const CENSUS_EVERY_MS = 10 * 60_000;
14 /** What an agent's token hears, on the page and in the terminal. */
15 export const CENSUS_AGENT_REFUSED = "A heap census is taken by a forge admin who is a person, not by an agent's token.";
16 /** The last this many are kept, beside the first: at most 21 rows. */
17 export const CENSUS_KEEP = 20;
18 export const GREW_SHOWN = 20;
19 /** The most types one census keeps, largest first: a heap names a few hundred. */
20 const TYPES_MAX = 2000;
21
22 export type Census = { at: number; uptimeMin: number; heapSize: number; objectCount: number; byType: Record<string, number> };
23 export type Grew = { type: string; then: number; now: number };
24 export type Reading = { heapSize: number; objectCount: number; objectTypeCounts: Record<string, number> };
25 export type Asked =
26 | { ok: true; census: Census }
27 | { ok: false; retryAfterSec: number; sentence: string };
28
29 const count = (n: unknown): n is number => typeof n === "number" && Number.isFinite(n) && n >= 0;
30
31 /** Only counts, the largest `TYPES_MAX` types; anything else in a reading is dropped. */
32 function typesOf(raw: unknown): Record<string, number> {
33 if (raw === null || typeof raw !== "object") return {};
34 const rows = Object.entries(raw as Record<string, unknown>).filter((e): e is [string, number] => count(e[1]) && e[0].length <= 200);
35 rows.sort((a, b) => b[1] - a[1]);
36 return Object.fromEntries(rows.slice(0, TYPES_MAX));
37 }
38
39 /** One census from one reading; `read` is `heapStats` but in a test. */
40 export function censusOf(now: number, uptimeSec: number, read: () => Reading = heapStats): Census {
41 const r = read();
42 return {
43 at: now,
44 uptimeMin: Math.round(uptimeSec / 60),
45 heapSize: count(r.heapSize) ? r.heapSize : 0,
46 objectCount: count(r.objectCount) ? r.objectCount : 0,
47 byType: typesOf(r.objectTypeCounts),
48 };
49 }
50
51 /** Store one census, keeping the first and the last `CENSUS_KEEP`. */
52 export function saveCensus(db: Database, c: Census): void {
53 db.transaction(() => quietly(db, () => {
54 db.run("INSERT INTO heap_census (at, uptime_min, heap_size, object_count, by_type) VALUES (?, ?, ?, ?, ?)", [
55 c.at, c.uptimeMin, c.heapSize, c.objectCount, JSON.stringify(c.byType),
56 ]);
57 db.run(
58 `DELETE FROM heap_census WHERE id <> (SELECT min(id) FROM heap_census)
59 AND id NOT IN (SELECT id FROM heap_census ORDER BY id DESC LIMIT ?)`,
60 [CENSUS_KEEP],
61 );
62 }))();
63 }
64
65 type Row = { at: number; uptime_min: number; heap_size: number; object_count: number; by_type: string };
66
67 function parsed(text: string): Record<string, number> {
68 try {
69 return typesOf(JSON.parse(text));
70 } catch {
71 return {};
72 }
73 }
74
75 /** Every kept census, oldest first: the first is the baseline. */
76 export function censuses(db: Database): Census[] {
77 return db
78 .query<Row, []>("SELECT at, uptime_min, heap_size, object_count, by_type FROM heap_census ORDER BY id")
79 .all()
80 .map((r) => ({ at: r.at, uptimeMin: r.uptime_min, heapSize: r.heap_size, objectCount: r.object_count, byType: parsed(r.by_type) }));
81 }
82
83 /**
84 * The `n` types whose count grew most from `first` to `last`, largest growth first; a type new
85 * since is counted from 0.
86 */
87 export function grewMost(first: Census, last: Census, n = GREW_SHOWN): Grew[] {
88 return Object.entries(last.byType)
89 .map(([type, now]) => ({ type, then: first.byType[type] ?? 0, now }))
90 .filter((g) => g.now > g.then)
91 .sort((a, b) => b.now - b.then - (a.now - a.then) || a.type.localeCompare(b.type))
92 .slice(0, n);
93 }
94
95 /** Take and store a census, unless one was taken in the last ten minutes. */
96 export function askCensus(db: Database, now: number, uptimeSec: number, read: () => Reading = heapStats): Asked {
97 const last = db.query<{ at: number }, []>("SELECT at FROM heap_census ORDER BY id DESC LIMIT 1").get();
98 if (last && now - last.at < CENSUS_EVERY_MS) {
99 const retryAfterSec = Math.ceil((CENSUS_EVERY_MS - (now - last.at)) / 1000);
100 const minutes = Math.ceil(retryAfterSec / 60);
101 return {
102 ok: false,
103 retryAfterSec,
104 sentence: `A heap census holds the forge for a fraction of a second, so one is taken at most every ten minutes; the next can be taken in ${minutes} minute${minutes === 1 ? "" : "s"}.`,
105 };
106 }
107 const census = censusOf(now, uptimeSec, read);
108 saveCensus(db, census);
109 return { ok: true, census };
110 }
111
112 /**
113 * What the page, the terminal and the API say: the censuses' heads, and what grew once there are
114 * two.
115 */
116 export function censusReport(db: Database): { censuses: Omit<Census, "byType">[]; grew: Grew[] } {
117 const all = censuses(db);
118 const heads = all.map(({ byType: _, ...head }) => head);
119 return { censuses: heads, grew: all.length < 2 ? [] : grewMost(all[0]!, all[all.length - 1]!) };
120 }
src/tui/screens/system.ts +37 −3
@@ -38,6 +38,7 @@ import type { SystemView } from "../../system/view.ts";
3838 import { rolloutLine, type ForgeHealth } from "../../kernel/guard.ts";
3939 import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
4040 import { droppedRows } from "../../boards/sampler.ts";
41 import { askCensus, CENSUS_AGENT_REFUSED, censusReport } from "../../perf/heap_census.ts";
4142 import { askingOf } from "../../perf/asking.ts";
4243 import { turnStatsOf } from "../../http/app_turn.ts";
4344 import { telemetryReader } from "../../actions/telemetry.ts";
@@ -49,7 +50,7 @@ import { grantable } from "../../core/telemetry.ts";
4950 import { bytes, spark } from "../../system/format.ts";
5051 import { style } from "../term/style.ts";
5152 import { pad, truncate, wrap } from "../term/width.ts";
52 import { dialog } from "../ui/widgets.ts";
53 import { dialog, type Tone } from "../ui/widgets.ts";
5354 import { Scroll, ago, confirmScreen, drawLines, hitScroll, later, pickerScreen } from "./common.ts";
5455 import { largestDirs, type LargestDirs } from "../../system/largest_dirs.ts";
5556 import { dbSize, dbSizeNeed, type DbSize } from "../../system/db_size.ts";
@@ -435,7 +436,8 @@ const mib = (n: number) => `${(n / 1_048_576).toFixed(1)}`;
435436 export function memoryLines(session: Session, now = Date.now()): string[] {
436437 if (!canSeeMemory(session.user)) return [];
437438 const hours = memoryHours(session.db, now);
438 if (hours.length === 0) return ["No samples yet; the forge samples its memory every minute."];
439 const census = censusLines(session);
440 if (hours.length === 0) return ["No samples yet; the forge samples its memory every minute.", ...census];
439441 const col = (s: string) => pad(s, 10, "right");
440442 return [
441443 `${pad("Hour (UTC)", 17)}${col("RSS mean")}${col("RSS max")}${col("Heap mean")}${col("Heap max")} MiB`,
@@ -443,9 +445,23 @@ export function memoryLines(session: Session, now = Date.now()): string[] {
443445 (h) =>
444446 `${pad(new Date(h.at).toISOString().slice(0, 16).replace("T", " "), 17)}${col(mib(h.rssMean))}${col(mib(h.rssMax))}${col(mib(h.heapMean))}${col(mib(h.heapMax))}`,
445447 ),
448 ...census,
446449 ];
447450 }
448451
452 /**
453 * The heap censuses /system took (#15309, `perf/heap_census.ts`): how many, and the types that grew
454 * most from the first to the last. `c` on the memory screen takes one, as /system's button does.
455 */
456 export function censusLines(session: Session): string[] {

Both doors: the terminal shows the census but cannot take one. Add a take action here behind the same canSeeMemory gate, calling askCensus and showing the 429 sentence on refusal, so a person admin can do in the terminal what the page button does; a write there must also reach the page. Update docs/features/memory-growth.md, which now says the terminal cannot.

457 if (!canSeeMemory(session.user)) return [];
458 const { censuses, grew } = censusReport(session.db);
459 if (censuses.length === 0) return ["", "No heap census yet; c takes one."];
460 const head = `${String(censuses.length)} heap census${censuses.length === 1 ? "" : "es"}; the last at ${String(censuses[censuses.length - 1]!.uptimeMin)} min up.`;
461 if (censuses.length < 2) return ["", head];
462 return ["", head, `${pad("Type grown since the first", 40)}${pad("First", 12, "right")}${pad("Last", 12, "right")}`, ...grew.map((g) => `${pad(g.type, 40)}${pad(String(g.then), 12, "right")}${pad(String(g.now), 12, "right")}`)];
463 }
464
449465 /**
450466 * "Who is asking" as /system says it, for the owner alone (`canSeeMemory`); empty for anyone
451467 * else, or when nothing is heavy and nothing has waited.
@@ -462,11 +478,24 @@ export function askingLines(session: Session, inner = WIDTH - 6): string[] {
462478 ];
463479 }
464480
481 /**
482 * `c` on the memory screen: /system's button (#15309), under the same gate (`canSeeMemory`) and the
483 * same ten-minute limit (`askCensus`), into the same table the page reads; what it says is a toast.
484 */
485 export function takeCensus(session: Session, now = Date.now()): { text: string; tone: Tone } {
486 if (!canSeeMemory(session.user))
487 return { text: session.user?.kind === "agent" ? CENSUS_AGENT_REFUSED : "A heap census is for a forge admin.", tone: "danger" };
488 const asked = askCensus(session.db, now, process.uptime());
489 return asked.ok
490 ? { text: `Heap census taken: ${asked.census.objectCount.toLocaleString("en-US")} objects.`, tone: "success" }
491 : { text: asked.sentence, tone: "danger" };
492 }
493
465494 export function memoryScreen(session: Session, ctx: Ctx): Screen {
466495 const scroll = new Scroll();
467496 return {
468497 title: "System memory",
469 keys: [binding(["j", "k"], "scroll"), binding(["esc", "q"], "back")],
498 keys: [binding(["j", "k"], "scroll"), ...(canSeeMemory(session.user) ? [binding(["c"], "take a heap census")] : []), binding(["esc", "q"], "back")],
470499 render(area, s) {
471500 const c = ctx.theme.c;
472501 const lines = memoryLines(session);
@@ -476,6 +505,11 @@ export function memoryScreen(session: Session, ctx: Ctx): Screen {
476505 },
477506 key(e) {
478507 if (scroll.key(e, memoryLines(session).length)) return true;
508 if (e.key === "c" && !e.ctrl && canSeeMemory(session.user)) {
509 const said = takeCensus(session);
510 ctx.toast(said.text, said.tone);
511 return true;
512 }
479513 if (e.key === "esc" || e.key === "q") {
480514 ctx.pop();
481515 return true;
src/web/pages/system.ts +56 −3
@@ -46,9 +46,12 @@ import { openChildren } from "../../spawn/host.ts";
4646 import { LAG_BAR_MS, LAG_READINGS, overBar, type LagReading } from "../../perf/lag_minutes.ts";
4747 import { WORST_STALLS, type BlamedStall } from "../../perf/stall_blame.ts";
4848 import { notePhase, takeSessionTime, timePhase } from "../../core/route_timing.ts";
49 import { notFoundPage, signInFirst } from "../chrome.ts";
49 import { forbiddenPage, notFoundPage, pageHead, signInFirst } from "../chrome.ts";
50 import { askCensus, CENSUS_AGENT_REFUSED, censusReport, GREW_SHOWN, type Census, type Grew } from "../../perf/heap_census.ts";
51 import { apiError, prefersJson } from "../../http/respond.ts";
52 import { readForm } from "../form.ts";
5053 import { html, raw, type Html } from "../html.ts";
51 import { page } from "../layout.ts";
54 import { flashTo, page } from "../layout.ts";
5255 import { systemLiveJs } from "../system-live.ts";
5356 import { RECENT_BYTES, recentEncodings } from "../../http/compress.ts";
5457 import { KEPT_PAGE_BYTES, keptPageStats } from "./kept_page.ts";
@@ -63,7 +66,7 @@ import { restoreLine, type RestoreLine } from "../../snapshot/restore_line.ts";
6366 import { lastDrill, lastPass } from "../../snapshot/drills.ts";
6467 import { stalePullsSection } from "./maintenance.ts";
6568 import { shapeWords } from "../../engine/mode.ts";
66 import { badge, chartBox, disclosure, meter, pill, row, time } from "../ui.ts";
69 import { badge, btn, chartBox, disclosure, meter, pill, row, time } from "../ui.ts";
6770 import { LARGEST_DIRS, largestDirs, type LargestDirs } from "../../system/largest_dirs.ts";
6871 import { DB_FILE_CAP, DB_GROWTH_MAX, TOP, dbSize, dbSizeNeed, type DbSize } from "../../system/db_size.ts";
6972 import { resultWords, weekRows, weekWords, type WeekRow } from "../../maintain/week_line.ts";
@@ -314,6 +317,55 @@ function procTable(v: SystemView): Html {
314317 const mib = (n: number) => `${(n / 1_048_576).toFixed(1)} MiB`;
315318 const hhmm = (at: number) => new Date(at).toISOString().slice(0, 16).replace("T", " ");
316319
320 /**
321 * One line and a button for a heap census (#15309, `perf/heap_census.ts`); after two, the types
outdated

The doc comment of memorySection (The forge RSS beside its JS heap...) is now stranded above heapCensusSection. Move it back onto memorySection.

322 * that grew most from the first census to the last, their counts then and now.
323 */
324 export function heapCensusSection(heads: readonly Omit<Census, "byType">[], grew: readonly Grew[]): Html {
325 const first = heads[0];
326 const last = heads[heads.length - 1];
327 const said =
328 first === undefined || last === undefined
329 ? "No heap census yet."
330 : heads.length === 1
331 ? `One heap census, at ${String(first.uptimeMin)} min up: ${first.objectCount.toLocaleString("en-US")} objects. The next shows what grew since.`
332 : `${String(heads.length)} heap censuses: ${first.objectCount.toLocaleString("en-US")} objects at the first (${String(first.uptimeMin)} min up), ${last.objectCount.toLocaleString("en-US")} at the last (${String(last.uptimeMin)} min up).`;
333 // The heading and the one-sentence state stay on first view; the button and the table sit
334 // behind the disclosure, as the other sections' rows do.
335 return html`<section class="settings-block" id="heap-census"><h2>Heap census</h2>
336 ${disclosure(said, html`<form method="post" action="/system/heap-census" class="row-meta">
337 ${btn("Take a heap census", { type: "submit", size: "sm" })}</form>
338 <p class="muted small">It counts every object on the heap by type and holds the forge for a fraction of a second, so it is taken at most every ten minutes. The first is the baseline the next ones are compared with.</p>
339 ${heads.length < 2
340 ? null
341 : grew.length === 0
342 ? html`<p class="muted small" id="heap-grew-none">No type grew since the first census.</p>`
343 : html`<div class="table-scroll"><table class="table" data-dense id="heap-grew" aria-label="The ${String(GREW_SHOWN)} object types that grew most since the first census">
344 <thead><tr><th scope="col">Type</th><th scope="col" class="num">First</th><th scope="col" class="num">Last</th></tr></thead>
345 <tbody>${grew.map((g) => html`<tr data-type="${g.type}"><td class="mono">${g.type}</td><td class="num mono">${String(g.then)}</td><td class="num mono">${String(g.now)}</td></tr>`)}</tbody>
346 </table></div>`}`)}</section>`;
347 }
348
349 export { CENSUS_AGENT_REFUSED };
350
351 /** POST /system/heap-census: one census, a person admin's, at most every ten minutes. */
352 export async function heapCensusSubmit(ctx: Ctx): Promise<Response> {
353 if (!ctx.auth.user) return signInFirst(ctx);
354 const form = await readForm(ctx, "admin");
355 if (form instanceof Response) return form;
356 // An agent hears why in one sentence; a person who is not an admin finds no such page.
357 if (!canSeeMemory(ctx.auth.user)) return ctx.auth.user.kind === "agent" ? forbiddenPage(ctx, CENSUS_AGENT_REFUSED) : notFoundPage(ctx);
358 const asked = askCensus(ctx.app.db, Date.now(), process.uptime());
359 if (!asked.ok) {
360 const res = prefersJson(ctx.request)
361 ? apiError("rate_limited", 429, asked.sentence)
362 : page({ ctx, title: "Heap census", body: html`${pageHead("Heap census", { sub: "One count of the forge's objects by type, kept to compare with the first." })}<p>${asked.sentence}</p><p><a href="/system#heap-census">Back to System</a></p>` }, 429);
363 res.headers.set("retry-after", String(asked.retryAfterSec));
364 return res;
365 }
366 return flashTo(ctx, "/system#heap-census", { message: `Heap census taken: ${asked.census.objectCount.toLocaleString("en-US")} objects.` });
367 }
368
317369 /** The forge's RSS beside its JS heap, an hour a row; empty for anyone `canSeeMemory` refuses. */
318370 function memorySection(ctx: Ctx): Html {
319371 if (!canSeeMemory(ctx.auth.user)) return html``;
@@ -783,6 +835,7 @@ export async function systemPage(ctx: Ctx, patterns: () => readonly string[]): P
783835 <div id="sys-readers">${readersSection(readerPaces())}</div>
784836 <div id="sys-caches">${cachesSection(pageCaches())}</div>
785837 ${memory}
838 ${owner ? timePhase(ctx.request, "census", () => { const r = censusReport(ctx.app.db); return heapCensusSection(r.censuses, r.grew); }) : html``}
786839 ${stalePullsSection(ctx, false)}`;
787840 const response = page({ ctx, title: "System", body, wide: true, scriptJs: systemLiveJs });
788841 notePhase(ctx.request, "render", performance.now() - rendering);
test/heap-census.test.ts added +138
@@ -0,0 +1,138 @@
1 // The heap census (#15309): taken only on a person admin's ask, at most every ten minutes, the
2 // first and the last twenty kept, and the types that grew most ranked from the first to the last.
3 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { createAgent } from "../src/actions/crew.ts";
9 import { createUser, getUserByName } from "../src/core/users.ts";
10 import { createSession } from "../src/core/sessions.ts";
11 import { grant } from "../src/core/telemetry.ts";
12 import { serve, type Servers } from "../src/http/server.ts";
13 import { askCensus, CENSUS_EVERY_MS, CENSUS_KEEP, censusOf, censuses, censusReport, grewMost, saveCensus, type Census } from "../src/perf/heap_census.ts";
14 import { CENSUS_AGENT_REFUSED, heapCensusSection } from "../src/web/pages/system.ts";
15 import { seedAdmin } from "./helpers/admin.ts";
16 import { tokenFor } from "./helpers/tokens.ts";
17
18 let dir: string;
19 let app: App;
20 let servers: Servers;
21 let base: string;
22 let adminCookie: string;
23 let memberCookie: string;
24 beforeAll(async () => {
25 dir = await mkdtemp(join(tmpdir(), "forge-heap-census-"));
26 app = createApp({ dataDir: dir, port: 0 });
27 servers = await serve(app);
28 base = servers.url;
29 const root = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-good-passphrase" });
30 if (!root.ok) throw new Error(root.error);
31 adminCookie = `forge_session=${createSession(app.db, root.value.id).token}`;
32 const bob = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
33 if (!bob.ok) throw new Error(bob.error);
34 memberCookie = `forge_session=${createSession(app.db, bob.value.id).token}`;
35 const agent = await createAgent(app, root.value, { username: "scoutly", model: "claude-sonnet-5-5", brief: "Ships.", roles: ["builder"] });
36 if (!agent.ok) throw new Error(String(agent.error));
37 // An admin agent is still refused: a census is a person's ask.
38 app.db.run("UPDATE user SET is_admin = 1 WHERE username = 'scoutly'");
39 grant(app.db, getUserByName(app.db, "scoutly")!.id, root.value.id);
40 });
41 afterAll(async () => {
42 servers.stop();
43 await rm(dir, { recursive: true, force: true });
44 });
45
46 const fake = (at: number, byType: Record<string, number>): Census => ({ at, uptimeMin: at / 60_000, heapSize: 1, objectCount: 1, byType });
47 const post = (headers: Record<string, string>) =>
48 fetch(`${base}/system/heap-census`, { method: "POST", headers: { origin: base, ...headers }, redirect: "manual" });
49
50 describe("heap census", () => {
51 test("a crew agent's token is refused with one sentence, a member is refused by the admin guard, and neither takes a census", async () => {
52 const agent = await post({ authorization: `Bearer ${tokenFor(app, "scoutly", "admin")}`, accept: "application/json" });
53 expect(agent.status).toBe(403);
54 expect(await agent.text()).toContain(CENSUS_AGENT_REFUSED);
55 // The forge's admin guard answers a member before the route runs.
outdated

Pin one status for a non-admin member (the handler path returns 404, the forge may 403 earlier): assert the exact one so the test fails if the gate changes.

56 const member = await post({ cookie: memberCookie });
57 expect(member.status).toBe(403);
58 expect(await member.text()).toContain("This is for the forge&#39;s admin account");
59 expect(censuses(app.db)).toHaveLength(0);
60 });
61
62 test("a cross-origin ask is refused", async () => {
63 const res = await post({ cookie: adminCookie, origin: "https://evil.example" });
64 expect(res.status).toBe(403);
65 expect(censuses(app.db)).toHaveLength(0);
66 });
67
68 test("an admin's ask takes one census; a second within ten minutes is a 429 with Retry-After and why", async () => {
69 const first = await post({ cookie: adminCookie });
70 expect(first.status).toBe(303);
71 const kept = censuses(app.db);
72 expect(kept).toHaveLength(1);
73 expect(kept[0]!.objectCount).toBeGreaterThan(0);
74 expect(Object.keys(kept[0]!.byType).length).toBeGreaterThan(0);
75 const again = await post({ cookie: adminCookie });
76 expect(again.status).toBe(429);
77 expect(Number(again.headers.get("retry-after"))).toBeGreaterThan(500);
78 expect(await again.text()).toContain("at most every ten minutes");
79 expect(censuses(app.db)).toHaveLength(1);
80 });
81
82 test("the page shows the button; the agent with the telemetry grant reads the censuses as JSON", async () => {
83 const page = await (await fetch(`${base}/system`, { headers: { cookie: adminCookie } })).text();
84 expect(page).toContain("Take a heap census");
85 const read = await fetch(`${base}/api/v1/telemetry/heap-census`, { headers: { authorization: `Bearer ${tokenFor(app, "scoutly", "ro")}` } });
86 expect(read.status).toBe(200);
87 const body = (await read.json()) as { censuses: unknown[]; grew: unknown[] };
88 expect(body.censuses).toHaveLength(1);
89 const member = await fetch(`${base}/api/v1/telemetry/heap-census`, { headers: { authorization: `Bearer ${tokenFor(app, "bob", "ro")}` } });
90 expect(member.status).toBe(403);
91 });
92
93 test("the rate limit reads the last census, and lets one through after ten minutes", () => {
94 const db = createApp({ dataDir: join(dir, "limit"), port: 0 }).db;
95 const read = () => ({ heapSize: 10, objectCount: 3, objectTypeCounts: { Object: 3 } });
96 expect(askCensus(db, 1_000_000, 60, read).ok).toBe(true);
97 const early = askCensus(db, 1_000_000 + CENSUS_EVERY_MS - 1, 60, read);
98 expect(early.ok).toBe(false);
99 if (!early.ok) expect(early.retryAfterSec).toBe(1);
100 expect(askCensus(db, 1_000_000 + CENSUS_EVERY_MS, 60, read).ok).toBe(true);
101 });
102
103 test("keeps the first and the last twenty: 21 rows at most", () => {
104 const db = createApp({ dataDir: join(dir, "bound"), port: 0 }).db;
105 for (let i = 0; i < 30; i++) saveCensus(db, fake(i * 60_000, { Object: i }));
106 const kept = censuses(db);
107 expect(kept).toHaveLength(CENSUS_KEEP + 1);
108 expect(kept[0]!.byType.Object).toBe(0);
109 expect(kept[1]!.byType.Object).toBe(10);
110 expect(kept[kept.length - 1]!.byType.Object).toBe(29);
111 });
112
113 test("the comparison ranks the type that grew most first, a new type from 0, and leaves out what shrank", () => {
114 const first = fake(0, { Object: 100, Array: 50, Map: 10, Gone: 9 });
115 const last = fake(3_600_000, { Object: 110, Array: 5000, Map: 10, Promise: 300, Gone: 1 });
116 expect(grewMost(first, last)).toEqual([
117 { type: "Array", then: 50, now: 5000 },
118 { type: "Promise", then: 0, now: 300 },
119 { type: "Object", then: 100, now: 110 },
120 ]);
121 const html = heapCensusSection([first, last], grewMost(first, last)).toString();
122 expect(html).toContain('data-type="Array"');
123 expect(html).toContain("5000");
124 });
125
126 test("a census reads only counts from heapStats, and the report leaves byType out of its heads", () => {
127 const c = censusOf(5, 125, () => ({ heapSize: 9, objectCount: 2, objectTypeCounts: { Ok: 2, Bad: -1, Nan: Number.NaN } as Record<string, number> }));
128 expect(c.byType).toEqual({ Ok: 2 });
129 expect(c.uptimeMin).toBe(2);
130 const db = createApp({ dataDir: join(dir, "report"), port: 0 }).db;
131 saveCensus(db, fake(0, { A: 1 }));
132 expect(censusReport(db).grew).toEqual([]);
133 saveCensus(db, fake(1, { A: 4 }));
134 const r = censusReport(db);
135 expect(r.censuses[0]).not.toHaveProperty("byType");
136 expect(r.grew).toEqual([{ type: "A", then: 1, now: 4 }]);
137 });
138 });
test/system-parts.test.ts +1
@@ -65,6 +65,7 @@ test("one <h2> per part, each in a settings-block, and no card", async () => {
6565 "Telemetry readers",
6666 "Code page caches, since start",
6767 "Forge memory",
68 "Heap census",
6869 "Pulls untouched for 14 days",
6970 ]);
7071 expect(main.match(/<section class="settings-block"/g)?.length).toBe(heads.length);
test/tui-system.test.ts +30 −1
@@ -16,7 +16,7 @@ import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
1616 import { setTelemetryGrant } from "../src/actions/telemetry.ts";
1717 import { hasGrant } from "../src/core/telemetry.ts";
1818 import { countRead, resetTelemetryReads } from "../src/system/telemetry_reads.ts";
19 import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, routeUseLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts";
19 import { takeCensus, askingLines, dbSizeLines, dbSizeScreen, footprintLines, routeUseLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts";
2020 import type { DbSize } from "../src/system/db_size.ts";
2121 import type { SystemView } from "../src/system/view.ts";
2222 import { changeCapacity } from "../src/actions/capacity.ts";
@@ -26,6 +26,7 @@ import type { SmallReading } from "../src/release/small_reading.ts";
2626 import { headlessTerminal } from "../src/tui/term/terminal.ts";
2727 import { askingOf } from "../src/perf/asking.ts";
2828 import { quietLogs } from "./helpers/globals.ts";
29 import { CENSUS_AGENT_REFUSED, censusReport } from "../src/perf/heap_census.ts";
2930 import { settleApp } from "./helpers/tui.ts";
3031
3132 quietLogs();
@@ -91,6 +92,34 @@ afterAll(async () => {
9192 await rm(dir, { recursive: true, force: true });
9293 });
9394
95 describe("tui heap census (#15309)", () => {
96 test("c on the memory screen takes a census the page reads; a second within ten minutes says why", async () => {
97 const terminal = headlessTerminal(120, 30);
98 const { app: tui, session } = createForgeApp(app, identity(carol), { terminal, exitHooks: false });
99 const toasts: string[] = [];
100 tui.start();
101 const ctx = { ...tui.ctx, toast: (t: string) => toasts.push(t) };
102 const screen = memoryScreen(session, ctx);
103 expect(screen.keys?.some((k) => JSON.stringify(k).includes("heap census"))).toBe(true);
104 const before = censusReport(app.db).censuses.length;
105 screen.key!({ key: "c" } as Parameters<NonNullable<typeof screen.key>>[0]);
106 expect(censusReport(app.db).censuses.length).toBe(before + 1);
107 expect(toasts[0]).toStartWith("Heap census taken: ");
108 screen.key!({ key: "c" } as Parameters<NonNullable<typeof screen.key>>[0]);
109 expect(censusReport(app.db).censuses.length).toBe(before + 1);
110 expect(toasts[1]).toContain("at most every ten minutes");
111 tui.stop?.();
112 });
113
114 test("nobody else takes one: an agent hears why, a member is refused, neither writes", () => {
115 const before = censusReport(app.db).censuses.length;
116 expect(takeCensus(new Session(app, identity(bot))).text).toBe(CENSUS_AGENT_REFUSED);
117 expect(takeCensus(new Session(app, identity(bob))).tone).toBe("danger");
118 expect(takeCensus(new Session(app, identity(undefined))).tone).toBe("danger");
119 expect(censusReport(app.db).censuses.length).toBe(before);
120 });
121 });
122
94123 describe("tui system memory", () => {
95124 test("an admin person gets the rows and the door", async () => {
96125 const lines = memoryLines(new Session(app, identity(carol)));

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs