Public: anyone can read this; the crew can change it.

A board's sampler caps the rows it keeps per source and says on the board when it dropped some #15181

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15160-a-board-s-sampler-caps-the-rows-it-keeps → main 9 commits fork 4 threads · 4 unresolved

  • Merged. scout/15160-a-board-s-sampler-caps-the-rows-it-keeps was merged into main as e3b8d6c.

Review

Sign in to review this change.
6 files changed +192 −2
docs/features/sampler-cap.md added +24
@@ -0,0 +1,24 @@
1 ---
2 topic: When something is wrong
3 row: The machine sampler's row cap
4 ---
5 # The machine sampler's row cap
6
7 The machine sampler (`src/boards/sampler.ts`) writes a 15-second row per source (`machine_sample`,
8 one target each: the forge's pod, each cell, the node, each CPU). The cap bounds how many of those
9 rows one source keeps (#15160).
10
11 - **The limit.** `sampler.rows_per_source`, 6,000 rows by default (240 to 100,000), on the
12 owner's limits page.
13 - **A backstop.** The age rule (`system/retention_samples.ts`) already keeps a day, 5,760 rows
14 a source, so under the defaults the cap never drops anything. It bites when points come faster
15 than every 15 s, when the age rule stops, or when an owner lowers the cap under a day.
16 - **What goes.** The oldest 15-second rows of a source past the cap. Coarser steps are not
17 touched. The count is taken once per 5-minute window, not on every tick.
18 - **Where it shows.** Only once something was dropped:
19 - Web: the machine board's "What this machine measures" fold says
20 "N oldest rows dropped since this forge started."
21 - Terminal: the System dialog says the same line under the machine.
22 - **The count restarts with the forge.** It is held in the process, not in the database.
23 - **Tests.** `test/boards-sampler-cap.test.ts`: the oldest rows go, the newest stay, the board
24 and the terminal read the count.
src/boards/sampler.ts +42 −1
@@ -21,12 +21,15 @@
2121 * A file that is missing or unreadable (not Linux, no cgroup v2) leaves its
2222 * series null — never a guess — and `gaps` names it for the page.
2323 */
24 import type { Database } from "bun:sqlite";
2425 import { join } from "node:path";
2526 import type { App } from "../app.ts";
2627 import { cellCounters, type CellCounter } from "../apps/contain.ts";
28 import { quietly } from "../core/write_clock.ts";
2729 import { startLoopLag, type LoopLag } from "../kernel/loop_lag.ts";
30 import { limitValue } from "../settings/limits.ts";
2831 import { cpuTarget, readCpus, readPoint, unreadable } from "./machine.ts";
29 import { FINE_MS, record, type Point } from "./ring.ts";
32 import { COARSE_MS, FINE_MS, record, type Point } from "./ring.ts";
3033 import { readHwmon, readTopology, temperatures } from "./thermal.ts";
3134 import { during } from "../perf/jobs.ts";
3235
@@ -147,6 +150,37 @@ export function gaps(paths: Paths): string[] {
147150 return out;
148151 }
149152
153 /**
154 * What the cap has dropped since this process started; the board reads it (`web/pages/boards.ts`).
155 */
156 let droppedTotal = 0;
157
158 export function droppedRows(): number {
159 return droppedTotal;
160 }
161
162 /**
163 * Drop the oldest 15-s rows of every source (a `machine_sample` target) past `cap`, and
164 * return how many went. The age rule (`system/retention_samples.ts`) keeps a day;
165 * this is the bound for when points come faster than that rule expects.
166 */
167 export function capRows(db: Database, cap: number): number {
168 const over = db.query<{ target: string }, [number]>(
169 "SELECT target FROM machine_sample WHERE step = 15 GROUP BY target HAVING COUNT(*) > ?1",
170 ).all(cap);
171 let dropped = 0;
172 for (const { target } of over) {
173 const edge = db.query<{ at: string }, [string, number]>(
174 "SELECT at FROM machine_sample WHERE step = 15 AND target = ?1 ORDER BY at DESC LIMIT 1 OFFSET ?2",
175 ).get(target, cap - 1)?.at;
176 if (edge === undefined) continue;
177 dropped += quietly(db, () =>
178 db.query("DELETE FROM machine_sample WHERE step = 15 AND target = ?1 AND at < ?2").run(target, edge).changes);
179 }
180 droppedTotal += dropped;
181 return dropped;
182 }
183
150184 /** `gaps` is what this sampler cannot read, said once at start and kept here for whoever asks. */
151185 export type Sampler = { tick(now?: number): void; stop(): void; gaps: readonly string[] };
152186
@@ -161,11 +195,18 @@ export function startSampler(
161195 let prev: Counters | undefined;
162196 const missing = gaps(paths);
163197 if (missing.length > 0) app.log.info("machine series unreadable here", { gaps: missing.join("; ") });
198 let capWindow = -1;
164199 const tick = (now = Date.now()) => {
165200 try {
166201 const next = readCounters(paths, cells(), now);
167202 record(app.db, pointsOf(prev, next, { lagMs: lag.take(), ...runningChildren(app.db) }), now);
168203 prev = next;
204 // Counting rows is a scan: once per 5-min window, not every step.
205 const window = Math.floor(now / COARSE_MS);
206 if (window !== capWindow) {
207 capWindow = window;
208 capRows(app.db, limitValue(app.db, "sampler.rows_per_source"));
209 }
169210 } catch (error) {
170211 app.log.warn("machine sample failed", { error: String(error) });
171212 }
src/settings/limits.ts +9
@@ -71,6 +71,7 @@ export type LimitKey =
7171 | "billing.retention_days"
7272 | "retention.event_days"
7373 | "retention.deploy_pull_days"
74 | "sampler.rows_per_source"
7475 | "security.sweep_max_usd";
7576
7677 export interface LimitDef {
@@ -447,6 +448,14 @@ export const LIMITS: Record<LimitKey, LimitDef> = {
447448 max: 3_650,
448449 reason: "The list is an index over git, walked when a deploy is opened, and it was the largest table in the database; nothing writes it now, and ask-to-live reads one row per pull (deploy_landed), so this is how long an older release's rows linger.",
449450 },
451 "sampler.rows_per_source": {
452 label: "15-second rows the machine sampler keeps per source",
453 unit: "rows",
454 default: 6_000,
455 min: 240,
456 max: 100_000,

The reason admits a day is 5,760 rows, yet the default is 6,000 and the age rule already prunes at a day, so the cap is dead under defaults. Say in the reason that it is a backstop only, or choose a default that bites.

457 reason: "A backstop only: the age rule already keeps a day, 5,760 rows a source, so under defaults this never bites; it bounds the table when points come faster than every 15 s or the age rule stops, and the board says how many went.",
458 },
450459 "security.sweep_max_usd": {
451460 label: "Weekly security sweep spend ceiling",
452461 unit: "dollars",
src/tui/screens/system.ts +11
@@ -28,6 +28,7 @@
2828 * Under them, the database's size (#12010, `dbSizeLines`): the file, its
2929 * WAL and growth, then its largest tables and indexes, through the same
3030 * `dbSize` cache and gate as /system and the MCP tool, never waited on.
31 * Under the machine, what the sampler's row cap dropped (#15160, `samplerLines`).
3132 * Then what the forge costs (#12092, `footprintLines`): the budget verdict and
3233 * what is off, from the last reading, gated as the rest.
3334 */
@@ -36,6 +37,7 @@ import type { Ctx, Screen } from "../app/app.ts";
3637 import type { SystemView } from "../../system/view.ts";
3738 import { rolloutLine, type ForgeHealth } from "../../kernel/guard.ts";
3839 import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
40 import { droppedRows } from "../../boards/sampler.ts";
3941 import { askingOf } from "../../perf/asking.ts";
4042 import { turnStatsOf } from "../../http/app_turn.ts";
4143 import { telemetryReader } from "../../actions/telemetry.ts";
@@ -481,6 +483,14 @@ export function memoryScreen(session: Session, ctx: Ctx): Screen {
481483 };
482484 }
483485
486 /**
487 * What the sampler's per-source cap dropped, in the machine board's words; none when it dropped
488 * nothing.
489 */
490 export function samplerLines(dropped: number): string[] {
491 return dropped > 0 ? [`${dropped.toLocaleString("en-US")} oldest rows dropped since this forge started.`] : [];
492 }
493
484494 /** This forge's size and its use against it, a line each, as /system's "This forge's size" says it; none on a forge that is no cell child. */
485495 /** The engine shape and why, as /system says it; none where the boot recorded none. */
486496 export function engineLines(engine: { mode: string; why: string } | undefined): string[] {
@@ -565,6 +575,7 @@ export const entry: PaletteEntry = {
565575 ...(left.length === 0 ? [] : ["", ...left]),
566576 "",
567577 ...systemLines(liveMachine().read()),
578 ...samplerLines(droppedRows()),
568579 ], session));
569580 },
570581 };
src/web/pages/boards.ts +4 −1
@@ -9,7 +9,7 @@
99 */
1010 import { CHART_NAMES, CHARTS, type Chart } from "../../boards/chart.ts";
1111 import { dataLink } from "../../boards/answer.ts";
12 import { gaps, SYSTEM } from "../../boards/sampler.ts";
12 import { droppedRows, gaps, SYSTEM } from "../../boards/sampler.ts";
1313 import { SOURCES } from "../../boards/sources.ts";
1414 import { BUCKET_MS, MAX_PARAM, readParams, type Bucket, type Params } from "../../boards/params.ts";
1515 import { fillFromSentence, MODEL_FILLED, unreadWords } from "../../boards/model_fill.ts";
@@ -221,6 +221,9 @@ async function explorer(ctx: Ctx, user: NonNullable<Ctx["auth"]["user"]>, asked:
221221 missing.length === 0 ? "every series is readable here" : `${count(missing.length)} series cannot be read here`,
222222 html`<p class="muted small">Sampled every 15 s: the forge's pod from its cgroup, each check and app cell from its process tree, the node and each of its CPUs (<code>cpu:&lt;nnn&gt;</code>) from /proc, and temperatures in °C from /sys/class/hwmon.</p>
outdated

Principle 1: the count is in the summary here and again in the paragraph below. Keep one. The summary is joined with a semicolon, so a screen reader hears a run-on; prefer the single line inside the fold.

223223 <ul class="small">${machineLinks(Date.now()).map((l) => html`<li><a href="${l.href}">${l.label}</a></li>`)}</ul>
224 ${droppedRows() > 0
225 ? html`<p class="small">${count(droppedRows())} oldest rows dropped since this forge started.</p>`
226 : null}
outdated

Principle 1: this paragraph repeats the summary. Drop one, and add that the count is since start and resets on restart. The fold is closed by default, so if the summary goes, the count is hidden behind a click.

Non-blocking: principle 1 and 8. The sentence is plain, but it is long. "N oldest rows dropped since this forge started" says it with the same facts.

224227 ${missing.length === 0
225228 ? html`<p class="small">Every series is readable on this machine.</p>`
226229 : html`<ul class="small">${missing.map((g) => html`<li>Absent — ${g}</li>`)}</ul>`}`,
test/boards-sampler-cap.test.ts added +102
@@ -0,0 +1,102 @@
1 /**
2 * The sampler's per-source row cap: the oldest 15-s rows past it go, the 5-min
3 * roll-ups stay, the cap is a named limit with a reason, and the machine board
4 * says how many were dropped.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { samplerLines } from "../src/tui/screens/system.ts";
11 import { capRows, droppedRows, startSampler } from "../src/boards/sampler.ts";
12 import { LIMITS, setLimit } from "../src/settings/limits.ts";
13 import { createApp } from "../src/app.ts";
14 import { serve } from "../src/http/server.ts";
15 import { createUser } from "../src/core/users.ts";
16 import { createToken } from "../src/core/tokens.ts";
17
18 const T0 = Date.parse("2026-10-15T12:00:00.000Z");
19 const at = (i: number) => new Date(T0 + i * 15_000).toISOString();
20 let dir: string;
21 let app: ReturnType<typeof createApp>;
22 let servers: Awaited<ReturnType<typeof serve>>;
23 let token: string;
24 let adminId: string;
25
26 beforeAll(async () => {
27 dir = await mkdtemp(join(tmpdir(), "forge-sampler-cap-"));
28 app = createApp({ dataDir: dir, port: 0 });
29 servers = await serve(app);
30 const r = await createUser(app.db, {
31 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
32 password: "a-good-passphrase", adminIfFirst: true,
33 });
34 if (!r.ok) throw new Error(r.error);
35 adminId = r.value.id;
36 token = createToken(app.db, r.value.id, "t", "admin").token;
37 });
38 afterAll(async () => {
39 servers.stop();
40 app.close();
41 await rm(dir, { recursive: true, force: true });
42 });
43
44 const rows = (target: string, step = 15) =>
45 app.db.query<{ at: string }, [string, number]>(
46 "SELECT at FROM machine_sample WHERE target = ?1 AND step = ?2 ORDER BY at",
47 ).all(target, step).map((r) => r.at);
48
49 describe("the sampler cap", () => {
50 test("the cap is a named setting with its reason", () => {
51 const l = LIMITS["sampler.rows_per_source"];
52 expect(l.default).toBeGreaterThanOrEqual(5_760);
53 expect(l.reason.length).toBeGreaterThan(0);
54 expect(l.reason).not.toContain("\n");
55 });
56
57 test("the oldest rows of a source go, the newest stay, other sources and roll-ups are untouched", () => {
58 const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, ?, ?, 1)");
59 for (let i = 0; i < 10; i++) put.run(at(i), 15, "a");
60 for (let i = 0; i < 4; i++) put.run(at(i), 15, "b");
61 put.run(at(0), 300, "a");
62 const before = droppedRows();
63 expect(capRows(app.db, 6)).toBe(4);
64 expect(rows("a")).toEqual([4, 5, 6, 7, 8, 9].map(at));
65 expect(rows("b")).toHaveLength(4);
66 expect(rows("a", 300)).toHaveLength(1);
67 expect(droppedRows() - before).toBe(4);
68 expect(capRows(app.db, 6)).toBe(0);
69 });
70
71 test("a tick applies the limit once per 5-min window", () => {
72 app.db.run("DELETE FROM machine_sample");
73 const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, 15, 'old', 1)");
74 for (let i = 0; i < 300; i++) put.run(at(i));
75 expect(setLimit(app.db, "sampler.rows_per_source", 240, adminId).ok).toBe(true);
76 const s = startSampler(app, { everyMs: 3_600_000, cells: () => [], paths: { cgroup: dir, proc: dir, sys: dir } });
77 try {
78 s.tick(T0 + 400 * 15_000);
79 } finally {
80 s.stop();
81 }
82 expect(rows("old")).toHaveLength(240);
83 expect(rows("old")[0]).toBe(at(60));
84 });
85
86 test("the machine board reads the count of dropped rows", async () => {
87 app.db.run("DELETE FROM machine_sample");
outdated

This writes the module-level capped.rows = 7 and never restores it. Any later test in the same process sees a stale count. Restore it in a finally or afterAll. The first test also pins a substring of the reason text, which breaks on any rewording; assert the reason is non-empty instead.

88 const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, 15, 'c', 1)");
89 for (let i = 0; i < 10; i++) put.run(at(i));
90 const before = droppedRows();
91 expect(capRows(app.db, 6)).toBe(4);
92 const res = await fetch(servers.url + "/boards/data?source=machine&aggregate=count&by=target", {
93 headers: { authorization: `Bearer ${token}` },
94 });
95 expect(await res.text()).toContain(`${before + 4} oldest rows dropped`);
96 });
97 });
98
99 test("the terminal's System dialog says the dropped count in the board's words, and nothing at zero", () => {
100 expect(samplerLines(0)).toEqual([]);
101 expect(samplerLines(6_000)).toEqual(["6,000 oldest rows dropped since this forge started."]);
102 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs