Public: anyone can read this; the crew can change it.
A board's sampler caps the rows it keeps per source and says on the board when it dropped some #15181
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/15160-a-board-s-sampler-caps-the-rows-it-keeps → main 9 commits fork 4 threads · 4 unresolved
docs/features/sampler-cap.md added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | --- | |
| 2 | topic: When something is wrong | |
| 3 | row: The machine sampler's row cap | |
| 4 | --- | |
| 5 | # The machine sampler's row cap | |
| 6 | ||
| 7 | The machine sampler (`src/boards/sampler.ts`) writes a 15-second row per source (`machine_sample`, | |
| 8 | one target each: the forge's pod, each cell, the node, each CPU). The cap bounds how many of those | |
| 9 | rows one source keeps (#15160). | |
| 10 | ||
| 11 | - **The limit.** `sampler.rows_per_source`, 6,000 rows by default (240 to 100,000), on the | |
| 12 | owner's limits page. | |
| 13 | - **A backstop.** The age rule (`system/retention_samples.ts`) already keeps a day, 5,760 rows | |
| 14 | a source, so under the defaults the cap never drops anything. It bites when points come faster | |
| 15 | than every 15 s, when the age rule stops, or when an owner lowers the cap under a day. | |
| 16 | - **What goes.** The oldest 15-second rows of a source past the cap. Coarser steps are not | |
| 17 | touched. The count is taken once per 5-minute window, not on every tick. | |
| 18 | - **Where it shows.** Only once something was dropped: | |
| 19 | - Web: the machine board's "What this machine measures" fold says | |
| 20 | "N oldest rows dropped since this forge started." | |
| 21 | - Terminal: the System dialog says the same line under the machine. | |
| 22 | - **The count restarts with the forge.** It is held in the process, not in the database. | |
| 23 | - **Tests.** `test/boards-sampler-cap.test.ts`: the oldest rows go, the newest stay, the board | |
| 24 | and the terminal read the count. |
src/boards/sampler.ts +42 −1
| @@ -21,12 +21,15 @@ | ||
| 21 | 21 | * A file that is missing or unreadable (not Linux, no cgroup v2) leaves its |
| 22 | 22 | * series null — never a guess — and `gaps` names it for the page. |
| 23 | 23 | */ |
| 24 | import type { Database } from "bun:sqlite"; | |
| 24 | 25 | import { join } from "node:path"; |
| 25 | 26 | import type { App } from "../app.ts"; |
| 26 | 27 | import { cellCounters, type CellCounter } from "../apps/contain.ts"; |
| 28 | import { quietly } from "../core/write_clock.ts"; | |
| 27 | 29 | import { startLoopLag, type LoopLag } from "../kernel/loop_lag.ts"; |
| 30 | import { limitValue } from "../settings/limits.ts"; | |
| 28 | 31 | import { cpuTarget, readCpus, readPoint, unreadable } from "./machine.ts"; |
| 29 | import { FINE_MS, record, type Point } from "./ring.ts"; | |
| 32 | import { COARSE_MS, FINE_MS, record, type Point } from "./ring.ts"; | |
| 30 | 33 | import { readHwmon, readTopology, temperatures } from "./thermal.ts"; |
| 31 | 34 | import { during } from "../perf/jobs.ts"; |
| 32 | 35 | |
| @@ -147,6 +150,37 @@ export function gaps(paths: Paths): string[] { | ||
| 147 | 150 | return out; |
| 148 | 151 | } |
| 149 | 152 | |
| 153 | /** | |
| 154 | * What the cap has dropped since this process started; the board reads it (`web/pages/boards.ts`). | |
| 155 | */ | |
| 156 | let droppedTotal = 0; | |
| 157 | ||
| 158 | export function droppedRows(): number { | |
| 159 | return droppedTotal; | |
| 160 | } | |
| 161 | ||
| 162 | /** | |
| 163 | * Drop the oldest 15-s rows of every source (a `machine_sample` target) past `cap`, and | |
| 164 | * return how many went. The age rule (`system/retention_samples.ts`) keeps a day; | |
| 165 | * this is the bound for when points come faster than that rule expects. | |
| 166 | */ | |
| 167 | export function capRows(db: Database, cap: number): number { | |
| 168 | const over = db.query<{ target: string }, [number]>( | |
| 169 | "SELECT target FROM machine_sample WHERE step = 15 GROUP BY target HAVING COUNT(*) > ?1", | |
| 170 | ).all(cap); | |
| 171 | let dropped = 0; | |
| 172 | for (const { target } of over) { | |
| 173 | const edge = db.query<{ at: string }, [string, number]>( | |
| 174 | "SELECT at FROM machine_sample WHERE step = 15 AND target = ?1 ORDER BY at DESC LIMIT 1 OFFSET ?2", | |
| 175 | ).get(target, cap - 1)?.at; | |
| 176 | if (edge === undefined) continue; | |
| 177 | dropped += quietly(db, () => | |
| 178 | db.query("DELETE FROM machine_sample WHERE step = 15 AND target = ?1 AND at < ?2").run(target, edge).changes); | |
| 179 | } | |
| 180 | droppedTotal += dropped; | |
| 181 | return dropped; | |
| 182 | } | |
| 183 | ||
| 150 | 184 | /** `gaps` is what this sampler cannot read, said once at start and kept here for whoever asks. */ |
| 151 | 185 | export type Sampler = { tick(now?: number): void; stop(): void; gaps: readonly string[] }; |
| 152 | 186 | |
| @@ -161,11 +195,18 @@ export function startSampler( | ||
| 161 | 195 | let prev: Counters | undefined; |
| 162 | 196 | const missing = gaps(paths); |
| 163 | 197 | if (missing.length > 0) app.log.info("machine series unreadable here", { gaps: missing.join("; ") }); |
| 198 | let capWindow = -1; | |
| 164 | 199 | const tick = (now = Date.now()) => { |
| 165 | 200 | try { |
| 166 | 201 | const next = readCounters(paths, cells(), now); |
| 167 | 202 | record(app.db, pointsOf(prev, next, { lagMs: lag.take(), ...runningChildren(app.db) }), now); |
| 168 | 203 | prev = next; |
| 204 | // Counting rows is a scan: once per 5-min window, not every step. | |
| 205 | const window = Math.floor(now / COARSE_MS); | |
| 206 | if (window !== capWindow) { | |
| 207 | capWindow = window; | |
| 208 | capRows(app.db, limitValue(app.db, "sampler.rows_per_source")); | |
| 209 | } | |
| 169 | 210 | } catch (error) { |
| 170 | 211 | app.log.warn("machine sample failed", { error: String(error) }); |
| 171 | 212 | } |
src/settings/limits.ts +9
| @@ -71,6 +71,7 @@ export type LimitKey = | ||
| 71 | 71 | | "billing.retention_days" |
| 72 | 72 | | "retention.event_days" |
| 73 | 73 | | "retention.deploy_pull_days" |
| 74 | | "sampler.rows_per_source" | |
| 74 | 75 | | "security.sweep_max_usd"; |
| 75 | 76 | |
| 76 | 77 | export interface LimitDef { |
| @@ -447,6 +448,14 @@ export const LIMITS: Record<LimitKey, LimitDef> = { | ||
| 447 | 448 | max: 3_650, |
| 448 | 449 | reason: "The list is an index over git, walked when a deploy is opened, and it was the largest table in the database; nothing writes it now, and ask-to-live reads one row per pull (deploy_landed), so this is how long an older release's rows linger.", |
| 449 | 450 | }, |
| 451 | "sampler.rows_per_source": { | |
| 452 | label: "15-second rows the machine sampler keeps per source", | |
| 453 | unit: "rows", | |
| 454 | default: 6_000, | |
| 455 | min: 240, | |
| 456 | max: 100_000, | |
| 457 | reason: "A backstop only: the age rule already keeps a day, 5,760 rows a source, so under defaults this never bites; it bounds the table when points come faster than every 15 s or the age rule stops, and the board says how many went.", | |
| 458 | }, | |
| 450 | 459 | "security.sweep_max_usd": { |
| 451 | 460 | label: "Weekly security sweep spend ceiling", |
| 452 | 461 | unit: "dollars", |
src/tui/screens/system.ts +11
| @@ -28,6 +28,7 @@ | ||
| 28 | 28 | * Under them, the database's size (#12010, `dbSizeLines`): the file, its |
| 29 | 29 | * WAL and growth, then its largest tables and indexes, through the same |
| 30 | 30 | * `dbSize` cache and gate as /system and the MCP tool, never waited on. |
| 31 | * Under the machine, what the sampler's row cap dropped (#15160, `samplerLines`). | |
| 31 | 32 | * Then what the forge costs (#12092, `footprintLines`): the budget verdict and |
| 32 | 33 | * what is off, from the last reading, gated as the rest. |
| 33 | 34 | */ |
| @@ -36,6 +37,7 @@ import type { Ctx, Screen } from "../app/app.ts"; | ||
| 36 | 37 | import type { SystemView } from "../../system/view.ts"; |
| 37 | 38 | import { rolloutLine, type ForgeHealth } from "../../kernel/guard.ts"; |
| 38 | 39 | import { canSeeMemory, memoryHours } from "../../boards/memory.ts"; |
| 40 | import { droppedRows } from "../../boards/sampler.ts"; | |
| 39 | 41 | import { askingOf } from "../../perf/asking.ts"; |
| 40 | 42 | import { turnStatsOf } from "../../http/app_turn.ts"; |
| 41 | 43 | import { telemetryReader } from "../../actions/telemetry.ts"; |
| @@ -481,6 +483,14 @@ export function memoryScreen(session: Session, ctx: Ctx): Screen { | ||
| 481 | 483 | }; |
| 482 | 484 | } |
| 483 | 485 | |
| 486 | /** | |
| 487 | * What the sampler's per-source cap dropped, in the machine board's words; none when it dropped | |
| 488 | * nothing. | |
| 489 | */ | |
| 490 | export function samplerLines(dropped: number): string[] { | |
| 491 | return dropped > 0 ? [`${dropped.toLocaleString("en-US")} oldest rows dropped since this forge started.`] : []; | |
| 492 | } | |
| 493 | ||
| 484 | 494 | /** This forge's size and its use against it, a line each, as /system's "This forge's size" says it; none on a forge that is no cell child. */ |
| 485 | 495 | /** The engine shape and why, as /system says it; none where the boot recorded none. */ |
| 486 | 496 | export function engineLines(engine: { mode: string; why: string } | undefined): string[] { |
| @@ -565,6 +575,7 @@ export const entry: PaletteEntry = { | ||
| 565 | 575 | ...(left.length === 0 ? [] : ["", ...left]), |
| 566 | 576 | "", |
| 567 | 577 | ...systemLines(liveMachine().read()), |
| 578 | ...samplerLines(droppedRows()), | |
| 568 | 579 | ], session)); |
| 569 | 580 | }, |
| 570 | 581 | }; |
src/web/pages/boards.ts +4 −1
| @@ -9,7 +9,7 @@ | ||
| 9 | 9 | */ |
| 10 | 10 | import { CHART_NAMES, CHARTS, type Chart } from "../../boards/chart.ts"; |
| 11 | 11 | import { dataLink } from "../../boards/answer.ts"; |
| 12 | import { gaps, SYSTEM } from "../../boards/sampler.ts"; | |
| 12 | import { droppedRows, gaps, SYSTEM } from "../../boards/sampler.ts"; | |
| 13 | 13 | import { SOURCES } from "../../boards/sources.ts"; |
| 14 | 14 | import { BUCKET_MS, MAX_PARAM, readParams, type Bucket, type Params } from "../../boards/params.ts"; |
| 15 | 15 | import { fillFromSentence, MODEL_FILLED, unreadWords } from "../../boards/model_fill.ts"; |
| @@ -221,6 +221,9 @@ async function explorer(ctx: Ctx, user: NonNullable<Ctx["auth"]["user"]>, asked: | ||
| 221 | 221 | missing.length === 0 ? "every series is readable here" : `${count(missing.length)} series cannot be read here`, |
| 222 | 222 | html`<p class="muted small">Sampled every 15 s: the forge's pod from its cgroup, each check and app cell from its process tree, the node and each of its CPUs (<code>cpu:<nnn></code>) from /proc, and temperatures in °C from /sys/class/hwmon.</p> |
outdated
Principle 1: the count is in the summary here and again in the paragraph below. Keep one. The summary is joined with a semicolon, so a screen reader hears a run-on; prefer the single line inside the fold. |
||
| 223 | 223 | <ul class="small">${machineLinks(Date.now()).map((l) => html`<li><a href="${l.href}">${l.label}</a></li>`)}</ul> |
| 224 | ${droppedRows() > 0 | |
| 225 | ? html`<p class="small">${count(droppedRows())} oldest rows dropped since this forge started.</p>` | |
| 226 | : null} | |
outdated
Principle 1: this paragraph repeats the summary. Drop one, and add that the count is since start and resets on restart. The fold is closed by default, so if the summary goes, the count is hidden behind a click. Non-blocking: principle 1 and 8. The sentence is plain, but it is long. "N oldest rows dropped since this forge started" says it with the same facts. |
||
| 224 | 227 | ${missing.length === 0 |
| 225 | 228 | ? html`<p class="small">Every series is readable on this machine.</p>` |
| 226 | 229 | : html`<ul class="small">${missing.map((g) => html`<li>Absent — ${g}</li>`)}</ul>`}`, |
test/boards-sampler-cap.test.ts added +102
| @@ -0,0 +1,102 @@ | ||
| 1 | /** | |
| 2 | * The sampler's per-source row cap: the oldest 15-s rows past it go, the 5-min | |
| 3 | * roll-ups stay, the cap is a named limit with a reason, and the machine board | |
| 4 | * says how many were dropped. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { samplerLines } from "../src/tui/screens/system.ts"; | |
| 11 | import { capRows, droppedRows, startSampler } from "../src/boards/sampler.ts"; | |
| 12 | import { LIMITS, setLimit } from "../src/settings/limits.ts"; | |
| 13 | import { createApp } from "../src/app.ts"; | |
| 14 | import { serve } from "../src/http/server.ts"; | |
| 15 | import { createUser } from "../src/core/users.ts"; | |
| 16 | import { createToken } from "../src/core/tokens.ts"; | |
| 17 | ||
| 18 | const T0 = Date.parse("2026-10-15T12:00:00.000Z"); | |
| 19 | const at = (i: number) => new Date(T0 + i * 15_000).toISOString(); | |
| 20 | let dir: string; | |
| 21 | let app: ReturnType<typeof createApp>; | |
| 22 | let servers: Awaited<ReturnType<typeof serve>>; | |
| 23 | let token: string; | |
| 24 | let adminId: string; | |
| 25 | ||
| 26 | beforeAll(async () => { | |
| 27 | dir = await mkdtemp(join(tmpdir(), "forge-sampler-cap-")); | |
| 28 | app = createApp({ dataDir: dir, port: 0 }); | |
| 29 | servers = await serve(app); | |
| 30 | const r = await createUser(app.db, { | |
| 31 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 32 | password: "a-good-passphrase", adminIfFirst: true, | |
| 33 | }); | |
| 34 | if (!r.ok) throw new Error(r.error); | |
| 35 | adminId = r.value.id; | |
| 36 | token = createToken(app.db, r.value.id, "t", "admin").token; | |
| 37 | }); | |
| 38 | afterAll(async () => { | |
| 39 | servers.stop(); | |
| 40 | app.close(); | |
| 41 | await rm(dir, { recursive: true, force: true }); | |
| 42 | }); | |
| 43 | ||
| 44 | const rows = (target: string, step = 15) => | |
| 45 | app.db.query<{ at: string }, [string, number]>( | |
| 46 | "SELECT at FROM machine_sample WHERE target = ?1 AND step = ?2 ORDER BY at", | |
| 47 | ).all(target, step).map((r) => r.at); | |
| 48 | ||
| 49 | describe("the sampler cap", () => { | |
| 50 | test("the cap is a named setting with its reason", () => { | |
| 51 | const l = LIMITS["sampler.rows_per_source"]; | |
| 52 | expect(l.default).toBeGreaterThanOrEqual(5_760); | |
| 53 | expect(l.reason.length).toBeGreaterThan(0); | |
| 54 | expect(l.reason).not.toContain("\n"); | |
| 55 | }); | |
| 56 | ||
| 57 | test("the oldest rows of a source go, the newest stay, other sources and roll-ups are untouched", () => { | |
| 58 | const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, ?, ?, 1)"); | |
| 59 | for (let i = 0; i < 10; i++) put.run(at(i), 15, "a"); | |
| 60 | for (let i = 0; i < 4; i++) put.run(at(i), 15, "b"); | |
| 61 | put.run(at(0), 300, "a"); | |
| 62 | const before = droppedRows(); | |
| 63 | expect(capRows(app.db, 6)).toBe(4); | |
| 64 | expect(rows("a")).toEqual([4, 5, 6, 7, 8, 9].map(at)); | |
| 65 | expect(rows("b")).toHaveLength(4); | |
| 66 | expect(rows("a", 300)).toHaveLength(1); | |
| 67 | expect(droppedRows() - before).toBe(4); | |
| 68 | expect(capRows(app.db, 6)).toBe(0); | |
| 69 | }); | |
| 70 | ||
| 71 | test("a tick applies the limit once per 5-min window", () => { | |
| 72 | app.db.run("DELETE FROM machine_sample"); | |
| 73 | const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, 15, 'old', 1)"); | |
| 74 | for (let i = 0; i < 300; i++) put.run(at(i)); | |
| 75 | expect(setLimit(app.db, "sampler.rows_per_source", 240, adminId).ok).toBe(true); | |
| 76 | const s = startSampler(app, { everyMs: 3_600_000, cells: () => [], paths: { cgroup: dir, proc: dir, sys: dir } }); | |
| 77 | try { | |
| 78 | s.tick(T0 + 400 * 15_000); | |
| 79 | } finally { | |
| 80 | s.stop(); | |
| 81 | } | |
| 82 | expect(rows("old")).toHaveLength(240); | |
| 83 | expect(rows("old")[0]).toBe(at(60)); | |
| 84 | }); | |
| 85 | ||
| 86 | test("the machine board reads the count of dropped rows", async () => { | |
| 87 | app.db.run("DELETE FROM machine_sample"); | |
outdated
This writes the module-level |
||
| 88 | const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores) VALUES (?, 15, 'c', 1)"); | |
| 89 | for (let i = 0; i < 10; i++) put.run(at(i)); | |
| 90 | const before = droppedRows(); | |
| 91 | expect(capRows(app.db, 6)).toBe(4); | |
| 92 | const res = await fetch(servers.url + "/boards/data?source=machine&aggregate=count&by=target", { | |
| 93 | headers: { authorization: `Bearer ${token}` }, | |
| 94 | }); | |
| 95 | expect(await res.text()).toContain(`${before + 4} oldest rows dropped`); | |
| 96 | }); | |
| 97 | }); | |
| 98 | ||
| 99 | test("the terminal's System dialog says the dropped count in the board's words, and nothing at zero", () => { | |
| 100 | expect(samplerLines(0)).toEqual([]); | |
| 101 | expect(samplerLines(6_000)).toEqual(["6,000 oldest rows dropped since this forge started."]); | |
| 102 | }); |
The reason admits a day is 5,760 rows, yet the default is 6,000 and the age rule already prunes at a day, so the cap is dead under defaults. Say in the reason that it is a backstop only, or choose a default that bites.