trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns13g rEventsFlows

No test's verdict depends on how full the check machine's disk is #2253

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/2248-no-test-s-verdict-depends-on-how-full-th → main 2 commits fork 1 thread · 1 unresolved

  • Merged. scout/2248-no-test-s-verdict-depends-on-how-full-th was merged into main as 5f12a23.

Review

Sign in to review this change.
7 files changed +80 −48
src/actions/health.ts +1
@@ -43,6 +43,7 @@ export function forgeVitals(app: App): Vitals {
4343 return vitals({
4444 db: app.db,
4545 dataDir: app.config.dataDir,
46 ...(app.config.disk ? { disk: app.config.disk } : {}),
4647 envs,
4748 signals: app.signals,
4849 forgeUrl: app.config.url ?? `http://localhost:${app.config.port}`,
src/app.ts +2
@@ -31,6 +31,8 @@ import { closeReadersUnder } from "./git/read.ts";
3131 export interface ForgeConfig {
3232 /** Everything lives here: index db, bare repos, the internal socket. */
3333 dataDir: string;
34 /** The disk probe vitals reads; the volume under `dataDir` when unset. Tests give it a reading. */
35 disk?: (dir: string) => { freeBytes: number; totalBytes: number };
3436 /** @default 3000 */
3537 port?: number;
3638 /** The hostname clone URLs are built from. @default "localhost" */
src/kernel/vitals.ts +18 −5
@@ -95,6 +95,21 @@ export interface Vitals {
9595 readonly at: string;
9696 }
9797
98 /** What a volume holds and has free. */
99 export interface DiskReading {
100 readonly freeBytes: number;
101 readonly totalBytes: number;
102 }
103
104 /** The volume under `dir`, as the operating system reports it. */
105 export function statDisk(dir: string): DiskReading {
106 const fs = statfsSync(dir);
107 return {
108 freeBytes: Number(fs.bavail) * Number(fs.bsize),
109 totalBytes: Number(fs.blocks) * Number(fs.bsize),
110 };
111 }
112
98113 export interface VitalsInput {
99114 readonly db: Database;
100115 /** The directory holding the forge's data: its volume is the one measured. */
@@ -104,6 +119,8 @@ export interface VitalsInput {
104119 readonly now?: number;
105120 /** The address people reach this forge at, named in the drill commands. */
106121 readonly forgeUrl?: string;
122 /** Reads the disk under `dataDir`; `statDisk` unless a test gives it a reading. */
123 readonly disk?: (dir: string) => DiskReading;
107124 /** The free-disk share below which the disk is a need; `DISK_FREE_MIN` unless a test says. */
108125 readonly diskFreeMin?: number;
109126 /** One forge's own signals (`App.signals`). */
@@ -260,11 +277,7 @@ export function vitals(input: VitalsInput): Vitals {
260277 degraded.push({ cause, sentence, since, href });
261278 };
262279
263 const fs = statfsSync(input.dataDir);
264 const disk = {
265 freeBytes: Number(fs.bavail) * Number(fs.bsize),
266 totalBytes: Number(fs.blocks) * Number(fs.bsize),
267 };
280 const disk = (input.disk ?? statDisk)(input.dataDir);
268281 if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN))
269282 need(
270283 "disk",
src/main.ts +1
@@ -446,6 +446,7 @@ const tickAsks = async (): Promise<void> => {
446446 vitals({
447447 db: app.db,
448448 dataDir: app.config.dataDir,
449 ...(app.config.disk ? { disk: app.config.disk } : {}),
449450 signals: app.signals,
450451 forgeUrl,
451452 snapshots: snapshotPlace(app),
test/domains.test.ts +4 −2
@@ -12,6 +12,8 @@ import { DOMAINS, domainRow } from "../src/kernel/domains.ts";
1212 import { vitals } from "../src/kernel/vitals.ts";
1313 import { recordVerdict } from "../src/snapshot/verdict.ts";
1414
15 /** A disk with room: no verdict here depends on the machine's volume. */
16 const roomy = () => ({ freeBytes: 95, totalBytes: 100 });
1517 let dataDir: string;
1618 let app: App;
1719 let repoId = "";
@@ -36,7 +38,7 @@ const said = (key: string): string[] => {
3638 return [...d.says];
3739 };
3840 const read = (now = Date.now()) =>
39 vitals({ db: app.db, signals: app.signals, dataDir, now }).needs.join("\n");
41 vitals({ db: app.db, signals: app.signals, dataDir, now, disk: roomy }).needs.join("\n");
4042
4143 test("seven domains, each complete", () => {
4244 expect(DOMAINS.map((d) => d.key)).toEqual([
@@ -56,7 +58,7 @@ test("the doc's table is the data", () => {
5658 });
5759
5860 test("a nearly full disk says the state's words", () => {
59 const v = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 1.1 });
61 const v = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, diskFreeMin: 1.1 });
6062 for (const s of said("state")) expect(v.needs.join("\n")).toContain(s);
6163 });
6264
test/tui-screens.test.ts +7 −4
@@ -126,6 +126,9 @@ function boot(
126126 };
127127 }
128128
129 /** A disk with room: the verdict must not depend on the machine's volume. */
130 const roomy = () => ({ freeBytes: 95, totalBytes: 100 });
131
129132 describe("the terminal UI, against a real forge", () => {
130133 let dataDir: string;
131134 let forge: Forge;
@@ -148,7 +151,7 @@ describe("the terminal UI, against a real forge", () => {
148151
149152 beforeAll(async () => {
150153 dataDir = await mkdtemp(join(tmpdir(), "forge-tui-"));
151 forge = createApp({ dataDir, port: 0 });
154 forge = createApp({ dataDir, port: 0, disk: roomy });
152155 const a = await signUp(forge, {
153156 username: "alice",
154157 email: "[email protected]",
@@ -587,7 +590,7 @@ describe("a forge with no accounts", () => {
587590
588591 beforeAll(async () => {
589592 dataDir = await mkdtemp(join(tmpdir(), "forge-tui-first-"));
590 forge = createApp({ dataDir, port: 0 });
593 forge = createApp({ dataDir, port: 0, disk: roomy });
591594 });
592595
593596 afterAll(async () => {
@@ -730,7 +733,7 @@ describe("the forge's health in the terminal", () => {
730733 const dataDir = await mkdtemp(join(tmpdir(), "forge-tui-health-"));
731734 const guard = await mkdtemp(join(tmpdir(), "forge-tui-guard-"));
732735 process.env["FORGE_GUARD_DIR"] = guard;
733 const forge = createApp({ dataDir, port: 0 });
736 const forge = createApp({ dataDir, port: 0, disk: roomy });
734737 try {
735738 const a = await signUp(forge, {
736739 username: "chief",
@@ -796,7 +799,7 @@ describe("the forge's health in the terminal", () => {
796799 const dataDir = await mkdtemp(join(tmpdir(), "forge-tui-snapvol-"));
797800 const guard = await mkdtemp(join(tmpdir(), "forge-tui-guard-"));
798801 process.env["FORGE_GUARD_DIR"] = guard;
799 const forge = createApp({ dataDir, port: 0 });
802 const forge = createApp({ dataDir, port: 0, disk: roomy });
800803 const store = dirStore(join(dataDir, "snapshots"));
801804 let svc = startSnapshots(forge, store, { version: "t", nightly: false });
802805 try {
test/vitals.test.ts +47 −37
@@ -38,6 +38,8 @@ const mint = async (username: string, scope: string) =>
3838 })
3939 ).json()) as { token: string }
4040 ).token;
41 /** A disk with room: no test but the disk's own depends on the machine's volume. */
42 const roomy = () => ({ freeBytes: 95, totalBytes: 100 });
4143 const health = (token?: string) =>
4244 fetch(servers.url + "/api/v1/health", {
4345 headers: token ? { authorization: `Bearer ${token}` } : {},
@@ -45,7 +47,7 @@ const health = (token?: string) =>
4547
4648 beforeAll(async () => {
4749 dataDir = await mkdtemp(join(tmpdir(), "forge-vitals-"));
48 app = createApp({ dataDir, port: 0 });
50 app = createApp({ dataDir, port: 0, disk: roomy });
4951 servers = await serve(app);
5052 for (const username of ["alice", "bob"])
5153 await post("/api/v1/users", {
@@ -107,11 +109,11 @@ test("a full queue shows in queue, and an old one in needs", async () => {
107109 app.db.query("DELETE FROM run").run();
108110 const old = Date.now() - 3_600_000;
109111 for (let i = 0; i < 3; i++) createRun(app.db, { repoId, kind: "agent" });
110 const v = vitals({ db: app.db, signals: app.signals, dataDir });
112 const v = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
111113 expect(v.queue).toEqual({ agent: 3 });
112114 // Process-wide signals (errors, refused calls) from other suites may speak too.
113115 expect(v.needs.join(" ")).not.toContain("queue");
114 const later = vitals({ db: app.db, signals: app.signals, dataDir, now: old + 2 * 3_600_000 + 3_600_000 });
116 const later = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: old + 2 * 3_600_000 + 3_600_000 });
115117 expect(later.ok).toBe(false);
116118 expect(later.needs.join(" ")).toContain("agent queue");
117119 });
@@ -132,7 +134,7 @@ test("a running run with a lapsed lease is stuck, and needs says so", async () =
132134 app.db
133135 .query("UPDATE run SET heartbeat_at=?2 WHERE id=?1")
134136 .run(run.id, new Date().toISOString());
135 expect(vitals({ db: app.db, signals: app.signals, dataDir }).stuck).toEqual([]);
137 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).stuck).toEqual([]);
136138 });
137139
138140 test("the number of queries does not grow with runs", () => {
@@ -145,7 +147,7 @@ test("the number of queries does not grow with runs", () => {
145147 return typeof v === "function" ? v.bind(t) : v;
146148 },
147149 });
148 vitals({ db: counting, signals: app.signals, dataDir });
150 vitals({ db: counting, signals: app.signals, dataDir, disk: roomy });
149151 return n;
150152 };
151153 const before = count();
@@ -182,8 +184,8 @@ test("errors are counted, bounded, redacted, and a burst is said", async () => {
182184 expect(seen.latest.length).toBe(ERROR_RING);
183185 expect(seen.lastHour).toBe(ERROR_RING + 51);
184186 expect(seen.lastHour).toBeGreaterThanOrEqual(ERROR_BURST);
185 expect(vitals({ db: app.db, signals: app.signals, dataDir }).needs.some((n) => n.includes("errors in the last hour"))).toBe(true);
186 expect(vitals({ db: app.db, signals: app.signals, dataDir, now: Date.now() + 2 * 3_600_000 }).errors.lastHour).toBe(0);
187 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).needs.some((n) => n.includes("errors in the last hour"))).toBe(true);
188 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: Date.now() + 2 * 3_600_000 }).errors.lastHour).toBe(0);
187189 } finally {
188190 console.error = orig;
189191 clearErrors();
@@ -196,10 +198,10 @@ test("the lane: depth, the running check and its age, the median of the last fin
196198 const now = Date.now();
197199 const iso = (ms: number) => new Date(ms).toISOString();
198200 // Empty lane: no line.
199 expect(vitals({ db: app.db, signals: app.signals, dataDir, now }).lane).toEqual({
201 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).lane).toEqual({
200202 queued: 0, running: null, medianMinutes: null, clearsInMinutes: null,
201203 });
202 expect(laneLine(vitals({ db: app.db, signals: app.signals, dataDir, now }).lane, now)).toBeNull();
204 expect(laneLine(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).lane, now)).toBeNull();
203205 // Three finished checks of 10, 20 and 30 minutes: the median is 20.
204206 for (const min of [10, 20, 30]) {
205207 const r = createRun(app.db, { repoId, kind: "check" });
@@ -212,7 +214,7 @@ test("the lane: depth, the running check and its age, the median of the last fin
212214 .query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?3 WHERE id=?1")
213215 .run(live.id, iso(now - 6 * 60_000), iso(now));
214216 for (let i = 0; i < 6; i++) createRun(app.db, { repoId, kind: "check" });
215 const lane = vitals({ db: app.db, signals: app.signals, dataDir, now }).lane;
217 const lane = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).lane;
216218 expect(lane.queued).toBe(6);
217219 expect(lane.running?.since).toBe(iso(now - 6 * 60_000));
218220 expect(lane.medianMinutes).toBe(20);
@@ -225,7 +227,7 @@ test("the lane: depth, the running check and its age, the median of the last fin
225227 expect(body.lane.queued).toBe(6);
226228 expect(body.lane.medianMinutes).toBe(20);
227229 // Waiting past thirty minutes names the cause, not only the symptom.
228 const later = vitals({ db: app.db, signals: app.signals, dataDir, now: now + 3_600_000 });
230 const later = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: now + 3_600_000 });
229231 const cause = later.needs.find((n) => n.includes("one lane"));
230232 expect(cause).toMatch(/^6 checks wait behind one lane; the oldest since \d\d:\d\d; the lane runs nothing\.$/);
231233 expect(cause).not.toContain("Lane:");
@@ -234,7 +236,7 @@ test("the lane: depth, the running check and its age, the median of the last fin
234236 test("an unauthenticated internal call is refused, logged and flagged; the probe is not", async () => {
235237 app.signals.refused.clear();
236238 const flagged = () =>
237 vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir }).needs.filter((n) =>
239 vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir, disk: roomy }).needs.filter((n) =>
238240 n.includes("internal call"),
239241 );
240242 const call = (path: string) =>
@@ -258,10 +260,10 @@ test("two forges in one process keep their own error and refusal signals", async
258260 (await import("../src/kernel/log.ts")).clearErrors();
259261 app.signals.refused.clear();
260262 const dirB = await mkdtemp(join(tmpdir(), "forge-vitals-b-"));
261 const other = createApp({ dataDir: dirB, port: 0 });
263 const other = createApp({ dataDir: dirB, port: 0, disk: roomy });
262264 try {
263265 const needs = (a: App) =>
264 vitals({ db: a.db, dataDir: a.config.dataDir, signals: a.signals }).needs;
266 vitals({ db: a.db, dataDir: a.config.dataDir, signals: a.signals, disk: roomy }).needs;
265267 expect(needs(app)).toEqual([]);
266268 expect(needs(other)).toEqual([]);
267269 for (let i = 0; i < ERROR_BURST; i++) other.log.error("boom");
@@ -275,7 +277,7 @@ test("two forges in one process keep their own error and refusal signals", async
275277 expect(needs(app).join(" ")).toContain("errors in the last hour");
276278 // A forge born later starts clean.
277279 const dirC = await mkdtemp(join(tmpdir(), "forge-vitals-c-"));
278 const later = createApp({ dataDir: dirC, port: 0 });
280 const later = createApp({ dataDir: dirC, port: 0, disk: roomy });
279281 expect(needs(later)).toEqual([]);
280282 later.close();
281283 await rm(dirC, { recursive: true, force: true });
@@ -294,7 +296,7 @@ test("degraded names each cause, stable, with when it began", async () => {
294296 try {
295297 clearErrors();
296298 app.signals.refused.clear();
297 const base = vitals({ db: app.db, signals: app.signals, dataDir });
299 const base = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
298300 expect(base.degraded.map((d) => d.cause)).not.toContain("errors");
299301
300302 // queue: since is the oldest queued run's createdAt.
@@ -304,7 +306,7 @@ test("degraded names each cause, stable, with when it began", async () => {
304306 app.db.query("SELECT MIN(created_at) AS m FROM run WHERE state='queued' AND kind='agent'").get() as { m: string }
305307 ).m;
306308 const later = Date.now() + 2 * QUEUE_WAIT_MAX_MS;
307 const q = vitals({ db: app.db, signals: app.signals, dataDir, now: later }).degraded.find((d) => d.cause === "queue:agent");
309 const q = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: later }).degraded.find((d) => d.cause === "queue:agent");
308310 expect(q?.since).toBe(new Date(oldest).toISOString());
309311 expect(q?.href).toBe("/settings/capacity");
310312
@@ -314,7 +316,7 @@ test("degraded names each cause, stable, with when it began", async () => {
314316 app.db
315317 .query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?2 WHERE id=?1")
316318 .run(run.id, moved);
317 const s = vitals({ db: app.db, signals: app.signals, dataDir });
319 const s = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
318320 const entry = s.degraded.find((d) => d.cause === `stuck:${run.id}`);
319321 expect(entry?.since).toBe(moved);
320322 expect(entry?.href).toBe(`/alice/r/runs/${run.id}`);
@@ -324,37 +326,37 @@ test("degraded names each cause, stable, with when it began", async () => {
324326 // errors: no recorded start, so since is at; the cause is stable across calls.
325327 const log = createLog("probe");
326328 for (let i = 0; i < ERROR_BURST; i++) log.error(`boom ${i}`);
327 const a = vitals({ db: app.db, signals: app.signals, dataDir });
329 const a = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
328330 const e = a.degraded.find((d) => d.cause === "errors");
329 const later2 = vitals({ db: app.db, signals: app.signals, dataDir, now: Date.now() + 5000 });
331 const later2 = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: Date.now() + 5000 });
330332 expect(e?.since).toBe(app.signals.errors.recent(100).oldest!);
331333 expect(later2.degraded.find((d) => d.cause === "errors")?.since).toBe(e!.since);
332334 expect(e?.href).toBe("/settings/capacity#errors");
333335
334336 // internal: refused calls.
335337 await fetchInternal(app.hookAddress.current, "/internal/hook", { method: "POST" });
336 const i = vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "internal");
338 const i = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).degraded.find((d) => d.cause === "internal");
337339 expect(i?.href).toBe("/settings/capacity#internal");
338 const i2 = vitals({ db: app.db, signals: app.signals, dataDir, now: Date.now() + 5000 }).degraded.find((d) => d.cause === "internal");
340 const i2 = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: Date.now() + 5000 }).degraded.find((d) => d.cause === "internal");
339341 expect(i2?.since).toBe(i!.since);
340342 expect(i!.since).toBe(app.signals.refused.recent().oldest!);
341343
342344 // a cause that clears is forgotten: when it returns it began again.
343345 const first = e!.since;
344346 clearErrors();
345 expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.some((d) => d.cause === "errors")).toBe(false);
347 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).degraded.some((d) => d.cause === "errors")).toBe(false);
346348 for (let k = 0; k < ERROR_BURST; k++) log.error(`again ${k}`);
347 const again = vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "errors");
349 const again = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).degraded.find((d) => d.cause === "errors");
348350 expect(again?.since).toBe(app.signals.errors.recent(100).oldest!);
349351 expect(again!.since > first).toBe(true);
350352
351353 // a caller's `now` never moves a fallback start: it is the real clock's.
352 const early = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 2, now: 1_000 });
354 const early = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, diskFreeMin: 2, now: 1_000 });
353355 const d0 = early.degraded.find((d) => d.cause === "disk")!;
354356 expect(Date.parse(d0.since)).toBeGreaterThan(Date.parse("2020-01-01"));
355357
356358 // every need has one matching sentence in degraded.
357 const all = vitals({ db: app.db, signals: app.signals, dataDir });
359 const all = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
358360 expect(new Set(all.degraded.map((d) => d.sentence))).toEqual(new Set(all.needs));
359361
360362 // the health API carries the same list.
@@ -368,17 +370,17 @@ test("degraded names each cause, stable, with when it began", async () => {
368370 });
369371
370372 test("disk: since rests on first sight, holds across calls, resets after clearing", async () => {
371 expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.some((d) => d.cause === "disk")).toBe(false);
372 const a = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
373 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).degraded.some((d) => d.cause === "disk")).toBe(false);
374 const a = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
373375 expect(a?.href).toBe("/settings/capacity");
374376 await Bun.sleep(5);
375 const b = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
377 const b = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
376378 expect(b?.since).toBe(a!.since);
377 expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.some((d) => d.cause === "disk")).toBe(false);
379 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).degraded.some((d) => d.cause === "disk")).toBe(false);
378380 await Bun.sleep(5);
379 const c = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
381 const c = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, diskFreeMin: 2 }).degraded.find((d) => d.cause === "disk");
380382 expect(c!.since > a!.since).toBe(true);
381 vitals({ db: app.db, signals: app.signals, dataDir });
383 vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy });
382384 });
383385
384386 test("since is said as a person would", async () => {
@@ -395,10 +397,10 @@ test("since is said as a person would", async () => {
395397
396398 test("since is kept per forge: one forge's cause neither lends nor loses its start to another", async () => {
397399 const dirB = await mkdtemp(join(tmpdir(), "forge-vitals-since-"));
398 const other = createApp({ dataDir: dirB, port: 0 });
400 const other = createApp({ dataDir: dirB, port: 0, disk: roomy });
399401 try {
400402 const disk = (a: App, diskFreeMin: number) =>
401 vitals({ db: a.db, dataDir: a.config.dataDir, signals: a.signals, diskFreeMin })
403 vitals({ db: a.db, dataDir: a.config.dataDir, signals: a.signals, diskFreeMin, disk: roomy })
402404 .degraded.find((d) => d.cause === "disk");
403405 const first = disk(app, 2)!;
404406 expect(first).toBeDefined();
@@ -428,7 +430,7 @@ test("a lane older than its budget is degraded with its cause; a silent run is s
428430 app.db.query("UPDATE run SET created_at=?2 WHERE id=?1").run(r.id, iso(now - (9 + i * 2) * 60_000));
429431 }
430432 // The oldest has waited 13 minutes: past the lane's ten.
431 const v = vitals({ db: app.db, signals: app.signals, dataDir, now });
433 const v = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now });
432434 const lane = v.degraded.find((d) => d.cause === "queue:check");
433435 expect(lane?.sentence).toMatch(
434436 /^3 checks wait behind one lane; the oldest since \d\d:\d\d; the lane runs a check for 31 min \(limit 60\)\.$/,
@@ -436,10 +438,18 @@ test("a lane older than its budget is degraded with its cause; a silent run is s
436438 expect(v.ok).toBe(false);
437439 // Five minutes in, the oldest is under its budget: nothing to say.
438440 app.db.query("UPDATE run SET created_at=?1 WHERE state='queued'").run(iso(now - 5 * 60_000));
439 expect(vitals({ db: app.db, signals: app.signals, dataDir, now }).degraded.map((d) => d.cause)).toEqual([]);
441 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).degraded.map((d) => d.cause)).toEqual([]);
440442 // A run whose heart stopped four minutes ago is stuck, in words of the budget.
441443 app.db.query("UPDATE run SET heartbeat_at=?2 WHERE id=?1").run(live.id, iso(now - 4 * 60_000));
442 const stuck = vitals({ db: app.db, signals: app.signals, dataDir, now }).degraded.find((d) => d.cause === `stuck:${live.id}`);
444 const stuck = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).degraded.find((d) => d.cause === `stuck:${live.id}`);
443445 expect(stuck?.sentence).toBe(`Run ${live.id} is stuck: no heartbeat for 4 min (limit 3).`);
444446 app.db.query("DELETE FROM run").run();
445447 });
448
449 test("only the disk need sees the disk: a 5% reading raises it, and nothing else does", () => {
450 const low = () => ({ freeBytes: 5, totalBytes: 100 });
451 const v = vitals({ db: app.db, signals: app.signals, dataDir, disk: low });
452 expect(v.degraded.map((d) => d.cause)).toEqual(["disk"]);
453 expect(v.needs).toEqual(["Disk is nearly full: 5% free."]);
454 expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy }).needs).toEqual([]);
455 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs