Public: anyone can read this; the crew can change it.
The backup alarm is a dead man's switch: a missed check is itself red #8980
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/8959-the-backup-alarm-is-a-dead-man-s-switch → main 3 commits fork 4 threads · 4 unresolved
6 files
changed +121
−9
docs/features/backup-truth.md +2 −1
| @@ -2,7 +2,8 @@ | ||
| 2 | 2 | |
| 3 | 3 | - **Read from the bucket.** `src/snapshot/bucket.ts` lists the primary and mirror stores every 5 minutes, in the serving process (no CronJob to die). Home's backup tile, vitals (one `backup` card) and the MCP `backups` tool's `verdict` all call `backupVerdict`. |
| 4 | 4 | - **Three states.** `green`, `red`, and `unknown`: nothing read yet since boot, or no restore drill yet. Unknown is never a calm tile and never an alarm; only red raises the one `backup` card. A store error is scrubbed of keys and presigned queries before it is kept. |
| 5 | - **Red when:** the newest snapshot is over 26 h old; a lane cannot be read or holds nothing; the newest restore drill failed, did not run, or brought back no rows; or the check itself has not run for 20 min (dead man's switch). | |
| 5 | - **Red when:** the newest snapshot is over 26 h old; a lane cannot be read or holds nothing; the newest restore drill failed, did not run, or brought back no rows; or the check itself has not run for 20 min (dead man's switch). The reason says when it last ran: `The backup check has not run for 25 min (last ran 2026-11-02T08:00:00Z); it is the alarm, and it is down.` | |
| 6 | - **The run record.** Every run of the check appends one JSON line (`at`, and each lane's `ok` and `newestAt`) to `backup-checks.jsonl` under the data directory. The count is kept in memory; once the file holds 400 runs it is cut back to the last 200. A disk that refuses the write never stops the check. The verdict carries `runs` and `checkLine` (`The backup check last ran <ISO> (3 min ago); 42 runs on record.`), shown in the Resilience dialog, the home tile's title and the `backup` card, so a person can see the check is alive. | |
| 6 | 7 | - **Catch-up.** A boot with the newest snapshot older than the last 08:00 UTC slot takes one two minutes later (`snapshotDue`). A snapshot cut short by a release has no manifest, so it counts as lost. |
| 7 | 8 | - **In the terminal.** The Resilience dialog (palette: Resilience, admins) opens with the same verdict, each lane's newest, and every reason. On the web it is home's backup tile and `/settings/backups`; for an agent, `verdict` in the MCP `backups` tool. |
| 8 | 9 | - **Off the serving thread.** `VACUUM INTO` runs in a worker (`vacuum.ts`). |
src/kernel/vitals.ts +1 −1
| @@ -479,7 +479,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 479 | 479 | // The bucket itself, read by this process: late, unreadable, a failed drill, or a check that stopped is one card. |
| 480 | 480 | if (isWatched(input.dataDir)) { |
| 481 | 481 | const backup = backupVerdict(input.dataDir, lastDrill(input.db), now); |
| 482 | if (backup.state === "red") need("backup", backup.reasons.join(" "), backup.checkedAt ?? undefined, "/settings/backups"); | |
| 482 | if (backup.state === "red") need("backup", [...backup.reasons, ...(backup.reasons.some((r) => r.includes("last ran")) ? [] : [backup.checkLine])].join(" "), backup.checkedAt ?? undefined, "/settings/backups"); | |
| 483 | 483 | } |
| 484 | 484 | |
| 485 | 485 | // A snapshot that did not land off the box, or a mirror a day behind, is a card until it does. |
src/snapshot/bucket.ts +53 −4
| @@ -13,6 +13,8 @@ | ||
| 13 | 13 | * without touching the network. Plain data in, plain data out. |
| 14 | 14 | */ |
| 15 | 15 | |
| 16 | import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; | |
| 17 | import { join } from "node:path"; | |
| 16 | 18 | import type { DrillResult } from "./drill.ts"; |
| 17 | 19 | import { isOffBox } from "./off_box.ts"; |
| 18 | 20 | import { listSnapshots } from "./snapshot.ts"; |
| @@ -63,6 +65,44 @@ export interface BackupVerdict { | ||
| 63 | 65 | readonly newestAt: string | null; |
| 64 | 66 | /** When the bucket was last read; null when it has not been. */ |
| 65 | 67 | readonly checkedAt: string | null; |
| 68 | /** Runs of the check on record (`backup-checks.jsonl`). */ | |
| 69 | readonly runs: number; | |
| 70 | /** When the check last ran and how many runs are on record, as a person reads it; the proof the check is alive. */ | |
| 71 | readonly checkLine: string; | |
| 72 | } | |
| 73 | ||
| 74 | /** Where each run of the check is written down, under the data directory: one JSON line per run. */ | |
| 75 | export const CHECK_LOG = "backup-checks.jsonl"; | |
| 76 | /** The log keeps this many runs; it is cut back to them once it holds twice as many. */ | |
| 77 | export const CHECK_LOG_KEEP = 200; | |
| 78 | ||
| 79 | /** Runs in each log, counted in memory so a run never reads the file to know its size. */ | |
outdated
Reads and splits the whole file on every run just to decide on trimming. Keep a count in memory, or trim only when size passes a bound. |
||
| 80 | const runCounts = new Map<string, number>(); | |
| 81 | ||
| 82 | function countRuns(dataDir: string): number { | |
| 83 | try { | |
| 84 | return readFileSync(join(dataDir, CHECK_LOG), "utf8").split("\n").filter(Boolean).length; | |
| 85 | } catch { | |
| 86 | return 0; | |
| 87 | } | |
| 88 | } | |
| 89 | ||
| 90 | /** Writes one run down; a disk that refuses must not stop the check, so it never throws. */ | |
| 91 | export function recordRun(dataDir: string, at: string, lanes: readonly LaneRead[]): void { | |
outdated
Exported, but only the test calls it. No page, API or terminal screen shows the run record. Wire it into the tile/vitals card and the TUI backups screen, or remove the log. |
||
| 92 | try { | |
| 93 | mkdirSync(dataDir, { recursive: true }); | |
| 94 | const file = join(dataDir, CHECK_LOG); | |
| 95 | const had = runCounts.get(dataDir) ?? countRuns(dataDir); | |
| 96 | appendFileSync(file, `${JSON.stringify({ at, lanes: lanes.map((l) => ({ lane: l.lane, ok: l.error === undefined, newestAt: l.newestAt })) })}\n`); | |
| 97 | let n = had + 1; | |
| 98 | if (n > CHECK_LOG_KEEP * 2) { | |
| 99 | writeFileSync(file, `${readFileSync(file, "utf8").split("\n").filter(Boolean).slice(-CHECK_LOG_KEEP).join("\n")}\n`); | |
| 100 | n = CHECK_LOG_KEEP; | |
| 101 | } | |
| 102 | runCounts.set(dataDir, n); | |
| 103 | } catch { | |
| 104 | /* the verdict does not depend on the log */ | |
| 105 | } | |
| 66 | 106 | } |
| 67 | 107 | |
| 68 | 108 | const reads = new Map<string, { lanes: LaneRead[]; checkedAt: string; since: string }>(); |
| @@ -154,7 +194,7 @@ export function judge( | ||
| 154 | 194 | drill: DrillResult | undefined, |
| 155 | 195 | now: number, |
| 156 | 196 | booted: number = now, |
| 157 | ): BackupVerdict { | |
| 197 | ): Omit<BackupVerdict, "runs" | "checkLine"> { | |
| 158 | 198 | const reasons: string[] = []; |
| 159 | 199 | /** A few words for each reason, for a tile's note. */ |
| 160 | 200 | const causes: string[] = []; |
| @@ -167,7 +207,7 @@ export function judge( | ||
| 167 | 207 | reasons.push( |
| 168 | 208 | checkedAt === null |
| 169 | 209 | ? "The backup check has never run, so nothing says a backup exists." |
| 170 | : `The backup check has not run for ${ageWords(sinceCheck)}; it is the alarm, and it is down.`, | |
| 210 | : `The backup check has not run for ${ageWords(sinceCheck)} (last ran ${checkedAt}); it is the alarm, and it is down.`, | |
| 171 | 211 | ); |
| 172 | 212 | } else if (checkedAt === null) unknowns.push("The bucket has not been read yet, so nothing is known about the backups."); |
| 173 | 213 | for (const r of state?.lanes ?? []) { |
| @@ -246,7 +286,13 @@ export function judge( | ||
| 246 | 286 | /** The verdict now, from what the watch last read. */ |
| 247 | 287 | export function backupVerdict(dataDir: string, drill: DrillResult | undefined, now = Date.now()): BackupVerdict { |
| 248 | 288 | const booted = Date.parse(watching.get(dataDir) ?? "") || now; |
| 249 | return judge(reads.get(dataDir), drill, now, booted); | |
| 289 | const v = judge(reads.get(dataDir), drill, now, booted); | |
| 290 | const runs = runCounts.get(dataDir) ?? 0; | |
| 291 | const checkLine = | |
| 292 | v.checkedAt === null | |
| 293 | ? "The backup check has not run yet." | |
| 294 | : `The backup check last ran ${v.checkedAt} (${ageWords(now - Date.parse(v.checkedAt))} ago); ${runs} runs on record.`; | |
| 295 | return { ...v, runs, checkLine }; | |
| 250 | 296 | } |
| 251 | 297 | |
| 252 | 298 | /** |
| @@ -264,6 +310,7 @@ export function watchBucket( | ||
| 264 | 310 | declared: (now: number) => Promise<readonly LaneRead[]> = async () => [], |
| 265 | 311 | ): { stop: () => void; pass: () => Promise<void> } { |
| 266 | 312 | watching.set(dataDir, new Date().toISOString()); |
| 313 | runCounts.set(dataDir, countRuns(dataDir)); | |
| 267 | 314 | let stopped = false; |
| 268 | 315 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 269 | 316 | const pass = async (): Promise<void> => { |
| @@ -273,7 +320,9 @@ export function watchBucket( | ||
| 273 | 320 | Promise.all(stores.map((l) => readLane(l.lane, l.store, now, targets))), |
| 274 | 321 | declared(now).catch(() => []), |
| 275 | 322 | ]); |
| 276 | keepReads(dataDir, [...own, ...more]); | |
| 323 | const all = [...own, ...more]; | |
| 324 | keepReads(dataDir, all); | |
| 325 | recordRun(dataDir, new Date().toISOString(), all); | |
| 277 | 326 | }; |
| 278 | 327 | const next = (ms: number): void => { |
| 279 | 328 | if (stopped) return; |
src/tui/screens/resilience.ts +1
| @@ -81,6 +81,7 @@ export function verdictLines(f: Session): string[] { | ||
| 81 | 81 | const mark = v.state === "red" ? "RED" : v.state === "unknown" ? "unknown" : "ok"; |
| 82 | 82 | return [ |
| 83 | 83 | ...wrap(`Backups ${mark}: ${v.line}`, INNER), |
| 84 | ...wrap(` ${v.checkLine}`, INNER), | |
| 84 | 85 | ...v.lanes.flatMap((l) => wrap(` ${l.state === "red" ? "!" : " "} ${l.says}`, INNER)), |
| 85 | 86 | ...[...v.reasons, ...v.unknowns].flatMap((r) => wrap(` - ${r}`, INNER)), |
| 86 | 87 | ]; |
src/web/studio/kpi_row.ts +1 −1
| @@ -145,7 +145,7 @@ export function kpiRowFor(k: Kpis, live: { sha: string; at: string } | null, now | ||
| 145 | 145 | k.backup?.state === "unknown" && k.backupAt === null ? "—" : backupAge(k.backupAt, now), |
| 146 | 146 | backupFull(k.backupAt), |
| 147 | 147 | k.backup ? k.backup.note : k.backupAt === null ? "no backup off the machine yet" : overdue ? "overdue, more than 24 h" : "off the machine", |
| 148 | k.backup ? (k.backup.reasons[0] ?? k.backup.unknowns[0] ?? k.backup.line) : overdue ? "Overdue: the last off-box backup is more than 24 h old" : "Age of the last off-box backup", | |
| 148 | k.backup ? `${k.backup.reasons[0] ?? k.backup.unknowns[0] ?? k.backup.line} ${k.backup.checkLine}` : overdue ? "Overdue: the last off-box backup is more than 24 h old" : "Age of the last off-box backup", | |
| 149 | 149 | null, |
| 150 | 150 | overdue ? "attn" : null, |
| 151 | 151 | )} |
test/backup-truth.test.ts +63 −2
| @@ -7,19 +7,23 @@ | ||
| 7 | 7 | |
| 8 | 8 | import { afterAll, beforeAll, expect, test } from "bun:test"; |
| 9 | 9 | import { Database } from "bun:sqlite"; |
| 10 | import { existsSync } from "node:fs"; | |
| 10 | import { existsSync, readFileSync, writeFileSync } from "node:fs"; | |
| 11 | 11 | import { mkdtemp, rm } from "node:fs/promises"; |
| 12 | 12 | import { tmpdir } from "node:os"; |
| 13 | 13 | import { join } from "node:path"; |
| 14 | 14 | import { createApp, type App } from "../src/app.ts"; |
| 15 | 15 | import { vitals } from "../src/kernel/vitals.ts"; |
| 16 | import { backupVerdict, CHECK_STALE_MS, judge, keepReads, LATE_MS, readLane, watchBucket, type LaneRead } from "../src/snapshot/bucket.ts"; | |
| 16 | import { backupVerdict, CHECK_LOG, CHECK_LOG_KEEP, CHECK_STALE_MS, recordRun, judge, keepReads, LATE_MS, readLane, watchBucket, type LaneRead } from "../src/snapshot/bucket.ts"; | |
| 17 | 17 | import type { DrillResult } from "../src/snapshot/drill.ts"; |
| 18 | 18 | import { lastNightly, snapshotDue } from "../src/snapshot/due.ts"; |
| 19 | 19 | import { snapshotsOf, startSnapshots } from "../src/snapshot/service.ts"; |
| 20 | 20 | import { listSnapshots } from "../src/snapshot/snapshot.ts"; |
| 21 | 21 | import { dirStore } from "../src/snapshot/store.ts"; |
| 22 | 22 | import { verdictLines } from "../src/tui/screens/resilience.ts"; |
| 23 | import { readHealth } from "../src/actions/health.ts"; | |
| 24 | import type { User } from "../src/core/users.ts"; | |
| 25 | import { NO_REACH } from "../src/flow/reach.ts"; | |
| 26 | import { kpiRowFor } from "../src/web/studio/kpi_row.ts"; | |
| 23 | 27 | import { vacuumInto } from "../src/snapshot/vacuum.ts"; |
| 24 | 28 | |
| 25 | 29 | let dataDir: string; |
| @@ -243,3 +247,60 @@ test("copying the database does not hold the serving thread (#7559)", async () = | ||
| 243 | 247 | copy.close(); |
outdated
Use a fresh temp data dir so the count does not depend on other tests, and add assertions for the home tile and /api/v1/health, which box 2 names; add a trim test. |
||
| 244 | 248 | db.close(); |
| 245 | 249 | }, 60_000); |
| 250 | ||
| 251 | const runLines = (dir: string): string[] => readFileSync(join(dir, CHECK_LOG), "utf8").split("\n").filter(Boolean); | |
| 252 | ||
| 253 | test("the check records each run; stopped past its interval, the verdict, tile, card, terminal and health are red saying when it last ran", async () => { | |
| 254 | const dir = await mkdtemp(join(tmpdir(), "forge-check-runs-")); | |
| 255 | const own = createApp({ dataDir: dir, port: 0 }); | |
| 256 | const w = watchBucket(dir, () => [{ lane: "primary", store: dirStore(dir) }], 3_600_000, 3_600_000); | |
| 257 | const realNow = Date.now; | |
| 258 | try { | |
| 259 | await w.pass(); | |
| 260 | await w.pass(); | |
| 261 | expect(runLines(dir)).toHaveLength(2); | |
| 262 | expect(JSON.parse(runLines(dir)[1]!).lanes[0].lane).toBe("primary"); | |
| 263 | const ranAt = backupVerdict(dir, undefined).checkedAt!; | |
| 264 | expect(backupVerdict(dir, undefined).runs).toBe(2); | |
| 265 | // The check is stopped: nothing reads the bucket while the clock moves on. | |
| 266 | const later = Date.parse(ranAt) + CHECK_STALE_MS + 60_000; | |
| 267 | const v = backupVerdict(dir, drill(), later); | |
| 268 | expect(v.state).toBe("red"); | |
| 269 | expect(v.reasons.join(" ")).toContain(`last ran ${ranAt}`); | |
| 270 | expect(v.checkLine).toContain("2 runs on record"); | |
| 271 | const tile = kpiRowFor({ landedHours: Array(24).fill(0), landed: 0, live: null, builders: { running: 0, slots: 1, idle: null }, spend: { todayUsd: 0, ceilingUsd: null }, medianCheckMs: null, backupAt: null, backup: v, reach: NO_REACH }, null, new Date(later)).toString(); | |
| 272 | expect(tile).toContain("attn"); | |
| 273 | expect(tile).toContain("check stopped"); | |
| 274 | expect(tile).toContain(`last ran ${ranAt}`); | |
| 275 | Date.now = () => later; | |
| 276 | const card = vitals({ db: own.db, signals: own.signals, dataDir: dir, now: later }).degraded.find((d) => d.cause === "backup"); | |
| 277 | expect(card?.sentence).toContain(`last ran ${ranAt}`); | |
| 278 | const health = readHealth(own, { isAdmin: true } as User) as { ok: boolean; degraded: { cause: string; sentence: string }[] }; | |
| 279 | expect(health.ok).toBe(false); | |
| 280 | expect(health.degraded.find((d) => d.cause === "backup")?.sentence).toContain(`last ran ${ranAt}`); | |
| 281 | expect(verdictLines({ app: own } as never).join("\n")).toContain("runs on record"); | |
| 282 | } finally { | |
| 283 | Date.now = realNow; | |
| 284 | w.stop(); | |
| 285 | own.db.close(); | |
| 286 | await rm(dir, { recursive: true, force: true }); | |
| 287 | } | |
| 288 | }); | |
| 289 | ||
| 290 | test("the run record is cut back to its keep once it holds twice as many, and a disk that refuses never throws", async () => { | |
| 291 | const dir = await mkdtemp(join(tmpdir(), "forge-check-trim-")); | |
| 292 | try { | |
| 293 | for (let i = 0; i < CHECK_LOG_KEEP * 2; i++) recordRun(dir, `2026-11-02T00:00:${String(i % 60).padStart(2, "0")}Z`, [lane(null)]); | |
| 294 | expect(runLines(dir)).toHaveLength(CHECK_LOG_KEEP * 2); | |
| 295 | recordRun(dir, "2026-11-02T01:00:00Z", [lane(null)]); | |
| 296 | const kept = runLines(dir); | |
| 297 | expect(kept).toHaveLength(CHECK_LOG_KEEP); | |
| 298 | expect(JSON.parse(kept.at(-1)!).at).toBe("2026-11-02T01:00:00Z"); | |
| 299 | // A data directory that is a file cannot be written to. | |
| 300 | const file = join(dir, "not-a-dir"); | |
| 301 | writeFileSync(file, "x"); | |
| 302 | expect(() => recordRun(file, "2026-11-02T02:00:00Z", [])).not.toThrow(); | |
| 303 | } finally { | |
| 304 | await rm(dir, { recursive: true, force: true }); | |
| 305 | } | |
| 306 | }); |
new above
offline
Non-blocking: matching reason text with includes("last ran") is brittle, and this line is long. Split it into a small helper or have the verdict carry the finished sentence.