Public: anyone can read this; the crew can change it.
A maintenance duty files one issue when the nightly backup is older than 36 hours #13349
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13331-a-maintenance-duty-files-one-issue-when → main 7 commits fork 2 threads · 2 unresolved
5 files
changed +243
−1
docs/features/README.md +1
| @@ -399,6 +399,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md) | ||
| 399 | 399 | Read with the guide page [](../guide-backups-moving.md). |
| 400 | 400 | |
| 401 | 401 | - [](backup-growth.md) |
| 402 | - [](backup-age.md) | |
| 402 | 403 | - [](disk-free.md) |
| 403 | 404 | - [](backup-refs.md) |
| 404 | 405 | - [](forge-origin.md) |
docs/features/backup-age.md added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | # A backup that stopped landing | |
| 2 | ||
| 3 | The `backup-age` maintenance duty (#13331) files one issue when the newest backup in the | |
| 4 | backup directory is more than 36 hours old: a missed night is loud once, not silent. | |
| 5 | ||
| 6 | - **What it watches.** The newest regular file under `FORGE_SNAPSHOT_DIR`, by modification | |
| 7 | time. File times only; nothing is opened, run or shelled out to. Symlinks are not followed. | |
| 8 | - **The limit.** 36 hours: a nightly, a missed night's slack, and a retry. | |
| 9 | - **The issue.** "The newest backup is older than 36 hours", naming the age in hours, the | |
| 10 | newest file's path and when it last changed. While one is open a later run files none | |
| 11 | (`fileIssueOnce`); closing it lets the next miss file a new one. | |
| 12 | - **Quiet cases.** `FORGE_SNAPSHOT_DIR` unset (a bucket destination has no file times here), | |
| 13 | a directory with no file yet, and a backup under 36 hours old. | |
| 14 | - **Too big to read.** The walk stops at 100,000 entries. A cut walk may have missed the newest | |
| 15 | file, so the duty then judges no age and files nothing; its finding says the directory was | |
| 16 | not read. | |
| 17 | - **Not the off-box alarm.** The flow alarm `alarm_backup` (#10924) fires at 26 hours on the | |
| 18 | newest copy that left the machine, from the snapshot record. This duty reads the local | |
| 19 | directory's files themselves, so it catches a record that says a backup landed while the | |
| 20 | files are not there or not fresh, and a directory nothing is writing to. The limits differ | |
| 21 | on purpose: the alarm is a live banner for the off-box copy, this is one issue for the local one. | |
| 22 | - The duty runs hourly, spends no money, and has a row on the maintenance board. | |
| 23 | - **Tests.** `test/maintain-backup-age.test.ts`: fresh is quiet, 40 hours files one issue | |
| 24 | naming age and path and not again, symlinks and the entry cap, the board row. |
src/maintain/duties/backup_age.ts added +106
| @@ -0,0 +1,106 @@ | ||
| 1 | /** | |
| 2 | * Backup age (#13331): when the newest file under the backup directory | |
| 3 | * (`FORGE_SNAPSHOT_DIR`) is over 36 hours old, file one issue naming the age and | |
| 4 | * the path. One open issue stands for the miss; it reads file times only. A | |
| 5 | * destination that is a bucket, or a directory with no file yet, has no time to | |
| 6 | * read here, so the duty says nothing of it. | |
| 7 | */ | |
| 8 | ||
| 9 | import { lstatSync, readdirSync } from "node:fs"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 12 | import { getRepo } from "../../core/repos.ts"; | |
| 13 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 14 | import type { Duty } from "../duty.ts"; | |
| 15 | ||
| 16 | export const TITLE = "The newest backup is older than 36 hours"; | |
| 17 | export const LIMIT_MS = 36 * 3_600_000; | |
| 18 | const MAX_ENTRIES = 100_000; | |
| 19 | ||
| 20 | /** | |
| 21 | * What a walk found: the newest regular file, and whether the walk stopped at the cap before it was | |
| 22 | * done. | |
| 23 | */ | |
| 24 | export interface Newest { | |
| 25 | readonly file: { readonly path: string; readonly mtimeMs: number } | undefined; | |
| 26 | readonly truncated: boolean; | |
| 27 | readonly seen: number; | |
| 28 | } | |
| 29 | ||
| 30 | /** | |
| 31 | * The newest regular file under `root` by modification time. Symlinks are not followed; past `cap` | |
| 32 | * entries the walk stops and says so. | |
| 33 | */ | |
| 34 | export function newestFile(root: string, cap: number = MAX_ENTRIES): Newest { | |
| 35 | let best: { path: string; mtimeMs: number } | undefined; | |
| 36 | let seen = 0; | |
| 37 | let truncated = false; | |
| 38 | const walk = (dir: string): void => { | |
| 39 | let names: string[]; | |
| 40 | try { | |
| 41 | names = readdirSync(dir); | |
| 42 | } catch { | |
| 43 | return; | |
| 44 | } | |
| 45 | for (const name of names) { | |
| 46 | if (truncated) return; | |
| 47 | if (++seen > cap) { | |
| 48 | truncated = true; | |
| 49 | return; | |
| 50 | } | |
| 51 | const path = join(dir, name); | |
| 52 | try { | |
| 53 | const st = lstatSync(path); | |
| 54 | if (st.isDirectory()) walk(path); | |
| 55 | else if (st.isFile() && (best === undefined || st.mtimeMs > best.mtimeMs)) best = { path, mtimeMs: st.mtimeMs }; | |
| 56 | } catch { | |
| 57 | // gone between listing and reading: skip it | |
| 58 | } | |
| 59 | } | |
| 60 | }; | |
| 61 | walk(root); | |
| 62 | return { file: best, truncated, seen: Math.min(seen, cap) }; | |
| 63 | } | |
| 64 | ||
| 65 | /** Age as "N hours", whole hours rounded down. */ | |
| 66 | export const ageSaid = (ms: number): string => `${Math.floor(ms / 3_600_000)} hours`; | |
| 67 | ||
| 68 | /** The duty, with the walk's entry cap injectable so a test can reach the cut-walk branch. */ | |
| 69 | export function backupAgeDuty(options: { readonly cap?: number } = {}): Duty { | |
| 70 | const cap = options.cap ?? MAX_ENTRIES; | |
| 71 | return { | |
| 72 | name: "backup-age", | |
| 73 | intervalMs: 3_600_000, | |
| 74 | budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 a day; the rest is room for a catch-up, and it spends no money" }, | |
| 75 | run({ db, bus, now, app }) { | |
| 76 | const root = process.env["FORGE_SNAPSHOT_DIR"]?.trim(); | |
| 77 | if (!root) return { findings: [], actions: [] }; | |
| 78 | const walked = newestFile(root, cap); | |
| 79 | // A cut walk may have missed the newest file: say so, and file nothing on a guess. | |
| 80 | if (walked.truncated) | |
| 81 | return { | |
| 82 | findings: [{ summary: `backup directory has over ${walked.seen} entries; not read, no age judged`, probe: { kind: "measurement" as const, what: "entries read under the backup directory", number: walked.seen, at: now.toISOString() } }], | |
| 83 | actions: [], | |
| 84 | }; | |
| 85 | const newest = walked.file; | |
| 86 | if (newest === undefined) return { findings: [], actions: [] }; | |
| 87 | const age = now.getTime() - newest.mtimeMs; | |
| 88 | if (age <= LIMIT_MS) return { findings: [], actions: [] }; | |
| 89 | const summary = `newest backup is ${ageSaid(age)} old: ${newest.path}`; | |
| 90 | const finding = { summary, probe: { kind: "measurement" as const, what: "hours since the newest backup file changed", number: Math.floor(age / 3_600_000), at: now.toISOString() } }; | |
| 91 | if (app === undefined) return { findings: [finding], actions: [] }; | |
| 92 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 93 | const repo = getRepo(db, owner, name); | |
| 94 | if (repo === undefined) return { findings: [finding], actions: [] }; | |
| 95 | const filed = fileIssueOnce(db, bus, repo, { | |
| 96 | title: TITLE, | |
| 97 | body: | |
| 98 | `_Filed by the backup age duty from file times, not written by a person._\n\n` + | |
| 99 | `The newest backup is ${ageSaid(age)} old (limit 36 hours).\n\n- Newest file: ${newest.path}\n- Last changed: ${new Date(newest.mtimeMs).toISOString()}`, | |
| 100 | }); | |
| 101 | return { findings: [finding], actions: filed.opened ? [`filed backup age issue #${filed.number}`] : [] }; | |
| 102 | }, | |
| 103 | }; | |
| 104 | } | |
| 105 | ||
| 106 | export const duty: Duty = backupAgeDuty(); |
src/maintain/registry.ts +2 −1
| @@ -8,7 +8,8 @@ import { duty as flowKeeper } from "./duties/flow_keeper.ts"; | ||
| 8 | 8 | import { duty as visitorRetention } from "./duties/visitor_retention.ts"; |
| 9 | 9 | import { duty as backupGrowth } from "./duties/backup_growth.ts"; |
| 10 | 10 | import { duty as v1Proven } from "./duties/v1_proven.ts"; |
| 11 | import { duty as backupAge } from "./duties/backup_age.ts"; | |
| 11 | 12 | import { duty as mergedBranches } from "./duties/merged_branches.ts"; |
| 12 | 13 | import { duty as diskFree } from "./duties/disk_free.ts"; |
| 13 | 14 | |
| 14 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven, diskFree, mergedBranches]; | |
| 15 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches]; |
test/maintain-backup-age.test.ts added +110
| @@ -0,0 +1,110 @@ | ||
| 1 | /** | |
| 2 | * Backup age (#13331): fresh files nothing; over 36 hours files one issue naming age and path, | |
| 3 | * once. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdirSync, mkdtempSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { weekRows } from "../src/maintain/week_line.ts"; | |
| 10 | import { backupAgeDuty, duty, newestFile, TITLE } from "../src/maintain/duties/backup_age.ts"; | |
| 11 | import { duties } from "../src/maintain/registry.ts"; | |
| 12 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 13 | ||
| 14 | const w = rulesWorld(); | |
| 15 | const HOUR = 3_600_000; | |
| 16 | const T0 = Date.parse("2026-11-10T00:00:00Z"); | |
| 17 | const root = mkdtempSync(join(tmpdir(), "backup-age-")); | |
| 18 | mkdirSync(join(root, "snap")); | |
| 19 | const file = join(root, "snap", "a.bundle"); | |
| 20 | writeFileSync(file, "x"); | |
| 21 | utimesSync(file, T0 / 1000, T0 / 1000); | |
| 22 | // Env is set for this file only and put back after it: bun runs every test file in one process. | |
outdated
Module-level env mutation that is never restored (later tests delete and re-point it too). Set in beforeAll, restore in afterAll, so other files in the same bun process are unaffected. |
||
| 23 | const saved = { FORGE_SENTINEL_REPO: process.env.FORGE_SENTINEL_REPO, FORGE_SNAPSHOT_DIR: process.env.FORGE_SNAPSHOT_DIR }; | |
| 24 | beforeAll(() => { | |
| 25 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 26 | process.env.FORGE_SNAPSHOT_DIR = root; | |
| 27 | }); | |
| 28 | afterAll(() => { | |
| 29 | for (const [key, value] of Object.entries(saved)) { | |
| 30 | if (value === undefined) delete process.env[key]; | |
| 31 | else process.env[key] = value; | |
| 32 | } | |
| 33 | }); | |
| 34 | const at = (hours: number) => ({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date(T0 + hours * HOUR) }); | |
| 35 | const run = (hours: number) => duty.run(at(hours)); | |
| 36 | const filed = () => w.titles().filter((t) => t === TITLE); | |
| 37 | ||
| 38 | test("the duty is registered", () => { | |
| 39 | expect(duties.map((d) => d.name)).toContain("backup-age"); | |
| 40 | }); | |
| 41 | ||
| 42 | test("a backup under 36 hours old files nothing", async () => { | |
| 43 | const r = await run(35); | |
| 44 | expect(r.findings).toEqual([]); | |
| 45 | expect(filed()).toHaveLength(0); | |
| 46 | }); | |
| 47 | ||
| 48 | test("a walk cut at the cap says 'not read' and files no issue, however old the files", async () => { | |
| 49 | // Runs before anything is filed, so an issue here would show. | |
| 50 | const big = mkdtempSync(join(tmpdir(), "backup-age-big-")); | |
| 51 | process.env.FORGE_SNAPSHOT_DIR = big; | |
| 52 | for (let i = 0; i < 5; i++) { | |
| 53 | writeFileSync(join(big, `f${i}`), "x"); | |
| 54 | utimesSync(join(big, `f${i}`), T0 / 1000, T0 / 1000); | |
| 55 | } | |
| 56 | const r = await backupAgeDuty({ cap: 3 }).run(at(500)); | |
| 57 | process.env.FORGE_SNAPSHOT_DIR = root; | |
| 58 | expect(r.findings).toHaveLength(1); | |
| 59 | expect(r.findings[0]?.summary).toBe("backup directory has over 3 entries; not read, no age judged"); | |
| 60 | expect(r.actions).toEqual([]); | |
| 61 | expect(filed()).toHaveLength(0); | |
| 62 | }); | |
| 63 | ||
| 64 | test("over 36 hours files one issue naming age and path, and not again while open", async () => { | |
| 65 | const r = await run(40); | |
| 66 | expect(r.actions).toHaveLength(1); | |
| 67 | expect(filed()).toHaveLength(1); | |
| 68 | const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body; | |
| 69 | expect(body).toContain("40 hours"); | |
| 70 | expect(body).toContain(file); | |
| 71 | expect((await run(50)).actions).toEqual([]); | |
| 72 | expect(filed()).toHaveLength(1); | |
| 73 | }); | |
| 74 | ||
| 75 | test("no backup directory set, or an empty one, is quiet", async () => { | |
| 76 | process.env.FORGE_SNAPSHOT_DIR = mkdtempSync(join(tmpdir(), "backup-age-empty-")); | |
| 77 | expect((await run(500)).findings).toEqual([]); | |
| 78 | delete process.env.FORGE_SNAPSHOT_DIR; | |
| 79 | expect((await run(500)).findings).toEqual([]); | |
| 80 | }); | |
| 81 | ||
| 82 | test("the maintenance board lists backup-age as a row", () => { | |
| 83 | const rows = weekRows(w.app.db, duties.map((d) => d.name)); | |
| 84 | expect(rows.map((r) => r.name)).toContain("backup-age"); | |
| 85 | }); | |
| 86 | ||
| 87 | test("newestFile does not follow symlinks and stops, saying so, at its cap", () => { | |
| 88 | const dir = mkdtempSync(join(tmpdir(), "backup-age-walk-")); | |
| 89 | const outside = mkdtempSync(join(tmpdir(), "backup-age-outside-")); | |
| 90 | writeFileSync(join(outside, "new.bundle"), "x"); // newer than anything in dir, but only reachable by a link | |
| 91 | symlinkSync(outside, join(dir, "link")); | |
| 92 | symlinkSync(join(outside, "new.bundle"), join(dir, "file-link")); | |
| 93 | writeFileSync(join(dir, "real.bundle"), "x"); | |
| 94 | utimesSync(join(dir, "real.bundle"), T0 / 1000, T0 / 1000); | |
| 95 | const whole = newestFile(dir); | |
| 96 | expect(whole.file?.path).toBe(join(dir, "real.bundle")); | |
| 97 | expect(whole.truncated).toBe(false); | |
| 98 | for (let i = 0; i < 5; i++) writeFileSync(join(dir, `f${i}`), "x"); | |
| 99 | const cut = newestFile(dir, 3); | |
| 100 | expect(cut.truncated).toBe(true); | |
| 101 | expect(cut.seen).toBe(3); | |
| 102 | }); | |
| 103 | ||
| 104 | test("a directory under the cap is still judged", async () => { | |
| 105 | const small = mkdtempSync(join(tmpdir(), "backup-age-small-")); | |
| 106 | process.env.FORGE_SNAPSHOT_DIR = small; | |
| 107 | writeFileSync(join(small, "a"), "x"); | |
| 108 | utimesSync(join(small, "a"), T0 / 1000, T0 / 1000); | |
| 109 | expect((await run(500)).findings[0]?.summary).toContain("newest backup is"); | |
| 110 | }); |
new above
offline
MAX_ENTRIES silently truncates the walk, so a big tree could miss the newest file and file a false 'old' issue. Test it, or report it instead of guessing.