Public: anyone can read this; the crew can change it.

A maintenance duty files one issue when the nightly backup is older than 36 hours #13349

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13331-a-maintenance-duty-files-one-issue-when → main 7 commits fork 2 threads · 2 unresolved

  • Merged. mason/13331-a-maintenance-duty-files-one-issue-when was merged into main as 50af297.

Review

Sign in to review this change.
5 files changed +243 −1
docs/features/README.md +1
@@ -399,6 +399,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md)
399399 Read with the guide page [Backups and moving](../guide-backups-moving.md).
400400
401401 - [A nightly backup that doubles](backup-growth.md)
402 - [A backup that stopped landing](backup-age.md)
402403 - [A data volume that is filling up](disk-free.md)
403404 - [A branch that cannot be backed up](backup-refs.md)
404405 - [A forge is an origin](forge-origin.md)
docs/features/backup-age.md added +24
@@ -0,0 +1,24 @@
1 # A backup that stopped landing
2
3 The `backup-age` maintenance duty (#13331) files one issue when the newest backup in the
4 backup directory is more than 36 hours old: a missed night is loud once, not silent.
5
6 - **What it watches.** The newest regular file under `FORGE_SNAPSHOT_DIR`, by modification
7 time. File times only; nothing is opened, run or shelled out to. Symlinks are not followed.
8 - **The limit.** 36 hours: a nightly, a missed night's slack, and a retry.
9 - **The issue.** "The newest backup is older than 36 hours", naming the age in hours, the
10 newest file's path and when it last changed. While one is open a later run files none
11 (`fileIssueOnce`); closing it lets the next miss file a new one.
12 - **Quiet cases.** `FORGE_SNAPSHOT_DIR` unset (a bucket destination has no file times here),
13 a directory with no file yet, and a backup under 36 hours old.
14 - **Too big to read.** The walk stops at 100,000 entries. A cut walk may have missed the newest
15 file, so the duty then judges no age and files nothing; its finding says the directory was
16 not read.
17 - **Not the off-box alarm.** The flow alarm `alarm_backup` (#10924) fires at 26 hours on the
18 newest copy that left the machine, from the snapshot record. This duty reads the local
19 directory's files themselves, so it catches a record that says a backup landed while the
20 files are not there or not fresh, and a directory nothing is writing to. The limits differ
21 on purpose: the alarm is a live banner for the off-box copy, this is one issue for the local one.
22 - The duty runs hourly, spends no money, and has a row on the maintenance board.
23 - **Tests.** `test/maintain-backup-age.test.ts`: fresh is quiet, 40 hours files one issue
24 naming age and path and not again, symlinks and the entry cap, the board row.
src/maintain/duties/backup_age.ts added +106
@@ -0,0 +1,106 @@
1 /**
2 * Backup age (#13331): when the newest file under the backup directory
3 * (`FORGE_SNAPSHOT_DIR`) is over 36 hours old, file one issue naming the age and
4 * the path. One open issue stands for the miss; it reads file times only. A
5 * destination that is a bucket, or a directory with no file yet, has no time to
6 * read here, so the duty says nothing of it.
7 */
8
9 import { lstatSync, readdirSync } from "node:fs";
10 import { join } from "node:path";
11 import { fileIssueOnce } from "../../checks/flake.ts";
12 import { getRepo } from "../../core/repos.ts";
13 import { ownRepoPath } from "../../kernel/own_repo.ts";
14 import type { Duty } from "../duty.ts";

MAX_ENTRIES silently truncates the walk, so a big tree could miss the newest file and file a false 'old' issue. Test it, or report it instead of guessing.

15
16 export const TITLE = "The newest backup is older than 36 hours";
17 export const LIMIT_MS = 36 * 3_600_000;
18 const MAX_ENTRIES = 100_000;
19
20 /**
21 * What a walk found: the newest regular file, and whether the walk stopped at the cap before it was
22 * done.
23 */
24 export interface Newest {
25 readonly file: { readonly path: string; readonly mtimeMs: number } | undefined;
26 readonly truncated: boolean;
27 readonly seen: number;
28 }
29
30 /**
31 * The newest regular file under `root` by modification time. Symlinks are not followed; past `cap`
32 * entries the walk stops and says so.
33 */
34 export function newestFile(root: string, cap: number = MAX_ENTRIES): Newest {
35 let best: { path: string; mtimeMs: number } | undefined;
36 let seen = 0;
37 let truncated = false;
38 const walk = (dir: string): void => {
39 let names: string[];
40 try {
41 names = readdirSync(dir);
42 } catch {
43 return;
44 }
45 for (const name of names) {
46 if (truncated) return;
47 if (++seen > cap) {
48 truncated = true;
49 return;
50 }
51 const path = join(dir, name);
52 try {
53 const st = lstatSync(path);
54 if (st.isDirectory()) walk(path);
55 else if (st.isFile() && (best === undefined || st.mtimeMs > best.mtimeMs)) best = { path, mtimeMs: st.mtimeMs };
56 } catch {
57 // gone between listing and reading: skip it
58 }
59 }
60 };
61 walk(root);
62 return { file: best, truncated, seen: Math.min(seen, cap) };
63 }
64
65 /** Age as "N hours", whole hours rounded down. */
66 export const ageSaid = (ms: number): string => `${Math.floor(ms / 3_600_000)} hours`;
67
68 /** The duty, with the walk's entry cap injectable so a test can reach the cut-walk branch. */
69 export function backupAgeDuty(options: { readonly cap?: number } = {}): Duty {
70 const cap = options.cap ?? MAX_ENTRIES;
71 return {
72 name: "backup-age",
73 intervalMs: 3_600_000,
74 budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 a day; the rest is room for a catch-up, and it spends no money" },
75 run({ db, bus, now, app }) {
76 const root = process.env["FORGE_SNAPSHOT_DIR"]?.trim();
77 if (!root) return { findings: [], actions: [] };
78 const walked = newestFile(root, cap);
79 // A cut walk may have missed the newest file: say so, and file nothing on a guess.
80 if (walked.truncated)
81 return {
82 findings: [{ summary: `backup directory has over ${walked.seen} entries; not read, no age judged`, probe: { kind: "measurement" as const, what: "entries read under the backup directory", number: walked.seen, at: now.toISOString() } }],
83 actions: [],
84 };
85 const newest = walked.file;
86 if (newest === undefined) return { findings: [], actions: [] };
87 const age = now.getTime() - newest.mtimeMs;
88 if (age <= LIMIT_MS) return { findings: [], actions: [] };
89 const summary = `newest backup is ${ageSaid(age)} old: ${newest.path}`;
90 const finding = { summary, probe: { kind: "measurement" as const, what: "hours since the newest backup file changed", number: Math.floor(age / 3_600_000), at: now.toISOString() } };
91 if (app === undefined) return { findings: [finding], actions: [] };
92 const [owner = "", name = ""] = ownRepoPath().split("/");
93 const repo = getRepo(db, owner, name);
94 if (repo === undefined) return { findings: [finding], actions: [] };
95 const filed = fileIssueOnce(db, bus, repo, {
96 title: TITLE,
97 body:
98 `_Filed by the backup age duty from file times, not written by a person._\n\n` +
99 `The newest backup is ${ageSaid(age)} old (limit 36 hours).\n\n- Newest file: ${newest.path}\n- Last changed: ${new Date(newest.mtimeMs).toISOString()}`,
100 });
101 return { findings: [finding], actions: filed.opened ? [`filed backup age issue #${filed.number}`] : [] };
102 },
103 };
104 }
105
106 export const duty: Duty = backupAgeDuty();
src/maintain/registry.ts +2 −1
@@ -8,7 +8,8 @@ import { duty as flowKeeper } from "./duties/flow_keeper.ts";
88 import { duty as visitorRetention } from "./duties/visitor_retention.ts";
99 import { duty as backupGrowth } from "./duties/backup_growth.ts";
1010 import { duty as v1Proven } from "./duties/v1_proven.ts";
11 import { duty as backupAge } from "./duties/backup_age.ts";
1112 import { duty as mergedBranches } from "./duties/merged_branches.ts";
1213 import { duty as diskFree } from "./duties/disk_free.ts";
1314
14 export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven, diskFree, mergedBranches];
15 export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches];
test/maintain-backup-age.test.ts added +110
@@ -0,0 +1,110 @@
1 /**
2 * Backup age (#13331): fresh files nothing; over 36 hours files one issue naming age and path,
3 * once.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdirSync, mkdtempSync, symlinkSync, utimesSync, writeFileSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { weekRows } from "../src/maintain/week_line.ts";
10 import { backupAgeDuty, duty, newestFile, TITLE } from "../src/maintain/duties/backup_age.ts";
11 import { duties } from "../src/maintain/registry.ts";
12 import { rulesWorld } from "./helpers/rules-app.ts";
13
14 const w = rulesWorld();
15 const HOUR = 3_600_000;
16 const T0 = Date.parse("2026-11-10T00:00:00Z");
17 const root = mkdtempSync(join(tmpdir(), "backup-age-"));
18 mkdirSync(join(root, "snap"));
19 const file = join(root, "snap", "a.bundle");
20 writeFileSync(file, "x");
21 utimesSync(file, T0 / 1000, T0 / 1000);
22 // Env is set for this file only and put back after it: bun runs every test file in one process.
outdated

Module-level env mutation that is never restored (later tests delete and re-point it too). Set in beforeAll, restore in afterAll, so other files in the same bun process are unaffected.

23 const saved = { FORGE_SENTINEL_REPO: process.env.FORGE_SENTINEL_REPO, FORGE_SNAPSHOT_DIR: process.env.FORGE_SNAPSHOT_DIR };
24 beforeAll(() => {
25 process.env.FORGE_SENTINEL_REPO = "alice/rules";
26 process.env.FORGE_SNAPSHOT_DIR = root;
27 });
28 afterAll(() => {
29 for (const [key, value] of Object.entries(saved)) {
30 if (value === undefined) delete process.env[key];
31 else process.env[key] = value;
32 }
33 });
34 const at = (hours: number) => ({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date(T0 + hours * HOUR) });
35 const run = (hours: number) => duty.run(at(hours));
36 const filed = () => w.titles().filter((t) => t === TITLE);
37
38 test("the duty is registered", () => {
39 expect(duties.map((d) => d.name)).toContain("backup-age");
40 });
41
42 test("a backup under 36 hours old files nothing", async () => {
43 const r = await run(35);
44 expect(r.findings).toEqual([]);
45 expect(filed()).toHaveLength(0);
46 });
47
48 test("a walk cut at the cap says 'not read' and files no issue, however old the files", async () => {
49 // Runs before anything is filed, so an issue here would show.
50 const big = mkdtempSync(join(tmpdir(), "backup-age-big-"));
51 process.env.FORGE_SNAPSHOT_DIR = big;
52 for (let i = 0; i < 5; i++) {
53 writeFileSync(join(big, `f${i}`), "x");
54 utimesSync(join(big, `f${i}`), T0 / 1000, T0 / 1000);
55 }
56 const r = await backupAgeDuty({ cap: 3 }).run(at(500));
57 process.env.FORGE_SNAPSHOT_DIR = root;
58 expect(r.findings).toHaveLength(1);
59 expect(r.findings[0]?.summary).toBe("backup directory has over 3 entries; not read, no age judged");
60 expect(r.actions).toEqual([]);
61 expect(filed()).toHaveLength(0);
62 });
63
64 test("over 36 hours files one issue naming age and path, and not again while open", async () => {
65 const r = await run(40);
66 expect(r.actions).toHaveLength(1);
67 expect(filed()).toHaveLength(1);
68 const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body;
69 expect(body).toContain("40 hours");
70 expect(body).toContain(file);
71 expect((await run(50)).actions).toEqual([]);
72 expect(filed()).toHaveLength(1);
73 });
74
75 test("no backup directory set, or an empty one, is quiet", async () => {
76 process.env.FORGE_SNAPSHOT_DIR = mkdtempSync(join(tmpdir(), "backup-age-empty-"));
77 expect((await run(500)).findings).toEqual([]);
78 delete process.env.FORGE_SNAPSHOT_DIR;
79 expect((await run(500)).findings).toEqual([]);
80 });
81
82 test("the maintenance board lists backup-age as a row", () => {
83 const rows = weekRows(w.app.db, duties.map((d) => d.name));
84 expect(rows.map((r) => r.name)).toContain("backup-age");
85 });
86
87 test("newestFile does not follow symlinks and stops, saying so, at its cap", () => {
88 const dir = mkdtempSync(join(tmpdir(), "backup-age-walk-"));
89 const outside = mkdtempSync(join(tmpdir(), "backup-age-outside-"));
90 writeFileSync(join(outside, "new.bundle"), "x"); // newer than anything in dir, but only reachable by a link
91 symlinkSync(outside, join(dir, "link"));
92 symlinkSync(join(outside, "new.bundle"), join(dir, "file-link"));
93 writeFileSync(join(dir, "real.bundle"), "x");
94 utimesSync(join(dir, "real.bundle"), T0 / 1000, T0 / 1000);
95 const whole = newestFile(dir);
96 expect(whole.file?.path).toBe(join(dir, "real.bundle"));
97 expect(whole.truncated).toBe(false);
98 for (let i = 0; i < 5; i++) writeFileSync(join(dir, `f${i}`), "x");
99 const cut = newestFile(dir, 3);
100 expect(cut.truncated).toBe(true);
101 expect(cut.seen).toBe(3);
102 });
103
104 test("a directory under the cap is still judged", async () => {
105 const small = mkdtempSync(join(tmpdir(), "backup-age-small-"));
106 process.env.FORGE_SNAPSHOT_DIR = small;
107 writeFileSync(join(small, "a"), "x");
108 utimesSync(join(small, "a"), T0 / 1000, T0 / 1000);
109 expect((await run(500)).findings[0]?.summary).toContain("newest backup is");
110 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs