fd7bd4c · ac38799
@@ -662,6 +662,12 @@
into scratch, opens it, counts what is there and unseals a value; the page
waits for it, up to 10 minutes, and the last drill shows there as one line.
Only one runs at a time. See `docs/SNAPSHOT.md`.
The forge also runs one itself every Sunday 09:00 UTC (a forge that was down
then runs it five minutes after waking, once). Each snapshot's manifest keeps
the five counts of its database; a restore that counts differently is red at
`count` and, like any red drill, one board line and one push. Snapshots from
before the counts were kept are not compared.
**Failure domains** (`/settings/resilience`, not `/resilience`, which is the THREATS.md page; admins only; a non-admin gets the same
404 as the other admin settings pages): one row per failure domain from
`docs/DOMAINS.md` — what breaks, the words the forge says, and the domain's
@@ -0,0 +1,31 @@
/**
 * What a forge holds, counted: the same five numbers when a snapshot is
 * taken (kept in its manifest) and when the drill restores it, so the two
 * can be set side by side. Plain data out.
 */
import type { Database } from "bun:sqlite";
export interface DrillCounts {
  readonly repositories: number;
  readonly issues: number;
  readonly users: number;
  readonly pulls: number;
  readonly apps: number;
}
export function countRows(db: Database): DrillCounts {
  const n = (sql: string): number => db.query<{ n: number }, []>(sql).get()?.n ?? 0;
  return {
    repositories: n("SELECT count(*) AS n FROM repo"),
    issues: n("SELECT count(*) AS n FROM issue"),
    users: n("SELECT count(*) AS n FROM user"),
    pulls: n("SELECT count(*) AS n FROM pull"),
    apps: n("SELECT count(*) AS n FROM app"),
  };
}
/** The names of the counts that differ, empty when they all match. */
export function mismatched(kept: DrillCounts, found: DrillCounts): string[] {
  return (Object.keys(kept) as (keyof DrillCounts)[]).filter((k) => kept[k] !== found[k]);
}
@@ -14,6 +14,7 @@
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { bootRestored } from "./boot.ts";
import { forgetKey, open } from "../kernel/secret.ts";
import { countRows, mismatched, type DrillCounts } from "./counts.ts";
import { restoreSnapshot } from "./restore.ts";
import { listSnapshots } from "./snapshot.ts";
import type { Store } from "./store.ts";
@@ -35,13 +36,7 @@
export interface DrillDeps {
  readonly boot?: typeof bootRestored | undefined;
}
export interface DrillCounts {
  readonly repositories: number;
  readonly issues: number;
  readonly users: number;
  readonly pulls: number;
  readonly apps: number;
}
export type { DrillCounts };
export interface DrillResult {
  readonly at: string;
@@ -72,7 +67,7 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
  const at = clock().toISOString();
  const timeoutMs = deps.timeoutMs ?? DRILL_TIMEOUT_MS;
  const restore = deps.restore ?? restoreSnapshot;
  const counts = { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 };
  const counts: { -readonly [K in keyof DrillCounts]: number } = { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 };
  let snapshotId: string | null = null;
  let step: DrillStep = "fetch";
  let scratch: string | undefined;
@@ -103,13 +98,16 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
      // The constructor is lazy; reading the header is what proves a database.
      db.query("PRAGMA schema_version").get();
      step = "count";
      const n = (sql: string): number =>
        db.query<{ n: number }, []>(sql).get()?.n ?? 0;
      counts.repositories = n("SELECT count(*) AS n FROM repo");
      counts.issues = n("SELECT count(*) AS n FROM issue");
      counts.users = n("SELECT count(*) AS n FROM user");
      counts.pulls = n("SELECT count(*) AS n FROM pull");
      counts.apps = n("SELECT count(*) AS n FROM app");
      Object.assign(counts, countRows(db));
      // A snapshot that kept its counts must restore to the same ones.
      const kept = newest.counts;
      const off = kept === undefined ? [] : mismatched(kept, counts);
      if (kept !== undefined && off.length > 0)
        throw new Error(
          `counts differ from the snapshot's: ${off
            .map((k) => `${k} ${kept[k as keyof DrillCounts]} became ${counts[k as keyof DrillCounts]}`)
            .join(", ")}`,
        );
      step = "unseal";
      const secret = db
@@ -0,0 +1,30 @@
/**
 * When the weekly drill is owed: every Sunday 09:00 UTC is a slot, and a
 * slot is kept by one drill at or after it, however it was asked for. A
 * forge that was down at the slot runs it on waking; one that restarts
 * after the drill ran does not run it twice. Plain data in, plain data out.
 */
const DAY = 86_400_000;
/** The latest Sunday 09:00 UTC at or before `now`, in ms. */
export function lastSlot(now: number): number {
  const d = new Date(now);
  d.setUTCHours(9, 0, 0, 0);
  let t = d.getTime();
  while (t > now || new Date(t).getUTCDay() !== 0) t -= DAY;
  return t;
}
/**
 * Whether a drill is owed at `now`. `lastAt` is the newest drill's time,
 * `since` when the forge first owed one; a forge younger than the slot
 * owes nothing for it.
 */
export function drillDue(lastAt: string | undefined, since: string | undefined, now: number): boolean {
  const slot = lastSlot(now);
  const last = Date.parse(lastAt ?? "");
  if (Number.isFinite(last)) return last < slot;
  const began = Date.parse(since ?? "");
  return Number.isFinite(began) && began < slot;
}
@@ -10,7 +10,7 @@
import type { App } from "../app.ts";
import { createLog } from "../kernel/log.ts";
import { reportVerdict, seedFirstSeen } from "./verdict.ts";
import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts";
import {
  KEEP,
  listSnapshots,
@@ -20,7 +20,8 @@
import {
  type Progress,
} from "./snapshot.ts";
import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { recordDrill } from "./drills.ts";
import { drillDue } from "./due.ts";
import { lastDrill, recordDrill } from "./drills.ts";
import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
import type { Store } from "./store.ts";
@@ -44,6 +45,8 @@
export interface SnapshotsHandle {
  readonly last: () => { manifest?: Manifest; error?: string; at?: string };
  /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
  readonly drill: () => Promise<DrillResult>;
  /** Drill if this week's slot has no drill yet; says whether it did. */
  readonly drillIfDue: () => Promise<boolean>;
  /** The off-box mirror's store; undefined when the forge has none. */
  readonly mirror: Store | undefined;
  /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */
@@ -64,6 +67,7 @@
export const snapshotsOf = (app: App): SnapshotsHandle | undefined =>
/** The weekly restore drill: Sunday, an hour after the nightly snapshot. */
const DRILL_DAY_UTC = 0;
const DRILL_HOUR_UTC = 9;
const CATCH_UP_MS = 5 * 60_000;
export function untilDrill(now = Date.now()): number {
  const next = new Date(now);
@@ -235,16 +239,25 @@
export function startSnapshots(
    log.warn("drill: first-seen time not kept; a missing drill will not be flagged", { error: String(error) });
  }
  let catchUp: ReturnType<typeof setTimeout> | undefined;
  let drillTimer: ReturnType<typeof setTimeout> | undefined;
  const scheduleDrill = (): void => {
    if (options.nightly === false || stopped) return;
    drillTimer = setTimeout(() => {
      drill()
        .catch((error) => log.error("restore drill failed to run", { error: String(error) }))
        .finally(scheduleDrill);
      runIfDue().finally(scheduleDrill);
    }, untilDrill());
  };
  // Owed and not yet kept: run it once. A restart after the drill ran finds it kept.
  const runIfDue = async (): Promise<boolean> => {
    if (!drillDue(lastDrill(app.db)?.at, firstSeen(app.config.dataDir), Date.now())) return false;
    await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) }));
    return true;
  };
  scheduleDrill();
  // A forge that was down at the slot catches up shortly after boot.
  if (options.nightly !== false) {
    catchUp = setTimeout(() => void runIfDue(), CATCH_UP_MS);
  }
      
        junipercrew
        29d ago
        
      
      Catch-up fires once at boot+5min and the weekly timer then also calls runIfDue; fine, but if the drill throws, nothing is recorded so it will be retried only next Sunday. Consider recording a red result on thrown errors so the board line appears.
    
  // Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
  let mirrorTimer: ReturnType<typeof setTimeout> | undefined;
@@ -266,6 +279,7 @@
export function startSnapshots(
    wraps: !!options.passphrase,
    now: take,
    drill,
    drillIfDue: runIfDue,
    list: async () => {
      const store = current();
      return store === undefined ? [] : listSnapshots(store);
@@ -286,6 +300,7 @@
export function startSnapshots(
      stopped = true;
      if (timer !== undefined) clearTimeout(timer);
      if (drillTimer !== undefined) clearTimeout(drillTimer);
      if (catchUp !== undefined) clearTimeout(catchUp);
      if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
      if (mirroring !== undefined) await mirroring.catch(() => {});
      if (inFlight !== undefined) await inFlight.catch(() => {});
@@ -21,7 +21,7 @@
 * lists them in the order they were handed over, not the order they landed.
 */
import type { Database } from "bun:sqlite";
import { Database } from "bun:sqlite";
import { mkdtempSync, readdirSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, relative } from "node:path";
@@ -31,6 +31,7 @@
import { listEveryRepo } from "../core/repos.ts";
import { runGit } from "../git/env.ts";
import { repoDir } from "../git/repos.ts";
import { createLog } from "../kernel/log.ts";
import { countRows, type DrillCounts } from "./counts.ts";
import type { Store } from "./store.ts";
import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts";
@@ -54,6 +55,8 @@
export interface Manifest {
  readonly apps: number;
  readonly files: readonly SnapshotFile[];
  readonly bytes: number;
  /** What the database copy held; the drill checks the restore against it. Absent before the drill compared. */
  readonly counts?: DrillCounts;
  /**
   * What the snapshot carries of the forge's key: `wrapped` under the
   * operator's passphrase, `none` (sealed values are lost on restore), or
@@ -213,6 +216,7 @@
export async function takeSnapshot(
  const prefix = `${SNAPSHOT_PREFIX}${name}/`;
  const scratch = mkdtempSync(join(tmpdir(), "forge-snapshot-"));
  const files: SnapshotFile[] = [];
  let dbCounts: DrillCounts | undefined;
  const started = Date.now();
  const everyObjects = forge.progressEvery?.objects ?? PROGRESS_OBJECTS;
  const everyMs = forge.progressEvery?.ms ?? PROGRESS_MS;
@@ -293,6 +297,14 @@
export async function takeSnapshot(
    // The database, consistently, from the live connection.
    const dbCopy = join(scratch, "forge.db");
    forge.db.run("VACUUM INTO ?1", [dbCopy]);
    const copy = new Database(dbCopy, { readonly: true });
    try {
      dbCounts = countRows(copy);
    } catch {
      // A database that cannot be counted is the drill's to find; the snapshot still goes up.
    } finally {
      copy.close();
    }
    await putFile("forge.db", dbCopy);
    // The key travels only wrapped. Without a passphrase it stays home.
    let key: Manifest["key"] = "none";
@@ -420,6 +432,7 @@
export async function takeSnapshot(
      apps: kept,
      files,
      bytes: files.reduce((n, f) => n + f.bytes, 0),
      ...(dbCounts === undefined ? {} : { counts: dbCounts }),
      key,
    };
    await store.put(
@@ -96,8 +96,8 @@
function drillLine(ctx: Ctx): Html {
      ? html`<p class="backups-drill">No restore drill has run yet.</p>`
      : d.verdict === "green"
        ? html`<p class="backups-drill">
            ${icon("check")} Last drill ${time(d.at)}: snapshot
            <code>${d.snapshotId ?? ""}</code> restored green in ${d.ms} ms —
            ${icon("check")} Last drill: ${time(d.at)}, ${d.ms} ms, all counts matched —
            snapshot <code>${d.snapshotId ?? ""}</code>, key opened —
            ${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.
          </p>`
        : html`<p class="backups-error backups-drill">
@@ -0,0 +1,155 @@
/**
 * The weekly drill runs on its own: owed once per Sunday slot, kept after,
 * and a count that differs from the snapshot's is red and a line.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { runDrill, type DrillResult } from "../src/snapshot/drill.ts";
import { drillDue, lastSlot } from "../src/snapshot/due.ts";
import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
import { startSnapshots } from "../src/snapshot/service.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { recordVerdict } from "../src/snapshot/verdict.ts";
const t = (iso: string): number => Date.parse(iso);
describe("when a weekly drill is owed", () => {
  // 2026-10-11 is a Sunday.
  test("the slot is the latest Sunday 09:00 UTC", () => {
    expect(new Date(lastSlot(t("2026-10-16T12:00:00Z"))).toISOString()).toBe("2026-10-11T09:00:00.000Z");
    expect(new Date(lastSlot(t("2026-10-11T08:59:00Z"))).toISOString()).toBe("2026-10-04T09:00:00.000Z");
  });
  test("owed once: a drill since the slot keeps it, an older one does not", () => {
    const now = t("2026-10-11T09:30:00Z");
    expect(drillDue("2026-10-04T09:05:00.000Z", undefined, now)).toBe(true);
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, now)).toBe(false);
    // a restart later the same day finds it kept
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
  });
  test("a forge with no drill owes one only if it is older than the slot", () => {
    const now = t("2026-10-11T09:30:00Z");
    expect(drillDue(undefined, "2026-10-01T00:00:00Z", now)).toBe(true);
    expect(drillDue(undefined, "2026-10-11T09:10:00Z", now)).toBe(false);
    expect(drillDue(undefined, undefined, now)).toBe(false);
  });
});
describe("the service runs a due drill once", () => {
  test("a second look after the drill was kept runs nothing", async () => {
    const root = mkdtempSync(join(tmpdir(), "forge-drill-once-"));
    const app = createApp({ dataDir: root, port: 0 });
    let ran = 0;
    const svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      nightly: false,
      drill: async (): Promise<DrillResult> => {
        ran++;
        return {
          at: new Date().toISOString(),
          snapshotId: "s",
          ms: 1,
          counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
          health: "ok",
          verdict: "green",
          failedStep: null,
          reason: null,
        };
      },
    });
    try {
      // The forge has been here since long before the latest slot.
      mkdirSync(join(root, "drill"), { recursive: true });
      writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
      expect(await svc.drillIfDue()).toBe(true);
      expect(await svc.drillIfDue()).toBe(false);
      expect(ran).toBe(1);
      expect(lastDrill(app.db)?.verdict).toBe("green");
    } finally {
      await svc.stop();
      app.close();
      await rm(root, { recursive: true, force: true });
    }
  });
});
describe("a count that differs is red", () => {
  let root: string;
  let app: App;
  beforeAll(async () => {
    root = mkdtempSync(join(tmpdir(), "forge-drill-weekly-"));
    app = createApp({ dataDir: join(root, "live"), port: 0 });
    const u = await createUser(app.db, {
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!u.ok) throw new Error(u.error);
    const r = await createRepo(app, u.value, { name: "one", visibility: "private" });
    if (!r.ok) throw new Error(String(r.error));
  });
  afterAll(async () => {
    app.close();
    await rm(root, { recursive: true, force: true });
  });
  const snapshot = (name: string, at: string) =>
    takeSnapshot(
      { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
      dirStore(join(root, name)),
      new Date(at),
    );
  const drill = (name: string) =>
    runDrill({
      store: dirStore(join(root, name)),
      dataDir: join(root, "scratch"),
      boot: async () => ({ ok: true, reason: "" }),
    });
  test("the manifest keeps the counts and a matching restore is green, scratch removed", async () => {
    const m = await snapshot("good", "2026-10-10T08:00:00Z");
    expect(m.counts).toMatchObject({ users: 1, repositories: 1 });
    const r = await drill("good");
    expect(r.verdict).toBe("green");
    expect(readdirSync(join(root, "scratch", "drill"))).toEqual([]);
  });
  test("a manifest that disagrees with the restore is red at count, names the count, and raises the line", async () => {
    await snapshot("bad", "2026-10-10T09:00:00Z");
    const path = join(root, "bad", "snapshots", "2026-10-10T09-00-00Z", "manifest.json");
    expect(existsSync(path)).toBe(true);
    const manifest = JSON.parse(await Bun.file(path).text());
    manifest.counts.users = 7;
    await Bun.write(path, JSON.stringify(manifest));
    const r = await drill("bad");
    expect(r.verdict).toBe("red");
    expect(r.failedStep).toBe("count");
    expect(r.reason).toContain("users 7 became 1");
    recordDrill(app.db, r);
    recordVerdict(app.config.dataDir, r);
    expect(lastDrill(app.db)?.verdict).toBe("red");
    const lines = vitals({
      db: app.db,
      dataDir: app.config.dataDir,
      signals: app.signals,
      now: Date.parse(r.at) + 1000,
    }).degraded.filter((d) => d.cause === "drill");
    expect(lines).toHaveLength(1);
    expect(lines[0]!.sentence).toContain("users 7 became 1");
  });
});
		@@ -662,6 +662,12 @@ into scratch, opens it, counts what is there and unseals a value; the page
662	662	waits for it, up to 10 minutes, and the last drill shows there as one line.
663	663	Only one runs at a time. See `docs/SNAPSHOT.md`.
664	664
	665	The forge also runs one itself every Sunday 09:00 UTC (a forge that was down
	666	then runs it five minutes after waking, once). Each snapshot's manifest keeps
	667	the five counts of its database; a restore that counts differently is red at
	668	`count` and, like any red drill, one board line and one push. Snapshots from
	669	before the counts were kept are not compared.
	670
665	671	Failure domains (`/settings/resilience`, not `/resilience`, which is the THREATS.md page; admins only; a non-admin gets the same
666	672	404 as the other admin settings pages): one row per failure domain from
667	673	`docs/DOMAINS.md` — what breaks, the words the forge says, and the domain's
		@@ -0,0 +1,31 @@
	1	/**
	2	* What a forge holds, counted: the same five numbers when a snapshot is
	3	* taken (kept in its manifest) and when the drill restores it, so the two
	4	* can be set side by side. Plain data out.
	5	*/
	6
	7	import type { Database } from "bun:sqlite";
	8
	9	export interface DrillCounts {
	10	readonly repositories: number;
	11	readonly issues: number;
	12	readonly users: number;
	13	readonly pulls: number;
	14	readonly apps: number;
	15	}
	16
	17	export function countRows(db: Database): DrillCounts {
	18	const n = (sql: string): number => db.query<{ n: number }, []>(sql).get()?.n ?? 0;
	19	return {
	20	repositories: n("SELECT count(*) AS n FROM repo"),
	21	issues: n("SELECT count(*) AS n FROM issue"),
	22	users: n("SELECT count(*) AS n FROM user"),
	23	pulls: n("SELECT count(*) AS n FROM pull"),
	24	apps: n("SELECT count(*) AS n FROM app"),
	25	};
	26	}
	27
	28	/** The names of the counts that differ, empty when they all match. */
	29	export function mismatched(kept: DrillCounts, found: DrillCounts): string[] {
	30	return (Object.keys(kept) as (keyof DrillCounts)[]).filter((k) => kept[k] !== found[k]);
	31	}
		@@ -14,6 +14,7 @@ import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
14	14	import { join } from "node:path";
15	15	import { bootRestored } from "./boot.ts";
16	16	import { forgetKey, open } from "../kernel/secret.ts";
	17	import { countRows, mismatched, type DrillCounts } from "./counts.ts";
17	18	import { restoreSnapshot } from "./restore.ts";
18	19	import { listSnapshots } from "./snapshot.ts";
19	20	import type { Store } from "./store.ts";
		@@ -35,13 +36,7 @@ export interface DrillDeps {
35	36	readonly boot?: typeof bootRestored \| undefined;
36	37	}
37	38
38		export interface DrillCounts {
39		readonly repositories: number;
40		readonly issues: number;
41		readonly users: number;
42		readonly pulls: number;
43		readonly apps: number;
44		}
	39	export type { DrillCounts };
45	40
46	41	export interface DrillResult {
47	42	readonly at: string;
		@@ -72,7 +67,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
72	67	const at = clock().toISOString();
73	68	const timeoutMs = deps.timeoutMs ?? DRILL_TIMEOUT_MS;
74	69	const restore = deps.restore ?? restoreSnapshot;
75		const counts = { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 };
	70	const counts: { -readonly [K in keyof DrillCounts]: number } = { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 };
76	71	let snapshotId: string \| null = null;
77	72	let step: DrillStep = "fetch";
78	73	let scratch: string \| undefined;
		@@ -103,13 +98,16 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
103	98	// The constructor is lazy; reading the header is what proves a database.
104	99	db.query("PRAGMA schema_version").get();
105	100	step = "count";
106		const n = (sql: string): number =>
107		db.query<{ n: number }, []>(sql).get()?.n ?? 0;
108		counts.repositories = n("SELECT count(*) AS n FROM repo");
109		counts.issues = n("SELECT count(*) AS n FROM issue");
110		counts.users = n("SELECT count(*) AS n FROM user");
111		counts.pulls = n("SELECT count(*) AS n FROM pull");
112		counts.apps = n("SELECT count(*) AS n FROM app");
	101	Object.assign(counts, countRows(db));
	102	// A snapshot that kept its counts must restore to the same ones.
	103	const kept = newest.counts;
	104	const off = kept === undefined ? [] : mismatched(kept, counts);
	105	if (kept !== undefined && off.length > 0)
	106	throw new Error(
	107	`counts differ from the snapshot's: ${off
	108	.map((k) => `${k} ${kept[k as keyof DrillCounts]} became ${counts[k as keyof DrillCounts]}`)
	109	.join(", ")}`,
	110	);
113	111
114	112	step = "unseal";
115	113	const secret = db
		@@ -0,0 +1,30 @@
	1	/**
	2	* When the weekly drill is owed: every Sunday 09:00 UTC is a slot, and a
	3	* slot is kept by one drill at or after it, however it was asked for. A
	4	* forge that was down at the slot runs it on waking; one that restarts
	5	* after the drill ran does not run it twice. Plain data in, plain data out.
	6	*/
	7
	8	const DAY = 86_400_000;
	9
	10	/** The latest Sunday 09:00 UTC at or before `now`, in ms. */
	11	export function lastSlot(now: number): number {
	12	const d = new Date(now);
	13	d.setUTCHours(9, 0, 0, 0);
	14	let t = d.getTime();
	15	while (t > now \|\| new Date(t).getUTCDay() !== 0) t -= DAY;
	16	return t;
	17	}
	18
	19	/**
	20	* Whether a drill is owed at `now`. `lastAt` is the newest drill's time,
	21	* `since` when the forge first owed one; a forge younger than the slot
	22	* owes nothing for it.
	23	*/
	24	export function drillDue(lastAt: string \| undefined, since: string \| undefined, now: number): boolean {
	25	const slot = lastSlot(now);
	26	const last = Date.parse(lastAt ?? "");
	27	if (Number.isFinite(last)) return last < slot;
	28	const began = Date.parse(since ?? "");
	29	return Number.isFinite(began) && began < slot;
	30	}
		@@ -10,7 +10,7 @@
10	10
11	11	import type { App } from "../app.ts";
12	12	import { createLog } from "../kernel/log.ts";
13		import { reportVerdict, seedFirstSeen } from "./verdict.ts";
	13	import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts";
14	14	import {
15	15	KEEP,
16	16	listSnapshots,
		@@ -20,7 +20,8 @@ import {
20	20	type Progress,
21	21	} from "./snapshot.ts";
22	22	import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
23		import { recordDrill } from "./drills.ts";
	23	import { drillDue } from "./due.ts";
	24	import { lastDrill, recordDrill } from "./drills.ts";
24	25	import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
25	26	import type { Store } from "./store.ts";
26	27
		@@ -44,6 +45,8 @@ export interface SnapshotsHandle {
44	45	readonly last: () => { manifest?: Manifest; error?: string; at?: string };
45	46	/** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
46	47	readonly drill: () => Promise<DrillResult>;
	48	/** Drill if this week's slot has no drill yet; says whether it did. */
	49	readonly drillIfDue: () => Promise<boolean>;
47	50	/** The off-box mirror's store; undefined when the forge has none. */
48	51	readonly mirror: Store \| undefined;
49	52	/** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */
		@@ -64,6 +67,7 @@ export const snapshotsOf = (app: App): SnapshotsHandle \| undefined =>
64	67	/** The weekly restore drill: Sunday, an hour after the nightly snapshot. */
65	68	const DRILL_DAY_UTC = 0;
66	69	const DRILL_HOUR_UTC = 9;
	70	const CATCH_UP_MS = 5 * 60_000;
67	71
68	72	export function untilDrill(now = Date.now()): number {
69	73	const next = new Date(now);
		@@ -235,16 +239,25 @@ export function startSnapshots(
235	239	log.warn("drill: first-seen time not kept; a missing drill will not be flagged", { error: String(error) });
236	240	}
237	241
	242	let catchUp: ReturnType<typeof setTimeout> \| undefined;
238	243	let drillTimer: ReturnType<typeof setTimeout> \| undefined;
239	244	const scheduleDrill = (): void => {
240	245	if (options.nightly === false \|\| stopped) return;
241	246	drillTimer = setTimeout(() => {
242		drill()
243		.catch((error) => log.error("restore drill failed to run", { error: String(error) }))
244		.finally(scheduleDrill);
	247	runIfDue().finally(scheduleDrill);
245	248	}, untilDrill());
246	249	};
	250	// Owed and not yet kept: run it once. A restart after the drill ran finds it kept.
	251	const runIfDue = async (): Promise<boolean> => {
	252	if (!drillDue(lastDrill(app.db)?.at, firstSeen(app.config.dataDir), Date.now())) return false;
	253	await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) }));
	254	return true;
	255	};
247	256	scheduleDrill();
	257	// A forge that was down at the slot catches up shortly after boot.
	258	if (options.nightly !== false) {
	259	catchUp = setTimeout(() => void runIfDue(), CATCH_UP_MS);
	260	}
		junipercrew 29d ago Catch-up fires once at boot+5min and the weekly timer then also calls runIfDue; fine, but if the drill throws, nothing is recorded so it will be retried only next Sunday. Consider recording a red result on thrown errors so the board line appears.
248	261
249	262	// Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
250	263	let mirrorTimer: ReturnType<typeof setTimeout> \| undefined;
		@@ -266,6 +279,7 @@ export function startSnapshots(
266	279	wraps: !!options.passphrase,
267	280	now: take,
268	281	drill,
	282	drillIfDue: runIfDue,
269	283	list: async () => {
270	284	const store = current();
271	285	return store === undefined ? [] : listSnapshots(store);
		@@ -286,6 +300,7 @@ export function startSnapshots(
286	300	stopped = true;
287	301	if (timer !== undefined) clearTimeout(timer);
288	302	if (drillTimer !== undefined) clearTimeout(drillTimer);
	303	if (catchUp !== undefined) clearTimeout(catchUp);
289	304	if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
290	305	if (mirroring !== undefined) await mirroring.catch(() => {});
291	306	if (inFlight !== undefined) await inFlight.catch(() => {});
		@@ -21,7 +21,7 @@
21	21	* lists them in the order they were handed over, not the order they landed.
22	22	*/
23	23
24		import type { Database } from "bun:sqlite";
	24	import { Database } from "bun:sqlite";
25	25	import { mkdtempSync, readdirSync, rmSync, statSync } from "node:fs";
26	26	import { tmpdir } from "node:os";
27	27	import { join, relative } from "node:path";
		@@ -31,6 +31,7 @@ import { listEveryRepo } from "../core/repos.ts";
31	31	import { runGit } from "../git/env.ts";
32	32	import { repoDir } from "../git/repos.ts";
33	33	import { createLog } from "../kernel/log.ts";
	34	import { countRows, type DrillCounts } from "./counts.ts";
34	35	import type { Store } from "./store.ts";
35	36	import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts";
36	37
		@@ -54,6 +55,8 @@ export interface Manifest {
54	55	readonly apps: number;
55	56	readonly files: readonly SnapshotFile[];
56	57	readonly bytes: number;
	58	/** What the database copy held; the drill checks the restore against it. Absent before the drill compared. */
	59	readonly counts?: DrillCounts;
57	60	/**
58	61	* What the snapshot carries of the forge's key: `wrapped` under the
59	62	* operator's passphrase, `none` (sealed values are lost on restore), or
		@@ -213,6 +216,7 @@ export async function takeSnapshot(
213	216	const prefix = `${SNAPSHOT_PREFIX}${name}/`;
214	217	const scratch = mkdtempSync(join(tmpdir(), "forge-snapshot-"));
215	218	const files: SnapshotFile[] = [];
	219	let dbCounts: DrillCounts \| undefined;
216	220	const started = Date.now();
217	221	const everyObjects = forge.progressEvery?.objects ?? PROGRESS_OBJECTS;
218	222	const everyMs = forge.progressEvery?.ms ?? PROGRESS_MS;
		@@ -293,6 +297,14 @@ export async function takeSnapshot(
293	297	// The database, consistently, from the live connection.
294	298	const dbCopy = join(scratch, "forge.db");
295	299	forge.db.run("VACUUM INTO ?1", [dbCopy]);
	300	const copy = new Database(dbCopy, { readonly: true });
	301	try {
	302	dbCounts = countRows(copy);
	303	} catch {
	304	// A database that cannot be counted is the drill's to find; the snapshot still goes up.
	305	} finally {
	306	copy.close();
	307	}
296	308	await putFile("forge.db", dbCopy);
297	309	// The key travels only wrapped. Without a passphrase it stays home.
298	310	let key: Manifest["key"] = "none";
		@@ -420,6 +432,7 @@ export async function takeSnapshot(
420	432	apps: kept,
421	433	files,
422	434	bytes: files.reduce((n, f) => n + f.bytes, 0),
	435	...(dbCounts === undefined ? {} : { counts: dbCounts }),
423	436	key,
424	437	};
425	438	await store.put(
		@@ -96,8 +96,8 @@ function drillLine(ctx: Ctx): Html {
96	96	? html`<p class="backups-drill">No restore drill has run yet.</p>`
97	97	: d.verdict === "green"
98	98	? html`<p class="backups-drill">
99		${icon("check")} Last drill ${time(d.at)}: snapshot
100		<code>${d.snapshotId ?? ""}</code> restored green in ${d.ms} ms —
	99	${icon("check")} Last drill: ${time(d.at)}, ${d.ms} ms, all counts matched —
	100	snapshot <code>${d.snapshotId ?? ""}</code>, key opened —
101	101	${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.
102	102	</p>`
103	103	: html`<p class="backups-error backups-drill">
		@@ -0,0 +1,155 @@
	1	/**
	2	* The weekly drill runs on its own: owed once per Sunday slot, kept after,
	3	* and a count that differs from the snapshot's is red and a line.
	4	*/
	5	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	6	import { existsSync, mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs";
	7	import { rm } from "node:fs/promises";
	8	import { tmpdir } from "node:os";
	9	import { join } from "node:path";
	10	import { createRepo } from "../src/actions/repos.ts";
	11	import { createApp, type App } from "../src/app.ts";
	12	import { createUser } from "../src/core/users.ts";
	13	import { vitals } from "../src/kernel/vitals.ts";
	14	import { runDrill, type DrillResult } from "../src/snapshot/drill.ts";
	15	import { drillDue, lastSlot } from "../src/snapshot/due.ts";
	16	import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
	17	import { startSnapshots } from "../src/snapshot/service.ts";
	18	import { takeSnapshot } from "../src/snapshot/snapshot.ts";
	19	import { dirStore } from "../src/snapshot/store.ts";
	20	import { recordVerdict } from "../src/snapshot/verdict.ts";
	21
	22	const t = (iso: string): number => Date.parse(iso);
	23
	24	describe("when a weekly drill is owed", () => {
	25	// 2026-10-11 is a Sunday.
	26	test("the slot is the latest Sunday 09:00 UTC", () => {
	27	expect(new Date(lastSlot(t("2026-10-16T12:00:00Z"))).toISOString()).toBe("2026-10-11T09:00:00.000Z");
	28	expect(new Date(lastSlot(t("2026-10-11T08:59:00Z"))).toISOString()).toBe("2026-10-04T09:00:00.000Z");
	29	});
	30
	31	test("owed once: a drill since the slot keeps it, an older one does not", () => {
	32	const now = t("2026-10-11T09:30:00Z");
	33	expect(drillDue("2026-10-04T09:05:00.000Z", undefined, now)).toBe(true);
	34	expect(drillDue("2026-10-11T09:05:00.000Z", undefined, now)).toBe(false);
	35	// a restart later the same day finds it kept
	36	expect(drillDue("2026-10-11T09:05:00.000Z", undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
	37	});
	38
	39	test("a forge with no drill owes one only if it is older than the slot", () => {
	40	const now = t("2026-10-11T09:30:00Z");
	41	expect(drillDue(undefined, "2026-10-01T00:00:00Z", now)).toBe(true);
	42	expect(drillDue(undefined, "2026-10-11T09:10:00Z", now)).toBe(false);
	43	expect(drillDue(undefined, undefined, now)).toBe(false);
	44	});
	45	});
	46
	47	describe("the service runs a due drill once", () => {
	48	test("a second look after the drill was kept runs nothing", async () => {
	49	const root = mkdtempSync(join(tmpdir(), "forge-drill-once-"));
	50	const app = createApp({ dataDir: root, port: 0 });
	51	let ran = 0;
	52	const svc = startSnapshots(app, dirStore(join(root, "store")), {
	53	version: "t",
	54	nightly: false,
	55	drill: async (): Promise<DrillResult> => {
	56	ran++;
	57	return {
	58	at: new Date().toISOString(),
	59	snapshotId: "s",
	60	ms: 1,
	61	counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
	62	health: "ok",
	63	verdict: "green",
	64	failedStep: null,
	65	reason: null,
	66	};
	67	},
	68	});
	69	try {
	70	// The forge has been here since long before the latest slot.
	71	mkdirSync(join(root, "drill"), { recursive: true });
	72	writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
	73	expect(await svc.drillIfDue()).toBe(true);
	74	expect(await svc.drillIfDue()).toBe(false);
	75	expect(ran).toBe(1);
	76	expect(lastDrill(app.db)?.verdict).toBe("green");
	77	} finally {
	78	await svc.stop();
	79	app.close();
	80	await rm(root, { recursive: true, force: true });
	81	}
	82	});
	83	});
	84
	85	describe("a count that differs is red", () => {
	86	let root: string;
	87	let app: App;
	88
	89	beforeAll(async () => {
	90	root = mkdtempSync(join(tmpdir(), "forge-drill-weekly-"));
	91	app = createApp({ dataDir: join(root, "live"), port: 0 });
	92	const u = await createUser(app.db, {
	93	trusted: true,
	94	username: "alice",
	95	email: "[email protected]",
	96	password: "a-good-passphrase",
	97	adminIfFirst: true,
	98	});
	99	if (!u.ok) throw new Error(u.error);
	100	const r = await createRepo(app, u.value, { name: "one", visibility: "private" });
	101	if (!r.ok) throw new Error(String(r.error));
	102	});
	103
	104	afterAll(async () => {
	105	app.close();
	106	await rm(root, { recursive: true, force: true });
	107	});
	108
	109	const snapshot = (name: string, at: string) =>
	110	takeSnapshot(
	111	{ db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
	112	dirStore(join(root, name)),
	113	new Date(at),
	114	);
	115	const drill = (name: string) =>
	116	runDrill({
	117	store: dirStore(join(root, name)),
	118	dataDir: join(root, "scratch"),
	119	boot: async () => ({ ok: true, reason: "" }),
	120	});
	121
	122	test("the manifest keeps the counts and a matching restore is green, scratch removed", async () => {
	123	const m = await snapshot("good", "2026-10-10T08:00:00Z");
	124	expect(m.counts).toMatchObject({ users: 1, repositories: 1 });
	125	const r = await drill("good");
	126	expect(r.verdict).toBe("green");
	127	expect(readdirSync(join(root, "scratch", "drill"))).toEqual([]);
	128	});
	129
	130	test("a manifest that disagrees with the restore is red at count, names the count, and raises the line", async () => {
	131	await snapshot("bad", "2026-10-10T09:00:00Z");
	132	const path = join(root, "bad", "snapshots", "2026-10-10T09-00-00Z", "manifest.json");
	133	expect(existsSync(path)).toBe(true);
	134	const manifest = JSON.parse(await Bun.file(path).text());
	135	manifest.counts.users = 7;
	136	await Bun.write(path, JSON.stringify(manifest));
	137
	138	const r = await drill("bad");
	139	expect(r.verdict).toBe("red");
	140	expect(r.failedStep).toBe("count");
	141	expect(r.reason).toContain("users 7 became 1");
	142
	143	recordDrill(app.db, r);
	144	recordVerdict(app.config.dataDir, r);
	145	expect(lastDrill(app.db)?.verdict).toBe("red");
	146	const lines = vitals({
	147	db: app.db,
	148	dataDir: app.config.dataDir,
	149	signals: app.signals,
	150	now: Date.parse(r.at) + 1000,
	151	}).degraded.filter((d) => d.cause === "drill");
	152	expect(lines).toHaveLength(1);
	153	expect(lines[0]!.sentence).toContain("users 7 became 1");
	154	});
	155	});