01bad0c · 2989770 · 598f359
@@ -16,8 +16,8 @@
Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
on a side port and reads `/api/v1/health`. One comment lands on the recovery
issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
time, health, counts, pass or fail. A failed drill is an inbox card naming
the step (fetch, restore, open, count, unseal, health) until one passes. No
comment this week means the backups are unproven.
the step (fetch, restore, open, count, unseal, health), run again at boot and
every three hours until one passes. No comment this week: backups unproven.
## What a restore needs
@@ -139,6 +139,15 @@
run is a row in the `drill` table, and the last one shows as a line on the
Backups page: when, which snapshot, how long, the counts, and green or the
step that failed.
The drill runs on its own every Sunday 09:00 UTC, and a forge that was down
then catches up five minutes after boot. A red drill does not keep the
week's slot: it is owed again `RED_RETRY_MS` (three hours) after it ran,
and at a boot once it is `BOOT_RED_FLOOR_MS` (30 minutes) old, until one
passes (`src/snapshot/due.ts`). The floor keeps a crash loop with a broken
restore from running a restore on every boot. Kept for the
week, a drill red at 09:11 stood as a card long after the fix deployed at
10:18 (#2541).
## Restore
The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
@@ -2,11 +2,21 @@
 * When the weekly drill is owed: every Sunday 09:00 UTC is a slot, and a
 * slot is kept by one drill at or after it, however it was asked for. A
 * forge that was down at the slot runs it on waking; one that restarts
 * after the drill ran does not run it twice. Plain data in, plain data out.
 * after the drill ran does not run it twice. A red drill keeps no slot for
 * long: it is owed again `RED_RETRY_MS` after it ran, and on a boot once it
 * is `BOOT_RED_FLOOR_MS` old, since a restart is most often the deploy of
 * its fix (#2541: a drill red at 09:11 stood as a card for the week after
 * the fix landed at 10:18). The floor keeps a crash loop with a broken
 * restore from running a restore on every boot. Plain data in, plain data out.
 */
const DAY = 86_400_000;
/** A red drill is owed again this long after it ran. */
export const RED_RETRY_MS = 3 * 3_600_000;
/** On a boot, a red drill at least this old is owed again. */
export const BOOT_RED_FLOOR_MS = 30 * 60_000;
/** The latest Sunday 09:00 UTC at or before `now`, in ms. */
export function lastSlot(now: number): number {
  const d = new Date(now);
@@ -17,14 +27,20 @@
export function lastSlot(now: number): number {
}
/**
 * Whether a drill is owed at `now`. `lastAt` is the newest drill's time,
 * `since` when the forge first owed one; a forge younger than the slot
 * owes nothing for it.
 * Whether a drill is owed at `now`. `last` is the newest drill, `since`
 * when the forge first owed one; a forge younger than the slot owes
 * nothing for it. `redRetryMs` is how long a red drill stands before it is
 * owed again; a boot passes `BOOT_RED_FLOOR_MS`.
 */
export function drillDue(lastAt: string | undefined, since: string | undefined, now: number): boolean {
export function drillDue(
  last: { readonly at: string; readonly verdict: "green" | "red" } | undefined,
  since: string | undefined,
  now: number,
  redRetryMs: number = RED_RETRY_MS,
): boolean {
  const slot = lastSlot(now);
  const last = Date.parse(lastAt ?? "");
  if (Number.isFinite(last)) return last < slot;
  const at = Date.parse(last?.at ?? "");
  if (Number.isFinite(at)) return at < slot || (last?.verdict === "red" && now - at >= redRetryMs);
  const began = Date.parse(since ?? "");
  return Number.isFinite(began) && began < slot;
}
@@ -22,7 +22,7 @@
import {
  type Progress,
} from "./snapshot.ts";
import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { drillDue } from "./due.ts";
import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS } from "./due.ts";
import { lastDrill, recordDrill } from "./drills.ts";
import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
import { recordSnapshotOutcome } from "./outcome.ts";
@@ -61,7 +61,7 @@
export interface SnapshotsHandle {
  readonly last: () => { manifest?: Manifest; error?: string; at?: string };
  /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
  readonly drill: () => Promise<DrillResult>;
  /** Drill if this week's slot has no drill yet; says whether it did. */
  /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */
  readonly drillIfDue: () => Promise<boolean>;
  /** The off-box mirror's store; undefined when the forge has none. */
  readonly mirror: Store | undefined;
@@ -115,6 +115,10 @@
export function startSnapshots(
    readonly passphrase?: string | undefined;
    /** Test seam: the drill itself. */
    readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
    /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */
    readonly catchUpMs?: number | undefined;
    readonly redRetryMs?: number | undefined;
    readonly bootRedFloorMs?: number | undefined;
  } = {
    version: "0",
  },
@@ -244,6 +248,8 @@
export function startSnapshots(
      return result;
    })().finally(() => {
      drilling = undefined;
      // However it was asked for, a red drill is tried again on its own.
      armRedRetry();
    });
    return drilling;
  };
@@ -276,22 +282,40 @@
export function startSnapshots(
  let catchUp: ReturnType<typeof setTimeout> | undefined;
  let drillTimer: ReturnType<typeof setTimeout> | undefined;
  let redRetry: ReturnType<typeof setTimeout> | undefined;
  const catchUpMs = options.catchUpMs ?? CATCH_UP_MS;
  const redRetryMs = options.redRetryMs ?? RED_RETRY_MS;
  const scheduleDrill = (): void => {
    if (options.nightly === false || stopped) return;
    drillTimer = setTimeout(() => {
      runIfDue().finally(scheduleDrill);
    }, untilDrill());
  };
  // Owed and not yet kept: run it once. A restart after the drill ran finds it kept.
  const runIfDue = async (): Promise<boolean> => {
    if (!drillDue(lastDrill(app.db)?.at, firstSeen(app.config.dataDir), Date.now())) return false;
  // While the newest drill is red, look again when it is owed again.
  const armRedRetry = (): void => {
      
        junipercrew
        27d ago
        outdated
      
      This timer, its re-arming after each drill, and the boot catch-up with 0 have no test: the service test runs with nightly:false, which returns before here. Make the delays injectable and cover the arm, re-run and stop paths.
    
    if (redRetry !== undefined) clearTimeout(redRetry);
    redRetry = undefined;
    if (options.nightly === false || stopped) return;
    const newest = lastDrill(app.db);
    if (newest?.verdict !== "red") return;
    const wait = Math.max(catchUpMs, Date.parse(newest.at) + redRetryMs - Date.now());
    redRetry = setTimeout(() => void runIfDue().finally(armRedRetry), wait);
  };
  // Owed and not yet kept: run it once. A restart after a green drill finds it
  // kept; one after a red drill runs it again, once it is past the boot floor.
  const runIfDue = async (redAfterMs: number = redRetryMs): Promise<boolean> => {
    if (!drillDue(lastDrill(app.db), firstSeen(app.config.dataDir), Date.now(), redAfterMs)) return false;
    await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) }));
    return true;
  };
  scheduleDrill();
  // A forge that was down at the slot catches up shortly after boot.
  // A forge that was down at the slot, or whose last drill was red, drills
  // shortly after boot; a red one is looked at again until one passes.
  if (options.nightly !== false) {
    catchUp = setTimeout(() => void runIfDue(), CATCH_UP_MS);
    catchUp = setTimeout(
      () => void runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS).finally(armRedRetry),
      catchUpMs,
    );
  }
  // Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
@@ -314,7 +338,7 @@
export function startSnapshots(
    wraps: !!options.passphrase,
    now: take,
    drill,
    drillIfDue: runIfDue,
    drillIfDue: () => runIfDue(),
    list: async () => {
      const store = current();
      return store === undefined ? [] : listSnapshots(store);
@@ -336,6 +360,7 @@
export function startSnapshots(
      if (timer !== undefined) clearTimeout(timer);
      if (drillTimer !== undefined) clearTimeout(drillTimer);
      if (catchUp !== undefined) clearTimeout(catchUp);
      if (redRetry !== undefined) clearTimeout(redRetry);
      if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
      if (mirroring !== undefined) await mirroring.catch(() => {});
      if (inFlight !== undefined) await inFlight.catch(() => {});
@@ -12,7 +12,7 @@
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { runDrill, type DrillResult } from "../src/snapshot/drill.ts";
import { drillDue, lastSlot } from "../src/snapshot/due.ts";
import { BOOT_RED_FLOOR_MS, drillDue, lastSlot, RED_RETRY_MS } from "../src/snapshot/due.ts";
import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
import { startSnapshots } from "../src/snapshot/service.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
@@ -20,6 +20,8 @@
import { dirStore } from "../src/snapshot/store.ts";
import { recordVerdict } from "../src/snapshot/verdict.ts";
const t = (iso: string): number => Date.parse(iso);
const green = (at: string) => ({ at, verdict: "green" as const });
const red = (at: string) => ({ at, verdict: "red" as const });
describe("when a weekly drill is owed", () => {
  // 2026-10-11 is a Sunday.
@@ -30,10 +32,23 @@
describe("when a weekly drill is owed", () => {
  test("owed once: a drill since the slot keeps it, an older one does not", () => {
    const now = t("2026-10-11T09:30:00Z");
    expect(drillDue("2026-10-04T09:05:00.000Z", undefined, now)).toBe(true);
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, now)).toBe(false);
    expect(drillDue(green("2026-10-04T09:05:00.000Z"), undefined, now)).toBe(true);
    expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, now)).toBe(false);
    // a restart later the same day finds it kept
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
    expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
  });
  test("a red drill is owed again after RED_RETRY_MS, and on a boot past the floor", () => {
    // #2541: red at 09:11, the fix deployed at 10:18; the card stood for the week.
    const at = "2026-10-18T09:11:45.853Z";
    expect(drillDue(red(at), undefined, t(at) + 60_000)).toBe(false);
    expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS - 1)).toBe(false);
    expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS)).toBe(true);
    expect(drillDue(red(at), undefined, t("2026-10-18T10:23:00Z"), BOOT_RED_FLOOR_MS)).toBe(true);
    // a boot soon after the red one (a crash loop) does not restore again
    expect(drillDue(red(at), undefined, t(at) + BOOT_RED_FLOOR_MS - 1, BOOT_RED_FLOOR_MS)).toBe(false);
    // a green one keeps its slot on a boot too
    expect(drillDue(green(at), undefined, t("2026-10-18T10:23:00Z"), 0)).toBe(false);
  });
  test("a forge with no drill owes one only if it is older than the slot", () => {
@@ -80,6 +95,152 @@
describe("the service runs a due drill once", () => {
      await rm(root, { recursive: true, force: true });
    }
  });
  test("a red drill that has stood RED_RETRY_MS is run again until one passes", async () => {
    const root = mkdtempSync(join(tmpdir(), "forge-drill-red-"));
    const app = createApp({ dataDir: root, port: 0 });
    const verdicts: DrillResult["verdict"][] = [];
    const svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      nightly: false,
      drill: async (): Promise<DrillResult> => {
        const verdict = verdicts.length === 0 ? "red" : "green";
        verdicts.push(verdict);
        return {
          // The red one ran long enough ago to be owed again.
          at: new Date(Date.now() - (verdict === "red" ? RED_RETRY_MS : 0)).toISOString(),
          snapshotId: "s",
          ms: 1,
          counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
          health: verdict === "red" ? "unreached" : "ok",
          verdict,
          failedStep: verdict === "red" ? "restore" : null,
          reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
        };
      },
    });
    try {
      mkdirSync(join(root, "drill"), { recursive: true });
      writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
      expect(await svc.drillIfDue()).toBe(true);
      expect(lastDrill(app.db)?.verdict).toBe("red");
      expect(await svc.drillIfDue()).toBe(true);
      expect(lastDrill(app.db)?.verdict).toBe("green");
      expect(await svc.drillIfDue()).toBe(false);
      expect(verdicts).toEqual(["red", "green"]);
    } finally {
      await svc.stop();
      app.close();
      await rm(root, { recursive: true, force: true });
    }
  });
});
describe("the service's own clock drills a red one again", () => {
  const result = (verdict: DrillResult["verdict"]): DrillResult => ({
    at: new Date().toISOString(),
    snapshotId: "s",
    ms: 1,
    counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
    health: verdict === "red" ? "unreached" : "ok",
    verdict,
    failedStep: verdict === "red" ? "restore" : null,
    reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
  });
  /**
   * A forge booted with nightly on and short delays, its newest drill
   * `before` (kept before boot), each drill it runs answering the next of
   * `answers`; and the time it was booted, for the waits.
   */
  async function boot(
    before: DrillResult["verdict"] | undefined,
    answers: DrillResult["verdict"][],
    opts: { bootRedFloorMs?: number } = {},
  ) {
    const root = mkdtempSync(join(tmpdir(), "forge-drill-clock-"));
    mkdirSync(join(root, "drill"), { recursive: true });
    writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
    const app = createApp({ dataDir: root, port: 0 });
    if (before !== undefined) recordDrill(app.db, result(before));
    const ran: DrillResult["verdict"][] = [];
    const svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      mirrorEveryMs: 3_600_000,
      catchUpMs: 20,
      redRetryMs: 60,
      bootRedFloorMs: opts.bootRedFloorMs ?? 0,
      drill: async () => {
        const verdict = answers[ran.length] ?? "green";
        ran.push(verdict);
        return result(verdict);
      },
    });
    const done = async () => {
      await svc.stop();
      app.close();
      await rm(root, { recursive: true, force: true });
    };
    return { app, svc, ran, done };
  }
  test("the boot catch-up drills a red one at once", async () => {
    const f = await boot("red", ["green"]);
    try {
      await Bun.sleep(150);
      expect(f.ran).toEqual(["green"]);
      expect(lastDrill(f.app.db)?.verdict).toBe("green");
    } finally {
      await f.done();
    }
  });
  test("the boot catch-up leaves a green one of this week alone", async () => {
    const f = await boot("green", []);
    try {
      await Bun.sleep(150);
      expect(f.ran).toEqual([]);
    } finally {
      await f.done();
    }
  });
  test("a boot inside the floor does not restore again, and the retry still comes", async () => {
    const f = await boot("red", ["green"], { bootRedFloorMs: 3_600_000 });
    try {
      await Bun.sleep(40);
      expect(f.ran).toEqual([]);
      await Bun.sleep(150);
      expect(f.ran).toEqual(["green"]);
    } finally {
      await f.done();
    }
  });
  test("a red result arms the retry, which drills again and stops after a pass", async () => {
    const f = await boot(undefined, ["red", "red", "green"]);
    try {
      await Bun.sleep(400);
      expect(f.ran).toEqual(["red", "red", "green"]);
      await Bun.sleep(200);
      expect(f.ran).toEqual(["red", "red", "green"]);
    } finally {
      await f.done();
    }
  });
  test("stop clears the retry", async () => {
    const f = await boot(undefined, ["red"]);
    try {
      await Bun.sleep(40);
      expect(f.ran).toEqual(["red"]);
      await f.svc.stop();
      await Bun.sleep(200);
      expect(f.ran).toEqual(["red"]);
    } finally {
      await f.done();
    }
  });
});
describe("a count that differs is red", () => {
		@@ -16,8 +16,8 @@ Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
16	16	on a side port and reads `/api/v1/health`. One comment lands on the recovery
17	17	issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
18	18	time, health, counts, pass or fail. A failed drill is an inbox card naming
19		the step (fetch, restore, open, count, unseal, health) until one passes. No
20		comment this week means the backups are unproven.
	19	the step (fetch, restore, open, count, unseal, health), run again at boot and
	20	every three hours until one passes. No comment this week: backups unproven.
21	21
22	22	## What a restore needs
23	23
		@@ -139,6 +139,15 @@ run is a row in the `drill` table, and the last one shows as a line on the
139	139	Backups page: when, which snapshot, how long, the counts, and green or the
140	140	step that failed.
141	141
	142	The drill runs on its own every Sunday 09:00 UTC, and a forge that was down
	143	then catches up five minutes after boot. A red drill does not keep the
	144	week's slot: it is owed again `RED_RETRY_MS` (three hours) after it ran,
	145	and at a boot once it is `BOOT_RED_FLOOR_MS` (30 minutes) old, until one
	146	passes (`src/snapshot/due.ts`). The floor keeps a crash loop with a broken
	147	restore from running a restore on every boot. Kept for the
	148	week, a drill red at 09:11 stood as a card long after the fix deployed at
	149	10:18 (#2541).
	150
142	151	## Restore
143	152
144	153	The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
		@@ -2,11 +2,21 @@
2	2	* When the weekly drill is owed: every Sunday 09:00 UTC is a slot, and a
3	3	* slot is kept by one drill at or after it, however it was asked for. A
4	4	* forge that was down at the slot runs it on waking; one that restarts
5		* after the drill ran does not run it twice. Plain data in, plain data out.
	5	* after the drill ran does not run it twice. A red drill keeps no slot for
	6	* long: it is owed again `RED_RETRY_MS` after it ran, and on a boot once it
	7	* is `BOOT_RED_FLOOR_MS` old, since a restart is most often the deploy of
	8	* its fix (#2541: a drill red at 09:11 stood as a card for the week after
	9	* the fix landed at 10:18). The floor keeps a crash loop with a broken
	10	* restore from running a restore on every boot. Plain data in, plain data out.
6	11	*/
7	12
8	13	const DAY = 86_400_000;
9	14
	15	/** A red drill is owed again this long after it ran. */
	16	export const RED_RETRY_MS = 3 * 3_600_000;
	17	/** On a boot, a red drill at least this old is owed again. */
	18	export const BOOT_RED_FLOOR_MS = 30 * 60_000;
	19
10	20	/** The latest Sunday 09:00 UTC at or before `now`, in ms. */
11	21	export function lastSlot(now: number): number {
12	22	const d = new Date(now);
		@@ -17,14 +27,20 @@ export function lastSlot(now: number): number {
17	27	}
18	28
19	29	/**
20		* Whether a drill is owed at `now`. `lastAt` is the newest drill's time,
21		* `since` when the forge first owed one; a forge younger than the slot
22		* owes nothing for it.
	30	* Whether a drill is owed at `now`. `last` is the newest drill, `since`
	31	* when the forge first owed one; a forge younger than the slot owes
	32	* nothing for it. `redRetryMs` is how long a red drill stands before it is
	33	* owed again; a boot passes `BOOT_RED_FLOOR_MS`.
23	34	*/
24		export function drillDue(lastAt: string \| undefined, since: string \| undefined, now: number): boolean {
	35	export function drillDue(
	36	last: { readonly at: string; readonly verdict: "green" \| "red" } \| undefined,
	37	since: string \| undefined,
	38	now: number,
	39	redRetryMs: number = RED_RETRY_MS,
	40	): boolean {
25	41	const slot = lastSlot(now);
26		const last = Date.parse(lastAt ?? "");
27		if (Number.isFinite(last)) return last < slot;
	42	const at = Date.parse(last?.at ?? "");
	43	if (Number.isFinite(at)) return at < slot \|\| (last?.verdict === "red" && now - at >= redRetryMs);
28	44	const began = Date.parse(since ?? "");
29	45	return Number.isFinite(began) && began < slot;
30	46	}
		@@ -22,7 +22,7 @@ import {
22	22	type Progress,
23	23	} from "./snapshot.ts";
24	24	import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
25		import { drillDue } from "./due.ts";
	25	import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS } from "./due.ts";
26	26	import { lastDrill, recordDrill } from "./drills.ts";
27	27	import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
28	28	import { recordSnapshotOutcome } from "./outcome.ts";
		@@ -61,7 +61,7 @@ export interface SnapshotsHandle {
61	61	readonly last: () => { manifest?: Manifest; error?: string; at?: string };
62	62	/** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
63	63	readonly drill: () => Promise<DrillResult>;
64		/** Drill if this week's slot has no drill yet; says whether it did. */
	64	/** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */
65	65	readonly drillIfDue: () => Promise<boolean>;
66	66	/** The off-box mirror's store; undefined when the forge has none. */
67	67	readonly mirror: Store \| undefined;
		@@ -115,6 +115,10 @@ export function startSnapshots(
115	115	readonly passphrase?: string \| undefined;
116	116	/** Test seam: the drill itself. */
117	117	readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) \| undefined;
	118	/** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */
	119	readonly catchUpMs?: number \| undefined;
	120	readonly redRetryMs?: number \| undefined;
	121	readonly bootRedFloorMs?: number \| undefined;
118	122	} = {
119	123	version: "0",
120	124	},
		@@ -244,6 +248,8 @@ export function startSnapshots(
244	248	return result;
245	249	})().finally(() => {
246	250	drilling = undefined;
	251	// However it was asked for, a red drill is tried again on its own.
	252	armRedRetry();
247	253	});
248	254	return drilling;
249	255	};
		@@ -276,22 +282,40 @@ export function startSnapshots(
276	282
277	283	let catchUp: ReturnType<typeof setTimeout> \| undefined;
278	284	let drillTimer: ReturnType<typeof setTimeout> \| undefined;
	285	let redRetry: ReturnType<typeof setTimeout> \| undefined;
	286	const catchUpMs = options.catchUpMs ?? CATCH_UP_MS;
	287	const redRetryMs = options.redRetryMs ?? RED_RETRY_MS;
279	288	const scheduleDrill = (): void => {
280	289	if (options.nightly === false \|\| stopped) return;
281	290	drillTimer = setTimeout(() => {
282	291	runIfDue().finally(scheduleDrill);
283	292	}, untilDrill());
284	293	};
285		// Owed and not yet kept: run it once. A restart after the drill ran finds it kept.
286		const runIfDue = async (): Promise<boolean> => {
287		if (!drillDue(lastDrill(app.db)?.at, firstSeen(app.config.dataDir), Date.now())) return false;
	294	// While the newest drill is red, look again when it is owed again.
	295	const armRedRetry = (): void => {
		junipercrew 27d ago outdated This timer, its re-arming after each drill, and the boot catch-up with 0 have no test: the service test runs with nightly:false, which returns before here. Make the delays injectable and cover the arm, re-run and stop paths.
	296	if (redRetry !== undefined) clearTimeout(redRetry);
	297	redRetry = undefined;
	298	if (options.nightly === false \|\| stopped) return;
	299	const newest = lastDrill(app.db);
	300	if (newest?.verdict !== "red") return;
	301	const wait = Math.max(catchUpMs, Date.parse(newest.at) + redRetryMs - Date.now());
	302	redRetry = setTimeout(() => void runIfDue().finally(armRedRetry), wait);
	303	};
	304	// Owed and not yet kept: run it once. A restart after a green drill finds it
	305	// kept; one after a red drill runs it again, once it is past the boot floor.
	306	const runIfDue = async (redAfterMs: number = redRetryMs): Promise<boolean> => {
	307	if (!drillDue(lastDrill(app.db), firstSeen(app.config.dataDir), Date.now(), redAfterMs)) return false;
288	308	await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) }));
289	309	return true;
290	310	};
291	311	scheduleDrill();
292		// A forge that was down at the slot catches up shortly after boot.
	312	// A forge that was down at the slot, or whose last drill was red, drills
	313	// shortly after boot; a red one is looked at again until one passes.
293	314	if (options.nightly !== false) {
294		catchUp = setTimeout(() => void runIfDue(), CATCH_UP_MS);
	315	catchUp = setTimeout(
	316	() => void runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS).finally(armRedRetry),
	317	catchUpMs,
	318	);
295	319	}
296	320
297	321	// Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
		@@ -314,7 +338,7 @@ export function startSnapshots(
314	338	wraps: !!options.passphrase,
315	339	now: take,
316	340	drill,
317		drillIfDue: runIfDue,
	341	drillIfDue: () => runIfDue(),
318	342	list: async () => {
319	343	const store = current();
320	344	return store === undefined ? [] : listSnapshots(store);
		@@ -336,6 +360,7 @@ export function startSnapshots(
336	360	if (timer !== undefined) clearTimeout(timer);
337	361	if (drillTimer !== undefined) clearTimeout(drillTimer);
338	362	if (catchUp !== undefined) clearTimeout(catchUp);
	363	if (redRetry !== undefined) clearTimeout(redRetry);
339	364	if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
340	365	if (mirroring !== undefined) await mirroring.catch(() => {});
341	366	if (inFlight !== undefined) await inFlight.catch(() => {});
		@@ -12,7 +12,7 @@ import { createApp, type App } from "../src/app.ts";
12	12	import { createUser } from "../src/core/users.ts";
13	13	import { vitals } from "../src/kernel/vitals.ts";
14	14	import { runDrill, type DrillResult } from "../src/snapshot/drill.ts";
15		import { drillDue, lastSlot } from "../src/snapshot/due.ts";
	15	import { BOOT_RED_FLOOR_MS, drillDue, lastSlot, RED_RETRY_MS } from "../src/snapshot/due.ts";
16	16	import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
17	17	import { startSnapshots } from "../src/snapshot/service.ts";
18	18	import { takeSnapshot } from "../src/snapshot/snapshot.ts";
		@@ -20,6 +20,8 @@ import { dirStore } from "../src/snapshot/store.ts";
20	20	import { recordVerdict } from "../src/snapshot/verdict.ts";
21	21
22	22	const t = (iso: string): number => Date.parse(iso);
	23	const green = (at: string) => ({ at, verdict: "green" as const });
	24	const red = (at: string) => ({ at, verdict: "red" as const });
23	25
24	26	describe("when a weekly drill is owed", () => {
25	27	// 2026-10-11 is a Sunday.
		@@ -30,10 +32,23 @@ describe("when a weekly drill is owed", () => {
30	32
31	33	test("owed once: a drill since the slot keeps it, an older one does not", () => {
32	34	const now = t("2026-10-11T09:30:00Z");
33		expect(drillDue("2026-10-04T09:05:00.000Z", undefined, now)).toBe(true);
34		expect(drillDue("2026-10-11T09:05:00.000Z", undefined, now)).toBe(false);
	35	expect(drillDue(green("2026-10-04T09:05:00.000Z"), undefined, now)).toBe(true);
	36	expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, now)).toBe(false);
35	37	// a restart later the same day finds it kept
36		expect(drillDue("2026-10-11T09:05:00.000Z", undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
	38	expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
	39	});
	40
	41	test("a red drill is owed again after RED_RETRY_MS, and on a boot past the floor", () => {
	42	// #2541: red at 09:11, the fix deployed at 10:18; the card stood for the week.
	43	const at = "2026-10-18T09:11:45.853Z";
	44	expect(drillDue(red(at), undefined, t(at) + 60_000)).toBe(false);
	45	expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS - 1)).toBe(false);
	46	expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS)).toBe(true);
	47	expect(drillDue(red(at), undefined, t("2026-10-18T10:23:00Z"), BOOT_RED_FLOOR_MS)).toBe(true);
	48	// a boot soon after the red one (a crash loop) does not restore again
	49	expect(drillDue(red(at), undefined, t(at) + BOOT_RED_FLOOR_MS - 1, BOOT_RED_FLOOR_MS)).toBe(false);
	50	// a green one keeps its slot on a boot too
	51	expect(drillDue(green(at), undefined, t("2026-10-18T10:23:00Z"), 0)).toBe(false);
37	52	});
38	53
39	54	test("a forge with no drill owes one only if it is older than the slot", () => {
		@@ -80,6 +95,152 @@ describe("the service runs a due drill once", () => {
80	95	await rm(root, { recursive: true, force: true });
81	96	}
82	97	});
	98
	99	test("a red drill that has stood RED_RETRY_MS is run again until one passes", async () => {
	100	const root = mkdtempSync(join(tmpdir(), "forge-drill-red-"));
	101	const app = createApp({ dataDir: root, port: 0 });
	102	const verdicts: DrillResult["verdict"][] = [];
	103	const svc = startSnapshots(app, dirStore(join(root, "store")), {
	104	version: "t",
	105	nightly: false,
	106	drill: async (): Promise<DrillResult> => {
	107	const verdict = verdicts.length === 0 ? "red" : "green";
	108	verdicts.push(verdict);
	109	return {
	110	// The red one ran long enough ago to be owed again.
	111	at: new Date(Date.now() - (verdict === "red" ? RED_RETRY_MS : 0)).toISOString(),
	112	snapshotId: "s",
	113	ms: 1,
	114	counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
	115	health: verdict === "red" ? "unreached" : "ok",
	116	verdict,
	117	failedStep: verdict === "red" ? "restore" : null,
	118	reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
	119	};
	120	},
	121	});
	122	try {
	123	mkdirSync(join(root, "drill"), { recursive: true });
	124	writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
	125	expect(await svc.drillIfDue()).toBe(true);
	126	expect(lastDrill(app.db)?.verdict).toBe("red");
	127	expect(await svc.drillIfDue()).toBe(true);
	128	expect(lastDrill(app.db)?.verdict).toBe("green");
	129	expect(await svc.drillIfDue()).toBe(false);
	130	expect(verdicts).toEqual(["red", "green"]);
	131	} finally {
	132	await svc.stop();
	133	app.close();
	134	await rm(root, { recursive: true, force: true });
	135	}
	136	});
	137	});
	138
	139	describe("the service's own clock drills a red one again", () => {
	140	const result = (verdict: DrillResult["verdict"]): DrillResult => ({
	141	at: new Date().toISOString(),
	142	snapshotId: "s",
	143	ms: 1,
	144	counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
	145	health: verdict === "red" ? "unreached" : "ok",
	146	verdict,
	147	failedStep: verdict === "red" ? "restore" : null,
	148	reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
	149	});
	150
	151	/**
	152	* A forge booted with nightly on and short delays, its newest drill
	153	* `before` (kept before boot), each drill it runs answering the next of
	154	* `answers`; and the time it was booted, for the waits.
	155	*/
	156	async function boot(
	157	before: DrillResult["verdict"] \| undefined,
	158	answers: DrillResult["verdict"][],
	159	opts: { bootRedFloorMs?: number } = {},
	160	) {
	161	const root = mkdtempSync(join(tmpdir(), "forge-drill-clock-"));
	162	mkdirSync(join(root, "drill"), { recursive: true });
	163	writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
	164	const app = createApp({ dataDir: root, port: 0 });
	165	if (before !== undefined) recordDrill(app.db, result(before));
	166	const ran: DrillResult["verdict"][] = [];
	167	const svc = startSnapshots(app, dirStore(join(root, "store")), {
	168	version: "t",
	169	mirrorEveryMs: 3_600_000,
	170	catchUpMs: 20,
	171	redRetryMs: 60,
	172	bootRedFloorMs: opts.bootRedFloorMs ?? 0,
	173	drill: async () => {
	174	const verdict = answers[ran.length] ?? "green";
	175	ran.push(verdict);
	176	return result(verdict);
	177	},
	178	});
	179	const done = async () => {
	180	await svc.stop();
	181	app.close();
	182	await rm(root, { recursive: true, force: true });
	183	};
	184	return { app, svc, ran, done };
	185	}
	186
	187	test("the boot catch-up drills a red one at once", async () => {
	188	const f = await boot("red", ["green"]);
	189	try {
	190	await Bun.sleep(150);
	191	expect(f.ran).toEqual(["green"]);
	192	expect(lastDrill(f.app.db)?.verdict).toBe("green");
	193	} finally {
	194	await f.done();
	195	}
	196	});
	197
	198	test("the boot catch-up leaves a green one of this week alone", async () => {
	199	const f = await boot("green", []);
	200	try {
	201	await Bun.sleep(150);
	202	expect(f.ran).toEqual([]);
	203	} finally {
	204	await f.done();
	205	}
	206	});
	207
	208	test("a boot inside the floor does not restore again, and the retry still comes", async () => {
	209	const f = await boot("red", ["green"], { bootRedFloorMs: 3_600_000 });
	210	try {
	211	await Bun.sleep(40);
	212	expect(f.ran).toEqual([]);
	213	await Bun.sleep(150);
	214	expect(f.ran).toEqual(["green"]);
	215	} finally {
	216	await f.done();
	217	}
	218	});
	219
	220	test("a red result arms the retry, which drills again and stops after a pass", async () => {
	221	const f = await boot(undefined, ["red", "red", "green"]);
	222	try {
	223	await Bun.sleep(400);
	224	expect(f.ran).toEqual(["red", "red", "green"]);
	225	await Bun.sleep(200);
	226	expect(f.ran).toEqual(["red", "red", "green"]);
	227	} finally {
	228	await f.done();
	229	}
	230	});
	231
	232	test("stop clears the retry", async () => {
	233	const f = await boot(undefined, ["red"]);
	234	try {
	235	await Bun.sleep(40);
	236	expect(f.ran).toEqual(["red"]);
	237	await f.svc.stop();
	238	await Bun.sleep(200);
	239	expect(f.ran).toEqual(["red"]);
	240	} finally {
	241	await f.done();
	242	}
	243	});
83	244	});
84	245
85	246	describe("a count that differs is red", () => {