36dd6c4 · a61d966 · 85a74bd · 0f35f4e · b1d3aba
@@ -10,6 +10,15 @@
region `S3_REGION`) under the prefix `snapshots/<name>/`, one directory per
snapshot, named like `2026-09-22T08-00-00Z`. The restore command accepts `latest` for
the newest; no object is called that.
## Where the verdicts are posted
Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
on a side port and reads `/api/v1/health`. One comment lands on the recovery
issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
time, health, counts, pass or fail. A failed drill is an inbox card naming
the step (fetch, restore, open, count, unseal, health) until one passes. No
comment this week means the backups are unproven.
## What a restore needs
Export these, with the values the forge ran under:
@@ -138,6 +138,7 @@
export const GATE: readonly string[] = [
  "test/helpers.test.ts",
  "test/shards.test.ts",
  "test/snapshot-drill.test.ts",
  "test/drill-verdict.test.ts",
  "test/snapshot-drill-page.test.ts",
  "test/snapshot-store.test.ts",
  "test/migrate.test.ts",
@@ -20,6 +20,7 @@
import {
  queueDepth,
} from "../core/runs.ts";
import { STALE_AFTER_MS } from "../runs/lease.ts";
import { failedSentence, lastVerdict } from "../snapshot/verdict.ts";
/** Free disk under this share of the volume is something to say. */
export const DISK_FREE_MIN = 0.1;
@@ -272,6 +273,12 @@
export function vitals(input: VitalsInput): Vitals {
      "/settings/capacity",
    );
  // A failed restore drill is a card until a drill passes.
  const drill = lastVerdict(input.dataDir);
  const failed = failedSentence(drill);
      
        junipercrew
        1mo ago
        
      
      A sync file read per vitals call. Fine at this size; move the verdict into the drill table when a migration is next due.
    
  if (failed !== undefined)
    need("drill", failed, drill?.at, "/settings/backups");
  const apps = { serving: 0, asleep: 0, previews: 0 };
  for (const e of input.envs ?? []) {
    if (e.env !== "production") apps.previews++;
@@ -0,0 +1,70 @@
/**
 * Boot a restored data directory on a side port and read its health, the
 * way `docs/RECOVERY.md` does by hand. The child gets a clean environment:
 * the directory, a free port, no SSH listener, and `FORGE_APP_ENV=preview`
 * so the crew stays off a copy of production's queue. It is killed whatever
 * the answer.
 */
import { join } from "node:path";
export interface Booted {
  readonly ok: boolean;
  /** Why not, for the person reading; never a value. */
  readonly reason: string;
}
const TREE = join(import.meta.dir, "../..");
async function freePort(): Promise<number> {
  const probe = Bun.serve({ port: 0, fetch: () => new Response("") });
  const port = probe.port ?? 0;
  await probe.stop(true);
  return port;
}
export async function bootRestored(
  dataDir: string,
  budgetMs: number,
): Promise<Booted> {
  const port = await freePort();
  const child = Bun.spawn([process.execPath, join(TREE, "src/main.ts")], {
    cwd: TREE,
    env: {
      PATH: process.env["PATH"] ?? "",
      FORGE_DATA: dataDir,
      PORT: String(port),
      FORGE_SSH_PORT: "0",
      FORGE_APP_ENV: "preview",
      // Only a test host sets this: its internal door is a port, not a socket.
      ...(process.env["FORGE_TEST_SOCKET"]
        ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] }
        : {}),
    },
    stdout: "ignore",
    stderr: "ignore",
  });
  const until = performance.now() + Math.min(budgetMs, 60_000);
  let last = "no answer";
  try {
    while (performance.now() < until) {
      if (child.exitCode !== null)
        return { ok: false, reason: `the restored forge exited with ${child.exitCode}` };
      try {
        const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
          signal: AbortSignal.timeout(2_000),
        });
        const body = (await res.json().catch(() => null)) as { ok?: unknown } | null;
        if (res.ok && body?.ok === true) return { ok: true, reason: "" };
        last = `health answered ${res.status}${body?.ok === false ? " not ok" : ""}`;
      } catch {
        last = "no answer yet";
      }
      await Bun.sleep(200);
    }
    return { ok: false, reason: `the restored forge never read healthy: ${last}` };
  } finally {
    child.kill();
    await child.exited;
  }
}
@@ -2,7 +2,8 @@
 * The restore drill: a backup nobody has restored is a hope. `runDrill`
 * takes the newest snapshot, restores it into a fresh directory, opens the
 * database read-only, counts what is in it, opens one sealed value with the
 * restored key, and removes the directory whatever happened.
 * restored key, boots the restored directory on a side port and reads
 * `/api/v1/health`, and removes the directory whatever happened.
 *
 * Plain data out: when it ran, which snapshot, how long, what it counted,
 * and a verdict. A red verdict names the step that failed; a step that
@@ -11,6 +12,7 @@
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { bootRestored } from "./boot.ts";
import { forgetKey, open } from "../kernel/secret.ts";
import { restoreSnapshot } from "./restore.ts";
import { listSnapshots } from "./snapshot.ts";
@@ -18,7 +20,7 @@
import type { Store } from "./store.ts";
export const DRILL_TIMEOUT_MS = 10 * 60_000;
export type DrillStep = "fetch" | "restore" | "open" | "count" | "unseal";
export type DrillStep = "fetch" | "restore" | "open" | "count" | "unseal" | "health";
export interface DrillDeps {
  readonly store: Store;
@@ -29,6 +31,8 @@
export interface DrillDeps {
  readonly now?: (() => Date) | undefined;
  /** Test seam: the restore itself. */
  readonly restore?: typeof restoreSnapshot | undefined;
  /** Test seam: boot a data directory on a side port and say whether it is healthy. */
  readonly boot?: typeof bootRestored | undefined;
}
export interface DrillResult {
@@ -36,6 +40,8 @@
export interface DrillResult {
  readonly snapshotId: string | null;
  readonly ms: number;
  readonly counts: { readonly repositories: number; readonly issues: number };
  /** The side-booted forge's word; "unreached" when an earlier step failed. */
  readonly health: "ok" | "failed" | "unreached";
  readonly verdict: "green" | "red";
  readonly failedStep: DrillStep | null;
  /** Why it is red, for the person reading; never a value. */
@@ -63,6 +69,7 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
  let step: DrillStep = "fetch";
  let scratch: string | undefined;
  let reason: string | null = null;
  let health: DrillResult["health"] = "unreached";
  try {
    // One clock for the whole drill, so a hang in any step ends it red.
    const deadline = started + timeoutMs;
@@ -122,6 +129,14 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
    } finally {
      db.close();
    }
    step = "health";
    const booted = await within(
      (deps.boot ?? bootRestored)(target, left()),
      left(),
    );
    health = booted.ok ? "ok" : "failed";
    if (!booted.ok) throw new Error(booted.reason);
  } catch (error) {
    reason =
      error instanceof StepTimeout
@@ -144,6 +159,7 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
      snapshotId,
      ms: Math.round(performance.now() - started),
      counts,
      health,
      verdict: failedStep === null ? "green" : "red",
      failedStep,
      reason,
@@ -46,6 +46,9 @@
export function lastDrill(db: Database): DrillResult | undefined {
        snapshotId: row.snapshot_id,
        ms: row.ms,
        counts: { repositories: row.repositories, issues: row.issues },
        // Not stored: green means health read ok; a red one failed at it or never got there.
        health:
          row.verdict === "green" ? "ok" : row.failed_step === "health" ? "failed" : "unreached",
        verdict: row.verdict,
        failedStep: row.failed_step,
        reason: row.reason,
@@ -10,6 +10,7 @@
import type { App } from "../app.ts";
import { createLog } from "../kernel/log.ts";
import { reportVerdict } from "./verdict.ts";
import {
  KEEP,
  listSnapshots,
@@ -51,6 +52,18 @@
const handles = new WeakMap<App, SnapshotsHandle>();
export const snapshotsOf = (app: App): SnapshotsHandle | undefined =>
  handles.get(app);
/** The weekly restore drill: Sunday, an hour after the nightly snapshot. */
const DRILL_DAY_UTC = 0;
const DRILL_HOUR_UTC = 9;
export function untilDrill(now = Date.now()): number {
  const next = new Date(now);
  next.setUTCHours(DRILL_HOUR_UTC, 0, 0, 0);
  while (next.getTime() <= now || next.getUTCDay() !== DRILL_DAY_UTC)
    next.setUTCDate(next.getUTCDate() + 1);
  return next.getTime() - now;
}
export function untilNightly(now = Date.now()): number {
  const next = new Date(now);
  next.setUTCHours(NIGHTLY_HOUR_UTC, 0, 0, 0);
@@ -139,6 +152,9 @@
export function startSnapshots(
        passphrase: options.passphrase,
      });
      recordDrill(app.db, result);
      // The card and the recovery issue hear of every drill, whoever asked.
      const posted = await reportVerdict(app.config.dataDir, result);
      log.info("restore drill", { verdict: result.verdict, step: result.failedStep ?? "none", posted });
      return result;
    })().finally(() => {
      drilling = undefined;
@@ -157,6 +173,17 @@
export function startSnapshots(
  };
  schedule();
  let drillTimer: ReturnType<typeof setTimeout> | undefined;
  const scheduleDrill = (): void => {
    if (options.nightly === false || stopped) return;
    drillTimer = setTimeout(() => {
      drill()
        .catch((error) => log.error("restore drill failed to run", { error: String(error) }))
      
        junipercrew
        1mo ago
        
      
      Untested timer. Add a seam for the delay so a test can fire it and check that it drills once and re-arms.
    
        .finally(scheduleDrill);
    }, untilDrill());
  };
  scheduleDrill();
  const handle: SnapshotsHandle = {
    get store() {
      return current();
@@ -176,6 +203,7 @@
export function startSnapshots(
    stop: async () => {
      stopped = true;
      if (timer !== undefined) clearTimeout(timer);
      if (drillTimer !== undefined) clearTimeout(drillTimer);
      if (inFlight !== undefined) await inFlight.catch(() => {});
      if (handles.get(app) === handle) handles.delete(app);
    },
@@ -0,0 +1,96 @@
/**
 * The drill's verdict, told: kept in one small file under the data
 * directory (`drill/last.json`), so the inbox card (`kernel/vitals.ts`)
 * stands for as long as the last drill was red and goes when the next one
 * passes; and posted as one comment on the recovery issue
 * (`FORGE_RECOVERY_REPO`, `FORGE_RECOVERY_ISSUE`, through `FORGE_URL` and
 * `FORGE_TOKEN`). Plain data in, plain data out.
 */
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { DrillResult, DrillStep } from "./drill.ts";
const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health"];
const lastFile = (dataDir: string): string => join(dataDir, "drill", "last.json");
export function recordVerdict(dataDir: string, result: DrillResult): void {
  mkdirSync(join(dataDir, "drill"), { recursive: true });
  const tmp = `${lastFile(dataDir)}.tmp`;
  writeFileSync(tmp, JSON.stringify(result));
  renameSync(tmp, lastFile(dataDir));
}
/** The last drill's verdict, or undefined when none was recorded or the file is not one. */
export function lastVerdict(dataDir: string): DrillResult | undefined {
  try {
    const r = JSON.parse(readFileSync(lastFile(dataDir), "utf8")) as DrillResult;
    if (r.verdict !== "green" && r.verdict !== "red") return undefined;
    if (r.failedStep !== null && !STEPS.includes(r.failedStep)) return undefined;
    if (typeof r.at !== "string") return undefined;
    return r;
  } catch {
    return undefined;
  }
}
/** The inbox card's sentence for a red drill; undefined for a green one. */
export function failedSentence(r: DrillResult | undefined): string | undefined {
  if (r === undefined || r.verdict !== "red") return undefined;
  return `The restore drill failed at ${r.failedStep ?? "an unknown step"}${r.reason ? `: ${r.reason}` : ""}. Snapshot ${r.snapshotId ?? "none"}; docs/RECOVERY.md says what to do.`;
}
/** The comment: snapshot, time taken, health, counts, pass or fail. */
export function verdictComment(r: DrillResult): string {
  const pass = r.verdict === "green";
  return [
    `Restore drill ${pass ? "PASSED" : `FAILED at ${r.failedStep}`} (${r.at}).`,
    "",
    `- snapshot: ${r.snapshotId ?? "none"}`,
    `- restore took: ${(r.ms / 1000).toFixed(1)} s`,
    `- health: ${r.health}`,
    `- repositories: ${r.counts.repositories}`,
    `- issues: ${r.counts.issues}`,
    ...(pass || !r.reason ? [] : [`- why: ${r.reason}`]),
  ].join("\n");
}
export type Poster = (body: string) => Promise<string>;
/** Post on the recovery issue named by the environment; says what it did. */
export const postToRecoveryIssue: Poster = async (body) => {
  const url = process.env["FORGE_URL"];
  const token = process.env["FORGE_TOKEN"];
  const repo = process.env["FORGE_RECOVERY_REPO"] ?? "trevato/forge";
  const issue = Number(process.env["FORGE_RECOVERY_ISSUE"]);
  if (!url || !token || !Number.isInteger(issue) || issue < 1)
    return "no FORGE_URL, FORGE_TOKEN and FORGE_RECOVERY_ISSUE; not posted";
  const res = await fetch(`${url}/api/v1/repos/${repo}/issues/${issue}/comments`, {
    method: "POST",
    // An unresponsive forge must not hold the shared drill open.
    signal: AbortSignal.timeout(10_000),
    headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
    body: JSON.stringify({ body }),
  });
  return res.ok ? `posted: ${res.status}` : `not posted: ${res.status}`;
};
/** Keep the verdict for the card and tell the recovery issue. A failed post never loses the verdict. */
export async function reportVerdict(
  dataDir: string,
  result: DrillResult,
  post: Poster = postToRecoveryIssue,
): Promise<string> {
  // A disk that will not take the file must not keep the comment from going.
  const kept = (() => {
    try {
      recordVerdict(dataDir, result);
      return "";
    } catch (error) {
      return ` (verdict not kept: ${String(error)})`;
    }
  })();
  const said = await post(verdictComment(result)).catch((error) => `not posted: ${String(error)}`);
  return said + kept;
}
@@ -0,0 +1,235 @@
/**
 * The drill tells people. A drill that finishes posts one comment on the
 * recovery issue (snapshot, time, health, counts, verdict); a failed one is
 * an inbox card naming the step, and the next passing drill clears it. A
 * restored directory is booted on a side port and must read healthy.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync } from "node:fs";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRepo } from "../src/actions/repos.ts";
import { createUser } from "../src/core/users.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { bootRestored } from "../src/snapshot/boot.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { runDrill } from "../src/snapshot/drill.ts";
import { untilDrill } from "../src/snapshot/service.ts";
import { lastVerdict, postToRecoveryIssue, reportVerdict, verdictComment } from "../src/snapshot/verdict.ts";
const PASS = "correct horse battery staple";
describe("the drill's verdict", () => {
  let root: string;
  let app: App;
  const store = () => dirStore(join(root, "store"));
  const card = () =>
    vitals({ db: app.db, dataDir: app.config.dataDir, signals: app.signals }).degraded.find(
      (d) => d.cause === "drill",
    );
  beforeAll(async () => {
    root = mkdtempSync(join(tmpdir(), "forge-verdict-"));
    app = createApp({ dataDir: join(root, "live"), port: 0 });
    const u = await createUser(app.db, {
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!u.ok) throw new Error(u.error);
    const repo = await createRepo(app, u.value, { name: "notes", visibility: "private" });
    if (!repo.ok) throw new Error(String(repo.error));
    await takeSnapshot(
      {
        db: app.db,
        dataDir: app.config.dataDir,
        reposRoot: app.reposRoot,
        version: "t",
        passphrase: PASS,
      },
      store(),
      new Date("2026-09-22T08:00:00Z"),
    );
  });
  afterAll(async () => {
    app.close();
    await rm(root, { recursive: true, force: true });
  });
  // The shipped path: `drill()` in the service runs the drill, then reports it.
  const drillAndReport = async (
    d: ReturnType<typeof deps>,
    post: (body: string) => Promise<string>,
  ) => {
    const result = await runDrill(d);
    return { result, posted: await reportVerdict(d.dataDir, result, post) };
  };
  const deps = (boot: () => Promise<{ ok: boolean; reason: string }>) => ({
    store: store(),
    dataDir: app.config.dataDir,
    passphrase: PASS,
    boot,
  });
  test("a stand-in drill posts one comment with snapshot, time, health, counts and the verdict", async () => {
    const posts: string[] = [];
    const { result } = await drillAndReport(
      deps(async () => ({ ok: true, reason: "" })),
      async (body) => {
        posts.push(body);
        return "posted: 201";
      },
    );
    expect(result.verdict).toBe("green");
    expect(result.health).toBe("ok");
    expect(posts).toHaveLength(1);
    expect(posts[0]).toContain("PASSED");
    expect(posts[0]).toContain("snapshot: 2026-09-22T08-00-00Z");
    expect(posts[0]).toContain("health: ok");
    expect(posts[0]).toContain("repositories: 1");
    expect(posts[0]).toContain("issues: 0");
    expect(posts[0]).toMatch(/restore took: \d+\.\d s/);
    expect(card()).toBeUndefined();
  });
  test("a failing drill makes the card naming the step, and the next passing drill clears it", async () => {
    const posts: string[] = [];
    const post = async (body: string) => (posts.push(body), "ok");
    const { result } = await drillAndReport(
      deps(async () => ({ ok: false, reason: "health answered 503" })),
      post,
    );
    expect(result.verdict).toBe("red");
    expect(result.failedStep).toBe("health");
    expect(posts[0]).toContain("FAILED at health");
    const c = card();
    expect(c?.sentence).toContain("failed at health");
    expect(c?.sentence).toContain("health answered 503");
    expect(lastVerdict(app.config.dataDir)?.verdict).toBe("red");
    await drillAndReport(deps(async () => ({ ok: true, reason: "" })), post);
    expect(card()).toBeUndefined();
  });
  test("a post that throws still keeps the verdict", async () => {
    const { posted } = await drillAndReport(
      deps(async () => ({ ok: false, reason: "no" })),
      async () => {
        throw new Error("forge unreachable");
      },
    );
    expect(posted).toContain("forge unreachable");
    expect(card()).toBeDefined();
  });
  test("the comment of a drill that never reached health says so", () => {
    const text = verdictComment({
      at: "2026-09-22T09:00:00Z",
      snapshotId: null,
      ms: 5,
      counts: { repositories: 0, issues: 0 },
      health: "unreached",
      verdict: "red",
      failedStep: "fetch",
      reason: "the store holds no snapshot",
    });
    expect(text).toContain("FAILED at fetch");
    expect(text).toContain("health: unreached");
  });
  test("the real boot reads a restored directory's health on a side port", async () => {
    const dir = join(root, "boot-data");
    const { restoreSnapshot } = await import("../src/snapshot/restore.ts");
    await restoreSnapshot(store(), "2026-09-22T08-00-00Z", dir, { passphrase: PASS });
    const booted = await bootRestored(dir, 40_000);
    expect(booted).toEqual({ ok: true, reason: "" });
  }, 60_000);
});
describe("the weekly schedule", () => {
  const at = (iso: string) => Date.parse(iso);
  const fires = (iso: string) => new Date(at(iso) + untilDrill(at(iso)));
  test("from a Saturday it fires the next day at 09:00 UTC", () => {
    expect(fires("2026-10-17T12:00:00Z").toISOString()).toBe("2026-10-18T09:00:00.000Z");
  });
  test("on a Sunday before 09:00 it fires that morning", () => {
    expect(fires("2026-10-18T08:59:00Z").toISOString()).toBe("2026-10-18T09:00:00.000Z");
  });
  test("on a Sunday at or after 09:00 it waits a week", () => {
    expect(fires("2026-10-18T09:00:00Z").toISOString()).toBe("2026-10-25T09:00:00.000Z");
    expect(fires("2026-10-18T09:00:01Z").toISOString()).toBe("2026-10-25T09:00:00.000Z");
  });
  test("whenever asked, it is a Sunday at 09:00 UTC, within a week", () => {
    for (let h = 0; h < 24 * 14; h += 7) {
      const now = at("2026-01-01T00:30:00Z") + h * 3_600_000;
      const wait = untilDrill(now);
      const d = new Date(now + wait);
      expect([d.getUTCDay(), d.getUTCHours(), d.getUTCMinutes()]).toEqual([0, 9, 0]);
      expect(wait).toBeGreaterThan(0);
      expect(wait).toBeLessThanOrEqual(7 * 24 * 3_600_000);
    }
  });
});
describe("telling the recovery issue", () => {
  const real = globalThis.fetch;
  const env = { ...process.env };
  afterAll(() => {
    globalThis.fetch = real;
    process.env = env;
  });
  const setup = (status: number) => {
    process.env["FORGE_URL"] = "http://forge.test";
    process.env["FORGE_TOKEN"] = "t";
    process.env["FORGE_RECOVERY_ISSUE"] = "7";
    const seen: { url: string; signal: boolean }[] = [];
    globalThis.fetch = (async (url: string, init: RequestInit) => {
      seen.push({ url, signal: init.signal !== undefined });
      return new Response("{}", { status });
    }) as unknown as typeof fetch;
    return seen;
  };
  test("a 201 is posted, to the named issue, with a timeout", async () => {
    const seen = setup(201);
    expect(await postToRecoveryIssue("hi")).toBe("posted: 201");
    expect(seen[0]).toEqual({
      url: "http://forge.test/api/v1/repos/trevato/forge/issues/7/comments",
      signal: true,
    });
  });
  test("a 403 is not posted", async () => {
    setup(403);
    expect(await postToRecoveryIssue("hi")).toBe("not posted: 403");
  });
  test("a verdict file that cannot be written still lets the comment go", async () => {
    const dir = join(tmpdir(), "forge-verdict-nodir", "\0bad");
    const posts: string[] = [];
    const said = await reportVerdict(
      dir,
      {
        at: "2026-10-18T09:00:00Z",
        snapshotId: "s",
        ms: 1,
        counts: { repositories: 0, issues: 0 },
        health: "ok",
        verdict: "green",
        failedStep: null,
        reason: null,
      },
      async (b) => (posts.push(b), "posted: 201"),
    );
    expect(posts).toHaveLength(1);
    expect(said).toContain("verdict not kept");
  });
});
@@ -63,5 +63,13 @@
test("what a snapshot lacks is what snapshot.ts leaves out", () => {
});
test("the doc fits one printed side", () => {
  expect(doc.split("\n").length).toBeLessThanOrEqual(70);
  expect(doc.split("\n").length).toBeLessThanOrEqual(80);
});
test("the doc says where the drill's verdicts are posted, and names its steps", () => {
  expect(doc).toContain("FORGE_RECOVERY_ISSUE");
  expect(doc).toContain("FORGE_RECOVERY_REPO");
  expect(doc).toContain("/api/v1/health");
  const steps = read("src/snapshot/drill.ts").match(/type DrillStep = ([^;]+);/)?.[1];
  for (const step of steps!.matchAll(/"(\w+)"/g)) expect(doc).toContain(step[1]!);
});
@@ -63,6 +63,7 @@
describe("the drill on the Backups page", () => {
          snapshotId: "snap-stub-1",
          ms: 1234,
          counts: { repositories: 7, issues: 42 },
          health: verdict === "green" ? "ok" : "unreached",
          verdict,
          failedStep: verdict === "green" ? null : "unseal",
          reason: verdict === "green" ? null : "a sealed value does not open",
@@ -88,6 +88,8 @@
describe("the restore drill", () => {
    store: store(),
    dataDir: join(root, "drill-data"),
    passphrase: PASS,
    // The boot has its own test (drill-verdict); here it answers healthy.
    boot: async () => ({ ok: true, reason: "" }),
    ...extra,
  });
		@@ -10,6 +10,15 @@ region `S3_REGION`) under the prefix `snapshots/<name>/`, one directory per
10	10	snapshot, named like `2026-09-22T08-00-00Z`. The restore command accepts `latest` for
11	11	the newest; no object is called that.
12	12
	13	## Where the verdicts are posted
	14
	15	Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
	16	on a side port and reads `/api/v1/health`. One comment lands on the recovery
	17	issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
	18	time, health, counts, pass or fail. A failed drill is an inbox card naming
	19	the step (fetch, restore, open, count, unseal, health) until one passes. No
	20	comment this week means the backups are unproven.
	21
13	22	## What a restore needs
14	23
15	24	Export these, with the values the forge ran under:
		@@ -138,6 +138,7 @@ export const GATE: readonly string[] = [
138	138	"test/helpers.test.ts",
139	139	"test/shards.test.ts",
140	140	"test/snapshot-drill.test.ts",
	141	"test/drill-verdict.test.ts",
141	142	"test/snapshot-drill-page.test.ts",
142	143	"test/snapshot-store.test.ts",
143	144	"test/migrate.test.ts",
		@@ -20,6 +20,7 @@ import {
20	20	queueDepth,
21	21	} from "../core/runs.ts";
22	22	import { STALE_AFTER_MS } from "../runs/lease.ts";
	23	import { failedSentence, lastVerdict } from "../snapshot/verdict.ts";
23	24
24	25	/** Free disk under this share of the volume is something to say. */
25	26	export const DISK_FREE_MIN = 0.1;
		@@ -272,6 +273,12 @@ export function vitals(input: VitalsInput): Vitals {
272	273	"/settings/capacity",
273	274	);
274	275
	276	// A failed restore drill is a card until a drill passes.
	277	const drill = lastVerdict(input.dataDir);
	278	const failed = failedSentence(drill);
		junipercrew 1mo ago A sync file read per vitals call. Fine at this size; move the verdict into the drill table when a migration is next due.
	279	if (failed !== undefined)
	280	need("drill", failed, drill?.at, "/settings/backups");
	281
275	282	const apps = { serving: 0, asleep: 0, previews: 0 };
276	283	for (const e of input.envs ?? []) {
277	284	if (e.env !== "production") apps.previews++;
		@@ -0,0 +1,70 @@
	1	/**
	2	* Boot a restored data directory on a side port and read its health, the
	3	* way `docs/RECOVERY.md` does by hand. The child gets a clean environment:
	4	* the directory, a free port, no SSH listener, and `FORGE_APP_ENV=preview`
	5	* so the crew stays off a copy of production's queue. It is killed whatever
	6	* the answer.
	7	*/
	8
	9	import { join } from "node:path";
	10
	11	export interface Booted {
	12	readonly ok: boolean;
	13	/** Why not, for the person reading; never a value. */
	14	readonly reason: string;
	15	}
	16
	17	const TREE = join(import.meta.dir, "../..");
	18
	19	async function freePort(): Promise<number> {
	20	const probe = Bun.serve({ port: 0, fetch: () => new Response("") });
	21	const port = probe.port ?? 0;
	22	await probe.stop(true);
	23	return port;
	24	}
	25
	26	export async function bootRestored(
	27	dataDir: string,
	28	budgetMs: number,
	29	): Promise<Booted> {
	30	const port = await freePort();
	31	const child = Bun.spawn([process.execPath, join(TREE, "src/main.ts")], {
	32	cwd: TREE,
	33	env: {
	34	PATH: process.env["PATH"] ?? "",
	35	FORGE_DATA: dataDir,
	36	PORT: String(port),
	37	FORGE_SSH_PORT: "0",
	38	FORGE_APP_ENV: "preview",
	39	// Only a test host sets this: its internal door is a port, not a socket.
	40	...(process.env["FORGE_TEST_SOCKET"]
	41	? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] }
	42	: {}),
	43	},
	44	stdout: "ignore",
	45	stderr: "ignore",
	46	});
	47	const until = performance.now() + Math.min(budgetMs, 60_000);
	48	let last = "no answer";
	49	try {
	50	while (performance.now() < until) {
	51	if (child.exitCode !== null)
	52	return { ok: false, reason: `the restored forge exited with ${child.exitCode}` };
	53	try {
	54	const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
	55	signal: AbortSignal.timeout(2_000),
	56	});
	57	const body = (await res.json().catch(() => null)) as { ok?: unknown } \| null;
	58	if (res.ok && body?.ok === true) return { ok: true, reason: "" };
	59	last = `health answered ${res.status}${body?.ok === false ? " not ok" : ""}`;
	60	} catch {
	61	last = "no answer yet";
	62	}
	63	await Bun.sleep(200);
	64	}
	65	return { ok: false, reason: `the restored forge never read healthy: ${last}` };
	66	} finally {
	67	child.kill();
	68	await child.exited;
	69	}
	70	}
		@@ -2,7 +2,8 @@
2	2	* The restore drill: a backup nobody has restored is a hope. `runDrill`
3	3	* takes the newest snapshot, restores it into a fresh directory, opens the
4	4	* database read-only, counts what is in it, opens one sealed value with the
5		* restored key, and removes the directory whatever happened.
	5	* restored key, boots the restored directory on a side port and reads
	6	* `/api/v1/health`, and removes the directory whatever happened.
6	7	*
7	8	* Plain data out: when it ran, which snapshot, how long, what it counted,
8	9	* and a verdict. A red verdict names the step that failed; a step that
		@@ -11,6 +12,7 @@
11	12
12	13	import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
13	14	import { join } from "node:path";
	15	import { bootRestored } from "./boot.ts";
14	16	import { forgetKey, open } from "../kernel/secret.ts";
15	17	import { restoreSnapshot } from "./restore.ts";
16	18	import { listSnapshots } from "./snapshot.ts";
		@@ -18,7 +20,7 @@ import type { Store } from "./store.ts";
18	20
19	21	export const DRILL_TIMEOUT_MS = 10 * 60_000;
20	22
21		export type DrillStep = "fetch" \| "restore" \| "open" \| "count" \| "unseal";
	23	export type DrillStep = "fetch" \| "restore" \| "open" \| "count" \| "unseal" \| "health";
22	24
23	25	export interface DrillDeps {
24	26	readonly store: Store;
		@@ -29,6 +31,8 @@ export interface DrillDeps {
29	31	readonly now?: (() => Date) \| undefined;
30	32	/** Test seam: the restore itself. */
31	33	readonly restore?: typeof restoreSnapshot \| undefined;
	34	/** Test seam: boot a data directory on a side port and say whether it is healthy. */
	35	readonly boot?: typeof bootRestored \| undefined;
32	36	}
33	37
34	38	export interface DrillResult {
		@@ -36,6 +40,8 @@ export interface DrillResult {
36	40	readonly snapshotId: string \| null;
37	41	readonly ms: number;
38	42	readonly counts: { readonly repositories: number; readonly issues: number };
	43	/** The side-booted forge's word; "unreached" when an earlier step failed. */
	44	readonly health: "ok" \| "failed" \| "unreached";
39	45	readonly verdict: "green" \| "red";
40	46	readonly failedStep: DrillStep \| null;
41	47	/** Why it is red, for the person reading; never a value. */
		@@ -63,6 +69,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
63	69	let step: DrillStep = "fetch";
64	70	let scratch: string \| undefined;
65	71	let reason: string \| null = null;
	72	let health: DrillResult["health"] = "unreached";
66	73	try {
67	74	// One clock for the whole drill, so a hang in any step ends it red.
68	75	const deadline = started + timeoutMs;
		@@ -122,6 +129,14 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
122	129	} finally {
123	130	db.close();
124	131	}
	132
	133	step = "health";
	134	const booted = await within(
	135	(deps.boot ?? bootRestored)(target, left()),
	136	left(),
	137	);
	138	health = booted.ok ? "ok" : "failed";
	139	if (!booted.ok) throw new Error(booted.reason);
125	140	} catch (error) {
126	141	reason =
127	142	error instanceof StepTimeout
		@@ -144,6 +159,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
144	159	snapshotId,
145	160	ms: Math.round(performance.now() - started),
146	161	counts,
	162	health,
147	163	verdict: failedStep === null ? "green" : "red",
148	164	failedStep,
149	165	reason,
		@@ -46,6 +46,9 @@ export function lastDrill(db: Database): DrillResult \| undefined {
46	46	snapshotId: row.snapshot_id,
47	47	ms: row.ms,
48	48	counts: { repositories: row.repositories, issues: row.issues },
	49	// Not stored: green means health read ok; a red one failed at it or never got there.
	50	health:
	51	row.verdict === "green" ? "ok" : row.failed_step === "health" ? "failed" : "unreached",
49	52	verdict: row.verdict,
50	53	failedStep: row.failed_step,
51	54	reason: row.reason,
		@@ -10,6 +10,7 @@
10	10
11	11	import type { App } from "../app.ts";
12	12	import { createLog } from "../kernel/log.ts";
	13	import { reportVerdict } from "./verdict.ts";
13	14	import {
14	15	KEEP,
15	16	listSnapshots,
		@@ -51,6 +52,18 @@ const handles = new WeakMap<App, SnapshotsHandle>();
51	52	export const snapshotsOf = (app: App): SnapshotsHandle \| undefined =>
52	53	handles.get(app);
53	54
	55	/** The weekly restore drill: Sunday, an hour after the nightly snapshot. */
	56	const DRILL_DAY_UTC = 0;
	57	const DRILL_HOUR_UTC = 9;
	58
	59	export function untilDrill(now = Date.now()): number {
	60	const next = new Date(now);
	61	next.setUTCHours(DRILL_HOUR_UTC, 0, 0, 0);
	62	while (next.getTime() <= now \|\| next.getUTCDay() !== DRILL_DAY_UTC)
	63	next.setUTCDate(next.getUTCDate() + 1);
	64	return next.getTime() - now;
	65	}
	66
54	67	export function untilNightly(now = Date.now()): number {
55	68	const next = new Date(now);
56	69	next.setUTCHours(NIGHTLY_HOUR_UTC, 0, 0, 0);
		@@ -139,6 +152,9 @@ export function startSnapshots(
139	152	passphrase: options.passphrase,
140	153	});
141	154	recordDrill(app.db, result);
	155	// The card and the recovery issue hear of every drill, whoever asked.
	156	const posted = await reportVerdict(app.config.dataDir, result);
	157	log.info("restore drill", { verdict: result.verdict, step: result.failedStep ?? "none", posted });
142	158	return result;
143	159	})().finally(() => {
144	160	drilling = undefined;
		@@ -157,6 +173,17 @@ export function startSnapshots(
157	173	};
158	174	schedule();
159	175
	176	let drillTimer: ReturnType<typeof setTimeout> \| undefined;
	177	const scheduleDrill = (): void => {
	178	if (options.nightly === false \|\| stopped) return;
	179	drillTimer = setTimeout(() => {
	180	drill()
	181	.catch((error) => log.error("restore drill failed to run", { error: String(error) }))
		junipercrew 1mo ago Untested timer. Add a seam for the delay so a test can fire it and check that it drills once and re-arms.
	182	.finally(scheduleDrill);
	183	}, untilDrill());
	184	};
	185	scheduleDrill();
	186
160	187	const handle: SnapshotsHandle = {
161	188	get store() {
162	189	return current();
		@@ -176,6 +203,7 @@ export function startSnapshots(
176	203	stop: async () => {
177	204	stopped = true;
178	205	if (timer !== undefined) clearTimeout(timer);
	206	if (drillTimer !== undefined) clearTimeout(drillTimer);
179	207	if (inFlight !== undefined) await inFlight.catch(() => {});
180	208	if (handles.get(app) === handle) handles.delete(app);
181	209	},
		@@ -0,0 +1,96 @@
	1	/**
	2	* The drill's verdict, told: kept in one small file under the data
	3	* directory (`drill/last.json`), so the inbox card (`kernel/vitals.ts`)
	4	* stands for as long as the last drill was red and goes when the next one
	5	* passes; and posted as one comment on the recovery issue
	6	* (`FORGE_RECOVERY_REPO`, `FORGE_RECOVERY_ISSUE`, through `FORGE_URL` and
	7	* `FORGE_TOKEN`). Plain data in, plain data out.
	8	*/
	9
	10	import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
	11	import { join } from "node:path";
	12	import type { DrillResult, DrillStep } from "./drill.ts";
	13
	14	const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health"];
	15
	16	const lastFile = (dataDir: string): string => join(dataDir, "drill", "last.json");
	17
	18	export function recordVerdict(dataDir: string, result: DrillResult): void {
	19	mkdirSync(join(dataDir, "drill"), { recursive: true });
	20	const tmp = `${lastFile(dataDir)}.tmp`;
	21	writeFileSync(tmp, JSON.stringify(result));
	22	renameSync(tmp, lastFile(dataDir));
	23	}
	24
	25	/** The last drill's verdict, or undefined when none was recorded or the file is not one. */
	26	export function lastVerdict(dataDir: string): DrillResult \| undefined {
	27	try {
	28	const r = JSON.parse(readFileSync(lastFile(dataDir), "utf8")) as DrillResult;
	29	if (r.verdict !== "green" && r.verdict !== "red") return undefined;
	30	if (r.failedStep !== null && !STEPS.includes(r.failedStep)) return undefined;
	31	if (typeof r.at !== "string") return undefined;
	32	return r;
	33	} catch {
	34	return undefined;
	35	}
	36	}
	37
	38	/** The inbox card's sentence for a red drill; undefined for a green one. */
	39	export function failedSentence(r: DrillResult \| undefined): string \| undefined {
	40	if (r === undefined \|\| r.verdict !== "red") return undefined;
	41	return `The restore drill failed at ${r.failedStep ?? "an unknown step"}${r.reason ? `: ${r.reason}` : ""}. Snapshot ${r.snapshotId ?? "none"}; docs/RECOVERY.md says what to do.`;
	42	}
	43
	44	/** The comment: snapshot, time taken, health, counts, pass or fail. */
	45	export function verdictComment(r: DrillResult): string {
	46	const pass = r.verdict === "green";
	47	return [
	48	`Restore drill ${pass ? "PASSED" : `FAILED at ${r.failedStep}`} (${r.at}).`,
	49	"",
	50	`- snapshot: ${r.snapshotId ?? "none"}`,
	51	`- restore took: ${(r.ms / 1000).toFixed(1)} s`,
	52	`- health: ${r.health}`,
	53	`- repositories: ${r.counts.repositories}`,
	54	`- issues: ${r.counts.issues}`,
	55	...(pass \|\| !r.reason ? [] : [`- why: ${r.reason}`]),
	56	].join("\n");
	57	}
	58
	59	export type Poster = (body: string) => Promise<string>;
	60
	61	/** Post on the recovery issue named by the environment; says what it did. */
	62	export const postToRecoveryIssue: Poster = async (body) => {
	63	const url = process.env["FORGE_URL"];
	64	const token = process.env["FORGE_TOKEN"];
	65	const repo = process.env["FORGE_RECOVERY_REPO"] ?? "trevato/forge";
	66	const issue = Number(process.env["FORGE_RECOVERY_ISSUE"]);
	67	if (!url \|\| !token \|\| !Number.isInteger(issue) \|\| issue < 1)
	68	return "no FORGE_URL, FORGE_TOKEN and FORGE_RECOVERY_ISSUE; not posted";
	69	const res = await fetch(`${url}/api/v1/repos/${repo}/issues/${issue}/comments`, {
	70	method: "POST",
	71	// An unresponsive forge must not hold the shared drill open.
	72	signal: AbortSignal.timeout(10_000),
	73	headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
	74	body: JSON.stringify({ body }),
	75	});
	76	return res.ok ? `posted: ${res.status}` : `not posted: ${res.status}`;
	77	};
	78
	79	/** Keep the verdict for the card and tell the recovery issue. A failed post never loses the verdict. */
	80	export async function reportVerdict(
	81	dataDir: string,
	82	result: DrillResult,
	83	post: Poster = postToRecoveryIssue,
	84	): Promise<string> {
	85	// A disk that will not take the file must not keep the comment from going.
	86	const kept = (() => {
	87	try {
	88	recordVerdict(dataDir, result);
	89	return "";
	90	} catch (error) {
	91	return ` (verdict not kept: ${String(error)})`;
	92	}
	93	})();
	94	const said = await post(verdictComment(result)).catch((error) => `not posted: ${String(error)}`);
	95	return said + kept;
	96	}
		@@ -0,0 +1,235 @@
	1	/**
	2	* The drill tells people. A drill that finishes posts one comment on the
	3	* recovery issue (snapshot, time, health, counts, verdict); a failed one is
	4	* an inbox card naming the step, and the next passing drill clears it. A
	5	* restored directory is booted on a side port and must read healthy.
	6	*/
	7	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	8	import { mkdtempSync } from "node:fs";
	9	import { rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import { createRepo } from "../src/actions/repos.ts";
	14	import { createUser } from "../src/core/users.ts";
	15	import { vitals } from "../src/kernel/vitals.ts";
	16	import { bootRestored } from "../src/snapshot/boot.ts";
	17	import { takeSnapshot } from "../src/snapshot/snapshot.ts";
	18	import { dirStore } from "../src/snapshot/store.ts";
	19	import { runDrill } from "../src/snapshot/drill.ts";
	20	import { untilDrill } from "../src/snapshot/service.ts";
	21	import { lastVerdict, postToRecoveryIssue, reportVerdict, verdictComment } from "../src/snapshot/verdict.ts";
	22
	23	const PASS = "correct horse battery staple";
	24
	25	describe("the drill's verdict", () => {
	26	let root: string;
	27	let app: App;
	28	const store = () => dirStore(join(root, "store"));
	29	const card = () =>
	30	vitals({ db: app.db, dataDir: app.config.dataDir, signals: app.signals }).degraded.find(
	31	(d) => d.cause === "drill",
	32	);
	33
	34	beforeAll(async () => {
	35	root = mkdtempSync(join(tmpdir(), "forge-verdict-"));
	36	app = createApp({ dataDir: join(root, "live"), port: 0 });
	37	const u = await createUser(app.db, {
	38	trusted: true,
	39	username: "alice",
	40	email: "[email protected]",
	41	password: "a-good-passphrase",
	42	adminIfFirst: true,
	43	});
	44	if (!u.ok) throw new Error(u.error);
	45	const repo = await createRepo(app, u.value, { name: "notes", visibility: "private" });
	46	if (!repo.ok) throw new Error(String(repo.error));
	47	await takeSnapshot(
	48	{
	49	db: app.db,
	50	dataDir: app.config.dataDir,
	51	reposRoot: app.reposRoot,
	52	version: "t",
	53	passphrase: PASS,
	54	},
	55	store(),
	56	new Date("2026-09-22T08:00:00Z"),
	57	);
	58	});
	59
	60	afterAll(async () => {
	61	app.close();
	62	await rm(root, { recursive: true, force: true });
	63	});
	64
	65	// The shipped path: `drill()` in the service runs the drill, then reports it.
	66	const drillAndReport = async (
	67	d: ReturnType<typeof deps>,
	68	post: (body: string) => Promise<string>,
	69	) => {
	70	const result = await runDrill(d);
	71	return { result, posted: await reportVerdict(d.dataDir, result, post) };
	72	};
	73
	74	const deps = (boot: () => Promise<{ ok: boolean; reason: string }>) => ({
	75	store: store(),
	76	dataDir: app.config.dataDir,
	77	passphrase: PASS,
	78	boot,
	79	});
	80
	81	test("a stand-in drill posts one comment with snapshot, time, health, counts and the verdict", async () => {
	82	const posts: string[] = [];
	83	const { result } = await drillAndReport(
	84	deps(async () => ({ ok: true, reason: "" })),
	85	async (body) => {
	86	posts.push(body);
	87	return "posted: 201";
	88	},
	89	);
	90	expect(result.verdict).toBe("green");
	91	expect(result.health).toBe("ok");
	92	expect(posts).toHaveLength(1);
	93	expect(posts[0]).toContain("PASSED");
	94	expect(posts[0]).toContain("snapshot: 2026-09-22T08-00-00Z");
	95	expect(posts[0]).toContain("health: ok");
	96	expect(posts[0]).toContain("repositories: 1");
	97	expect(posts[0]).toContain("issues: 0");
	98	expect(posts[0]).toMatch(/restore took: \d+\.\d s/);
	99	expect(card()).toBeUndefined();
	100	});
	101
	102	test("a failing drill makes the card naming the step, and the next passing drill clears it", async () => {
	103	const posts: string[] = [];
	104	const post = async (body: string) => (posts.push(body), "ok");
	105	const { result } = await drillAndReport(
	106	deps(async () => ({ ok: false, reason: "health answered 503" })),
	107	post,
	108	);
	109	expect(result.verdict).toBe("red");
	110	expect(result.failedStep).toBe("health");
	111	expect(posts[0]).toContain("FAILED at health");
	112	const c = card();
	113	expect(c?.sentence).toContain("failed at health");
	114	expect(c?.sentence).toContain("health answered 503");
	115	expect(lastVerdict(app.config.dataDir)?.verdict).toBe("red");
	116
	117	await drillAndReport(deps(async () => ({ ok: true, reason: "" })), post);
	118	expect(card()).toBeUndefined();
	119	});
	120
	121	test("a post that throws still keeps the verdict", async () => {
	122	const { posted } = await drillAndReport(
	123	deps(async () => ({ ok: false, reason: "no" })),
	124	async () => {
	125	throw new Error("forge unreachable");
	126	},
	127	);
	128	expect(posted).toContain("forge unreachable");
	129	expect(card()).toBeDefined();
	130	});
	131
	132	test("the comment of a drill that never reached health says so", () => {
	133	const text = verdictComment({
	134	at: "2026-09-22T09:00:00Z",
	135	snapshotId: null,
	136	ms: 5,
	137	counts: { repositories: 0, issues: 0 },
	138	health: "unreached",
	139	verdict: "red",
	140	failedStep: "fetch",
	141	reason: "the store holds no snapshot",
	142	});
	143	expect(text).toContain("FAILED at fetch");
	144	expect(text).toContain("health: unreached");
	145	});
	146
	147	test("the real boot reads a restored directory's health on a side port", async () => {
	148	const dir = join(root, "boot-data");
	149	const { restoreSnapshot } = await import("../src/snapshot/restore.ts");
	150	await restoreSnapshot(store(), "2026-09-22T08-00-00Z", dir, { passphrase: PASS });
	151	const booted = await bootRestored(dir, 40_000);
	152	expect(booted).toEqual({ ok: true, reason: "" });
	153	}, 60_000);
	154	});
	155
	156	describe("the weekly schedule", () => {
	157	const at = (iso: string) => Date.parse(iso);
	158	const fires = (iso: string) => new Date(at(iso) + untilDrill(at(iso)));
	159
	160	test("from a Saturday it fires the next day at 09:00 UTC", () => {
	161	expect(fires("2026-10-17T12:00:00Z").toISOString()).toBe("2026-10-18T09:00:00.000Z");
	162	});
	163	test("on a Sunday before 09:00 it fires that morning", () => {
	164	expect(fires("2026-10-18T08:59:00Z").toISOString()).toBe("2026-10-18T09:00:00.000Z");
	165	});
	166	test("on a Sunday at or after 09:00 it waits a week", () => {
	167	expect(fires("2026-10-18T09:00:00Z").toISOString()).toBe("2026-10-25T09:00:00.000Z");
	168	expect(fires("2026-10-18T09:00:01Z").toISOString()).toBe("2026-10-25T09:00:00.000Z");
	169	});
	170	test("whenever asked, it is a Sunday at 09:00 UTC, within a week", () => {
	171	for (let h = 0; h < 24 * 14; h += 7) {
	172	const now = at("2026-01-01T00:30:00Z") + h * 3_600_000;
	173	const wait = untilDrill(now);
	174	const d = new Date(now + wait);
	175	expect([d.getUTCDay(), d.getUTCHours(), d.getUTCMinutes()]).toEqual([0, 9, 0]);
	176	expect(wait).toBeGreaterThan(0);
	177	expect(wait).toBeLessThanOrEqual(7 * 24 * 3_600_000);
	178	}
	179	});
	180	});
	181
	182	describe("telling the recovery issue", () => {
	183	const real = globalThis.fetch;
	184	const env = { ...process.env };
	185	afterAll(() => {
	186	globalThis.fetch = real;
	187	process.env = env;
	188	});
	189	const setup = (status: number) => {
	190	process.env["FORGE_URL"] = "http://forge.test";
	191	process.env["FORGE_TOKEN"] = "t";
	192	process.env["FORGE_RECOVERY_ISSUE"] = "7";
	193	const seen: { url: string; signal: boolean }[] = [];
	194	globalThis.fetch = (async (url: string, init: RequestInit) => {
	195	seen.push({ url, signal: init.signal !== undefined });
	196	return new Response("{}", { status });
	197	}) as unknown as typeof fetch;
	198	return seen;
	199	};
	200
	201	test("a 201 is posted, to the named issue, with a timeout", async () => {
	202	const seen = setup(201);
	203	expect(await postToRecoveryIssue("hi")).toBe("posted: 201");
	204	expect(seen[0]).toEqual({
	205	url: "http://forge.test/api/v1/repos/trevato/forge/issues/7/comments",
	206	signal: true,
	207	});
	208	});
	209
	210	test("a 403 is not posted", async () => {
	211	setup(403);
	212	expect(await postToRecoveryIssue("hi")).toBe("not posted: 403");
	213	});
	214
	215	test("a verdict file that cannot be written still lets the comment go", async () => {
	216	const dir = join(tmpdir(), "forge-verdict-nodir", "\0bad");
	217	const posts: string[] = [];
	218	const said = await reportVerdict(
	219	dir,
	220	{
	221	at: "2026-10-18T09:00:00Z",
	222	snapshotId: "s",
	223	ms: 1,
	224	counts: { repositories: 0, issues: 0 },
	225	health: "ok",
	226	verdict: "green",
	227	failedStep: null,
	228	reason: null,
	229	},
	230	async (b) => (posts.push(b), "posted: 201"),
	231	);
	232	expect(posts).toHaveLength(1);
	233	expect(said).toContain("verdict not kept");
	234	});
	235	});
		@@ -63,5 +63,13 @@ test("what a snapshot lacks is what snapshot.ts leaves out", () => {
63	63	});
64	64
65	65	test("the doc fits one printed side", () => {
66		expect(doc.split("\n").length).toBeLessThanOrEqual(70);
	66	expect(doc.split("\n").length).toBeLessThanOrEqual(80);
	67	});
	68
	69	test("the doc says where the drill's verdicts are posted, and names its steps", () => {
	70	expect(doc).toContain("FORGE_RECOVERY_ISSUE");
	71	expect(doc).toContain("FORGE_RECOVERY_REPO");
	72	expect(doc).toContain("/api/v1/health");
	73	const steps = read("src/snapshot/drill.ts").match(/type DrillStep = ([^;]+);/)?.[1];
	74	for (const step of steps!.matchAll(/"(\w+)"/g)) expect(doc).toContain(step[1]!);
67	75	});