5007475 · 75303f0 · ac523e0
@@ -6,4 +6,29 @@
- **Catch-up.** A boot with the newest snapshot older than the last 08:00 UTC slot takes one two minutes later (`snapshotDue`). A snapshot cut short by a release has no manifest, so it counts as lost.
- **In the terminal.** The Resilience dialog (palette: Resilience, admins) opens with the same verdict, each lane's newest, and every reason. On the web it is home's backup tile and `/settings/backups`; for an agent, `verdict` in the MCP `backups` tool.
- **Off the serving thread.** `VACUUM INTO` runs in a worker (`vacuum.ts`).
- **Not done, tracked:** platform/K8s lanes on home (#8805); an off-process probe and snapshots outside the serving process or resumable (#8806). Catch-up retakes a lost snapshot whole.
- **Not done, tracked:** platform/K8s lanes on home (#8805); snapshots outside the serving process or resumable (#8828; the off-process probe is below). Catch-up retakes a lost snapshot whole.
## The probe outside the forge (#8806, part 1)
The in-process bucket check dies with the forge, so a dead forge looked like
a quiet one. `scripts/backup-probe.ts` is a separate process (run it from cron
or another host) that reads the forge and the bucket and goes red when either
is silent.
```
S3_ENDPOINT=… S3_BUCKET=… S3_ACCESS_KEY=… S3_SECRET_KEY=… \
  bun scripts/backup-probe.ts --forge https://your.forge [--max-age-hours 36]
bun scripts/backup-probe.ts --forge http://127.0.0.1:3000 --dir /path/to/backups
```
Exit 0 when green; exit 1 and a `RED` line per failing read otherwise. Red is:
`/healthz` unanswered or not 200 within 10 s; no store configured; the store
unreadable or empty; or the newest manifest's `finishedAt` older than the
limit (default 36 h: one nightly plus grace). Use read-only credentials.
Logic lives in `src/snapshot/probe.ts`; tests in `test/backup-probe.test.ts`.
### Not done yet
Snapshots are still taken inside the serving process; a release mid-snapshot
is caught up by retaking it whole after boot. Resuming across a release (or
moving the snapshot into its own process) is a separate, larger change, tracked in #8828.
@@ -0,0 +1,24 @@
/**
 * `bun scripts/backup-probe.ts --forge <url> [--dir <path>] [--max-age-hours 36]`
 * — the dead-man probe, run from cron or another host, not by the forge.
 * Reads the bucket (`S3_*` from the environment, or `--dir`) and the forge's
 * `/healthz`; prints one line each and exits 1 when either is silent. See
 * `docs/features/backup-truth.md`.
 */
import { parseProbeArgs, probe } from "../src/snapshot/probe.ts";
import { dirStore, s3FromEnv, s3Store, withRetries, type Store } from "../src/snapshot/store.ts";
const parsed = parseProbeArgs(process.argv.slice(2), process.env);
if ("error" in parsed) {
  console.error(parsed.error);
  process.exit(2);
}
const s3 = s3FromEnv();
const store: Store | undefined =
  parsed.dir !== undefined ? dirStore(parsed.dir) : s3 ? withRetries(s3Store(s3), { attempts: 2 }) : undefined;
const result = await probe({ store, forgeUrl: parsed.forgeUrl, maxAgeMs: parsed.maxAgeMs });
for (const f of result.findings) console.log(`${f.ok ? "ok " : "RED"} ${f.what}: ${f.detail}`);
process.exit(result.ok ? 0 : 1);
@@ -0,0 +1,117 @@
/**
 * The dead-man probe: a read of the bucket and of the forge from outside
 * the serving process. The in-process bucket check goes quiet with the
 * forge, so a dead forge looked like a forge with nothing to report. This
 * goes red when either is silent: the forge does not answer `/healthz`, or
 * the newest snapshot in the store is older than it should be (one a night,
 * so 36 h leaves a retry hour and a late start). Plain data in, plain data
 * out; `scripts/backup-probe.ts` is the process that runs it.
 */
import { listSnapshots } from "./snapshot.ts";
import type { Store } from "./store.ts";
/** One nightly plus a day-and-a-half's grace: older than this is a missed night. */
export const MAX_AGE_MS = 36 * 3_600_000;
/** How long the forge may take to answer before it counts as silent. */
export const FORGE_TIMEOUT_MS = 10_000;
export interface Finding {
  readonly what: "forge" | "bucket";
  readonly ok: boolean;
  /** One plain line; never a secret. */
  readonly detail: string;
}
export interface ProbeResult {
  readonly ok: boolean;
  readonly findings: readonly Finding[];
}
export interface ProbeDeps {
  /** The bucket to read; undefined is itself red: nothing is watching a backup that has nowhere to land. */
  readonly store: Store | undefined;
  /** The forge's base URL; its `/healthz` is read. */
  readonly forgeUrl: string | undefined;
  readonly now?: number;
  readonly maxAgeMs?: number;
  readonly fetch?: (url: string, init: { signal: AbortSignal; redirect: "manual" }) => Promise<{ status: number }>;
  readonly timeoutMs?: number;
}
/** A finish time this far ahead of the clock is a bad clock or manifest, not a fresh snapshot. */
export const FUTURE_SLACK_MS = 5 * 60_000;
const hours = (ms: number): string => `${Math.round(ms / 360_000) / 10} h`;
export async function probeBucket(store: Store | undefined, now: number, maxAgeMs: number): Promise<Finding> {
  if (store === undefined) return { what: "bucket", ok: false, detail: "no store to read" };
  try {
    const newest = (await listSnapshots(store))[0];
    if (newest === undefined) return { what: "bucket", ok: false, detail: `${store.where} holds no snapshot` };
    const at = Date.parse(newest.finishedAt);
    if (!Number.isFinite(at)) return { what: "bucket", ok: false, detail: `${newest.name} has no readable finish time` };
    const age = now - at;
    if (age < -FUTURE_SLACK_MS) return { what: "bucket", ok: false, detail: `${newest.name} finished in the future (${hours(-age)} ahead of this clock)` };
    return age > maxAgeMs
      ? { what: "bucket", ok: false, detail: `newest snapshot ${newest.name} is ${hours(age)} old; the limit is ${hours(maxAgeMs)}` }
      : { what: "bucket", ok: true, detail: `newest snapshot ${newest.name} is ${hours(age)} old` };
  } catch (error) {
    return { what: "bucket", ok: false, detail: `unreadable: ${error instanceof Error ? error.message : String(error)}` };
  }
}
export async function probeForge(
  url: string | undefined,
  doFetch: NonNullable<ProbeDeps["fetch"]>,
  timeoutMs: number,
): Promise<Finding> {
  if (url === undefined) return { what: "forge", ok: false, detail: "no forge URL to read" };
  let target: string;
  try {
    const u = new URL("/healthz", url);
    if (u.protocol !== "https:" && u.protocol !== "http:") throw new Error("not http");
    target = u.href;
  } catch {
    return { what: "forge", ok: false, detail: "the forge URL is not an http(s) URL" };
  }
  try {
    const res = await doFetch(target, { signal: AbortSignal.timeout(timeoutMs), redirect: "manual" });
    // Redirects are not followed: another host answering 200 is not the forge being up.
    return res.status === 200
      ? { what: "forge", ok: true, detail: `${new URL(target).host} answers /healthz` }
      : { what: "forge", ok: false, detail: `${new URL(target).host} answered /healthz with ${res.status}` };
  } catch {
    return { what: "forge", ok: false, detail: `${new URL(target).host} did not answer /healthz` };
  }
}
/** Both reads, always both: a dead forge must not hide a stale bucket. */
export async function probe(deps: ProbeDeps): Promise<ProbeResult> {
  const [bucket, forge] = await Promise.all([
    probeBucket(deps.store, deps.now ?? Date.now(), deps.maxAgeMs ?? MAX_AGE_MS),
    probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, i)), deps.timeoutMs ?? FORGE_TIMEOUT_MS),
  ]);
  return { ok: bucket.ok && forge.ok, findings: [forge, bucket] };
}
export interface ProbeArgs {
  readonly forgeUrl: string | undefined;
  readonly dir: string | undefined;
  readonly maxAgeMs: number;
}
/** The CLI's flags: `--forge <url>`, `--dir <path>`, `--max-age-hours <n>`. A flag without its value, or an unknown one, is an error. */
export function parseProbeArgs(argv: readonly string[], env: Record<string, string | undefined> = {}): ProbeArgs | { readonly error: string } {
  const got: Record<string, string> = {};
  for (let i = 0; i < argv.length; i += 2) {
    const name = argv[i]!;
    if (name !== "--forge" && name !== "--dir" && name !== "--max-age-hours") return { error: `unknown argument ${name}` };
    const value = argv[i + 1];
    if (value === undefined || value.startsWith("--")) return { error: `${name} needs a value` };
    got[name] = value;
  }
  const hours = got["--max-age-hours"] === undefined ? MAX_AGE_MS / 3_600_000 : Number(got["--max-age-hours"]);
  if (!Number.isFinite(hours) || hours <= 0) return { error: "--max-age-hours must be a positive number" };
  return { forgeUrl: got["--forge"] ?? (env["FORGE_URL"] || undefined), dir: got["--dir"], maxAgeMs: hours * 3_600_000 };
}
@@ -0,0 +1,133 @@
/** The dead-man probe goes red when the forge or the bucket is silent, and green only when both speak. */
import { afterAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { parseProbeArgs, probe } from "../src/snapshot/probe.ts";
import { dirStore } from "../src/snapshot/store.ts";
const root = mkdtempSync(join(tmpdir(), "probe-"));
afterAll(() => rmSync(root, { recursive: true, force: true }));
const NOW = Date.parse("2026-11-02T12:00:00Z");
async function bucketWith(finishedAt: string | undefined) {
  const dir = mkdtempSync(join(root, "b-"));
  const store = dirStore(dir);
  if (finishedAt)
    await store.put("snapshots/s1/manifest.json", JSON.stringify({ name: "s1", finishedAt, files: [] }));
  return store;
}
const up = async () => ({ status: 200 });
const FORGE = "https://forge.example";
describe("probe", () => {
  test("green: a fresh snapshot and a forge that answers", async () => {
    const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
    expect(r.ok).toBe(true);
  });
  test("red: the forge is silent while the bucket is fresh", async () => {
    const r = await probe({
      store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW,
      fetch: async () => { throw new Error("refused"); },
    });
    expect(r.ok).toBe(false);
    expect(r.findings.find((f) => f.what === "forge")?.ok).toBe(false);
    expect(r.findings.find((f) => f.what === "bucket")?.ok).toBe(true);
  });
  test("red: the forge answers 503", async () => {
    const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: async () => ({ status: 503 }) });
    expect(r.ok).toBe(false);
  });
  test("red: the bucket's newest snapshot is older than 36 h, forge fine", async () => {
    const r = await probe({ store: await bucketWith("2026-10-31T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
    expect(r.ok).toBe(false);
    expect(r.findings.find((f) => f.what === "bucket")?.detail).toContain("52 h old");
  });
  test("red: an empty bucket, no store, no URL, a non-http URL", async () => {
    expect((await probe({ store: await bucketWith(undefined), forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(false);
    expect((await probe({ store: undefined, forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(false);
    expect((await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: undefined, now: NOW, fetch: up })).ok).toBe(false);
    expect((await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: "file:///etc", now: NOW, fetch: up })).ok).toBe(false);
  });
  test("red: a bucket that cannot be read", async () => {
    const broken = { ...dirStore(root), names: async () => { throw new Error("403"); } };
    const r = await probe({ store: broken, forgeUrl: FORGE, now: NOW, fetch: up });
    expect(r.findings[1]?.detail).toContain("unreadable");
  });
  test("red: a forge that never answers, within the timeout", async () => {
    const hang = (_u: string, i: { signal: AbortSignal }) =>
      new Promise<{ status: number }>((_, rej) => i.signal.addEventListener("abort", () => rej(new Error("aborted"))));
    const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: hang, timeoutMs: 20 });
    expect(r.ok).toBe(false);
    expect(r.findings[0]?.detail).toContain("did not answer");
  });
  test("red: a redirect is not followed and is not green", async () => {
    let init: { redirect?: string } | undefined;
    const r = await probe({
      store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW,
      fetch: async (_u, i) => { init = i; return { status: 301 }; },
    });
    expect(init?.redirect).toBe("manual");
    expect(r.ok).toBe(false);
    expect(r.findings[0]?.detail).toContain("301");
  });
  test("red: a snapshot dated in the future is not fresh; a few minutes of skew is", async () => {
    const far = await probe({ store: await bucketWith("2026-11-03T12:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
    expect(far.ok).toBe(false);
    expect(far.findings[1]?.detail).toContain("future");
    const skew = await probe({ store: await bucketWith("2026-11-02T12:02:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
    expect(skew.ok).toBe(true);
  });
  test("the newest snapshot decides, whatever order the store lists them or they were written", async () => {
    const store = dirStore(mkdtempSync(join(root, "two-")));
    const put = (name: string, finishedAt: string) =>
      store.put(`snapshots/${name}/manifest.json`, JSON.stringify({ name, finishedAt, files: [] }));
    await put("2026-11-02T08-00-00Z", "2026-11-02T08:00:00Z");
    await put("2026-10-01T08-00-00Z", "2026-10-01T08:00:00Z");
    expect((await probe({ store, forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(true);
    const stale = dirStore(mkdtempSync(join(root, "stale-")));
    await stale.put("snapshots/b/manifest.json", JSON.stringify({ name: "b", finishedAt: "2026-10-01T08:00:00Z", files: [] }));
    await stale.put("snapshots/a/manifest.json", JSON.stringify({ name: "a", finishedAt: "2026-09-01T08:00:00Z", files: [] }));
    expect((await probe({ store: stale, forgeUrl: FORGE, now: NOW, fetch: up })).findings[1]?.detail).toContain("snapshot b ");
  });
});
describe("parseProbeArgs", () => {
  test("reads flags and defaults to 36 h and FORGE_URL", () => {
    expect(parseProbeArgs(["--forge", "https://f", "--dir", "/d"])).toEqual({ forgeUrl: "https://f", dir: "/d", maxAgeMs: 36 * 3_600_000 });
    expect(parseProbeArgs([], { FORGE_URL: "https://e" })).toMatchObject({ forgeUrl: "https://e" });
    expect(parseProbeArgs(["--max-age-hours", "2"])).toMatchObject({ maxAgeMs: 7_200_000 });
  });
  test("a flag with no value, an unknown flag, and a bad age are errors", () => {
    expect(parseProbeArgs(["--forge"])).toEqual({ error: "--forge needs a value" });
    expect(parseProbeArgs(["--forge", "--dir", "x"])).toEqual({ error: "--forge needs a value" });
    expect(parseProbeArgs(["--nope", "1"])).toHaveProperty("error");
    for (const bad of ["0", "-1", "abc", ""]) expect(parseProbeArgs(["--max-age-hours", bad])).toHaveProperty("error");
  });
});
describe("scripts/backup-probe.ts", () => {
  const run = async (args: string[]) => {
    const p = Bun.spawn([process.execPath, join(import.meta.dir, "../scripts/backup-probe.ts"), ...args], { stdout: "pipe", stderr: "pipe", env: { PATH: process.env["PATH"] ?? "" } });
    return { code: await p.exited, out: await new Response(p.stdout).text() };
  };
  test("exit 2 on bad arguments; exit 1 when red", async () => {
    expect((await run(["--max-age-hours", "0"])).code).toBe(2);
    expect((await run(["--forge"])).code).toBe(2);
    const dir = mkdtempSync(join(root, "cli-"));
    const r = await run(["--forge", "http://127.0.0.1:1", "--dir", dir]);
    expect(r.code).toBe(1);
    expect(r.out).toContain("RED forge");
  });
  test("exit 0 when both are green", async () => {
    const dir = mkdtempSync(join(root, "cli-"));
    await dirStore(dir).put("snapshots/s/manifest.json", JSON.stringify({ name: "s", finishedAt: new Date().toISOString(), files: [] }));
    const server = Bun.serve({ port: 0, fetch: () => new Response("ok") });
    try {
      expect((await run(["--forge", `http://127.0.0.1:${server.port}`, "--dir", dir])).code).toBe(0);
    } finally {
      await server.stop(true);
    }
  });
});
		@@ -6,4 +6,29 @@
6	6	- Catch-up. A boot with the newest snapshot older than the last 08:00 UTC slot takes one two minutes later (`snapshotDue`). A snapshot cut short by a release has no manifest, so it counts as lost.
7	7	- In the terminal. The Resilience dialog (palette: Resilience, admins) opens with the same verdict, each lane's newest, and every reason. On the web it is home's backup tile and `/settings/backups`; for an agent, `verdict` in the MCP `backups` tool.
8	8	- Off the serving thread. `VACUUM INTO` runs in a worker (`vacuum.ts`).
9		- Not done, tracked: platform/K8s lanes on home (#8805); an off-process probe and snapshots outside the serving process or resumable (#8806). Catch-up retakes a lost snapshot whole.
	9	- Not done, tracked: platform/K8s lanes on home (#8805); snapshots outside the serving process or resumable (#8828; the off-process probe is below). Catch-up retakes a lost snapshot whole.
	10
	11	## The probe outside the forge (#8806, part 1)
	12
	13	The in-process bucket check dies with the forge, so a dead forge looked like
	14	a quiet one. `scripts/backup-probe.ts` is a separate process (run it from cron
	15	or another host) that reads the forge and the bucket and goes red when either
	16	is silent.
	17
	18	```
	19	S3_ENDPOINT=… S3_BUCKET=… S3_ACCESS_KEY=… S3_SECRET_KEY=… \
	20	bun scripts/backup-probe.ts --forge https://your.forge [--max-age-hours 36]
	21	bun scripts/backup-probe.ts --forge http://127.0.0.1:3000 --dir /path/to/backups
	22	```
	23
	24	Exit 0 when green; exit 1 and a `RED` line per failing read otherwise. Red is:
	25	`/healthz` unanswered or not 200 within 10 s; no store configured; the store
	26	unreadable or empty; or the newest manifest's `finishedAt` older than the
	27	limit (default 36 h: one nightly plus grace). Use read-only credentials.
	28	Logic lives in `src/snapshot/probe.ts`; tests in `test/backup-probe.test.ts`.
	29
	30	### Not done yet
	31
	32	Snapshots are still taken inside the serving process; a release mid-snapshot
	33	is caught up by retaking it whole after boot. Resuming across a release (or
	34	moving the snapshot into its own process) is a separate, larger change, tracked in #8828.
		@@ -0,0 +1,24 @@
	1	/**
	2	* `bun scripts/backup-probe.ts --forge <url> [--dir <path>] [--max-age-hours 36]`
	3	* — the dead-man probe, run from cron or another host, not by the forge.
	4	* Reads the bucket (`S3_*` from the environment, or `--dir`) and the forge's
	5	* `/healthz`; prints one line each and exits 1 when either is silent. See
	6	* `docs/features/backup-truth.md`.
	7	*/
	8
	9	import { parseProbeArgs, probe } from "../src/snapshot/probe.ts";
	10	import { dirStore, s3FromEnv, s3Store, withRetries, type Store } from "../src/snapshot/store.ts";
	11
	12	const parsed = parseProbeArgs(process.argv.slice(2), process.env);
	13	if ("error" in parsed) {
	14	console.error(parsed.error);
	15	process.exit(2);
	16	}
	17
	18	const s3 = s3FromEnv();
	19	const store: Store \| undefined =
	20	parsed.dir !== undefined ? dirStore(parsed.dir) : s3 ? withRetries(s3Store(s3), { attempts: 2 }) : undefined;
	21
	22	const result = await probe({ store, forgeUrl: parsed.forgeUrl, maxAgeMs: parsed.maxAgeMs });
	23	for (const f of result.findings) console.log(`${f.ok ? "ok " : "RED"} ${f.what}: ${f.detail}`);
	24	process.exit(result.ok ? 0 : 1);
		@@ -0,0 +1,117 @@
	1	/**
	2	* The dead-man probe: a read of the bucket and of the forge from outside
	3	* the serving process. The in-process bucket check goes quiet with the
	4	* forge, so a dead forge looked like a forge with nothing to report. This
	5	* goes red when either is silent: the forge does not answer `/healthz`, or
	6	* the newest snapshot in the store is older than it should be (one a night,
	7	* so 36 h leaves a retry hour and a late start). Plain data in, plain data
	8	* out; `scripts/backup-probe.ts` is the process that runs it.
	9	*/
	10
	11	import { listSnapshots } from "./snapshot.ts";
	12	import type { Store } from "./store.ts";
	13
	14	/** One nightly plus a day-and-a-half's grace: older than this is a missed night. */
	15	export const MAX_AGE_MS = 36 * 3_600_000;
	16	/** How long the forge may take to answer before it counts as silent. */
	17	export const FORGE_TIMEOUT_MS = 10_000;
	18
	19	export interface Finding {
	20	readonly what: "forge" \| "bucket";
	21	readonly ok: boolean;
	22	/** One plain line; never a secret. */
	23	readonly detail: string;
	24	}
	25
	26	export interface ProbeResult {
	27	readonly ok: boolean;
	28	readonly findings: readonly Finding[];
	29	}
	30
	31	export interface ProbeDeps {
	32	/** The bucket to read; undefined is itself red: nothing is watching a backup that has nowhere to land. */
	33	readonly store: Store \| undefined;
	34	/** The forge's base URL; its `/healthz` is read. */
	35	readonly forgeUrl: string \| undefined;
	36	readonly now?: number;
	37	readonly maxAgeMs?: number;
	38	readonly fetch?: (url: string, init: { signal: AbortSignal; redirect: "manual" }) => Promise<{ status: number }>;
	39	readonly timeoutMs?: number;
	40	}
	41
	42	/** A finish time this far ahead of the clock is a bad clock or manifest, not a fresh snapshot. */
	43	export const FUTURE_SLACK_MS = 5 * 60_000;
	44
	45	const hours = (ms: number): string => `${Math.round(ms / 360_000) / 10} h`;
	46
	47	export async function probeBucket(store: Store \| undefined, now: number, maxAgeMs: number): Promise<Finding> {
	48	if (store === undefined) return { what: "bucket", ok: false, detail: "no store to read" };
	49	try {
	50	const newest = (await listSnapshots(store))[0];
	51	if (newest === undefined) return { what: "bucket", ok: false, detail: `${store.where} holds no snapshot` };
	52	const at = Date.parse(newest.finishedAt);
	53	if (!Number.isFinite(at)) return { what: "bucket", ok: false, detail: `${newest.name} has no readable finish time` };
	54	const age = now - at;
	55	if (age < -FUTURE_SLACK_MS) return { what: "bucket", ok: false, detail: `${newest.name} finished in the future (${hours(-age)} ahead of this clock)` };
	56	return age > maxAgeMs
	57	? { what: "bucket", ok: false, detail: `newest snapshot ${newest.name} is ${hours(age)} old; the limit is ${hours(maxAgeMs)}` }
	58	: { what: "bucket", ok: true, detail: `newest snapshot ${newest.name} is ${hours(age)} old` };
	59	} catch (error) {
	60	return { what: "bucket", ok: false, detail: `unreadable: ${error instanceof Error ? error.message : String(error)}` };
	61	}
	62	}
	63
	64	export async function probeForge(
	65	url: string \| undefined,
	66	doFetch: NonNullable<ProbeDeps["fetch"]>,
	67	timeoutMs: number,
	68	): Promise<Finding> {
	69	if (url === undefined) return { what: "forge", ok: false, detail: "no forge URL to read" };
	70	let target: string;
	71	try {
	72	const u = new URL("/healthz", url);
	73	if (u.protocol !== "https:" && u.protocol !== "http:") throw new Error("not http");
	74	target = u.href;
	75	} catch {
	76	return { what: "forge", ok: false, detail: "the forge URL is not an http(s) URL" };
	77	}
	78	try {
	79	const res = await doFetch(target, { signal: AbortSignal.timeout(timeoutMs), redirect: "manual" });
	80	// Redirects are not followed: another host answering 200 is not the forge being up.
	81	return res.status === 200
	82	? { what: "forge", ok: true, detail: `${new URL(target).host} answers /healthz` }
	83	: { what: "forge", ok: false, detail: `${new URL(target).host} answered /healthz with ${res.status}` };
	84	} catch {
	85	return { what: "forge", ok: false, detail: `${new URL(target).host} did not answer /healthz` };
	86	}
	87	}
	88
	89	/** Both reads, always both: a dead forge must not hide a stale bucket. */
	90	export async function probe(deps: ProbeDeps): Promise<ProbeResult> {
	91	const [bucket, forge] = await Promise.all([
	92	probeBucket(deps.store, deps.now ?? Date.now(), deps.maxAgeMs ?? MAX_AGE_MS),
	93	probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, i)), deps.timeoutMs ?? FORGE_TIMEOUT_MS),
	94	]);
	95	return { ok: bucket.ok && forge.ok, findings: [forge, bucket] };
	96	}
	97
	98	export interface ProbeArgs {
	99	readonly forgeUrl: string \| undefined;
	100	readonly dir: string \| undefined;
	101	readonly maxAgeMs: number;
	102	}
	103
	104	/** The CLI's flags: `--forge <url>`, `--dir <path>`, `--max-age-hours <n>`. A flag without its value, or an unknown one, is an error. */
	105	export function parseProbeArgs(argv: readonly string[], env: Record<string, string \| undefined> = {}): ProbeArgs \| { readonly error: string } {
	106	const got: Record<string, string> = {};
	107	for (let i = 0; i < argv.length; i += 2) {
	108	const name = argv[i]!;
	109	if (name !== "--forge" && name !== "--dir" && name !== "--max-age-hours") return { error: `unknown argument ${name}` };
	110	const value = argv[i + 1];
	111	if (value === undefined \|\| value.startsWith("--")) return { error: `${name} needs a value` };
	112	got[name] = value;
	113	}
	114	const hours = got["--max-age-hours"] === undefined ? MAX_AGE_MS / 3_600_000 : Number(got["--max-age-hours"]);
	115	if (!Number.isFinite(hours) \|\| hours <= 0) return { error: "--max-age-hours must be a positive number" };
	116	return { forgeUrl: got["--forge"] ?? (env["FORGE_URL"] \|\| undefined), dir: got["--dir"], maxAgeMs: hours * 3_600_000 };
	117	}
		@@ -0,0 +1,133 @@
	1	/** The dead-man probe goes red when the forge or the bucket is silent, and green only when both speak. */
	2
	3	import { afterAll, describe, expect, test } from "bun:test";
	4	import { mkdtempSync, rmSync } from "node:fs";
	5	import { tmpdir } from "node:os";
	6	import { join } from "node:path";
	7	import { parseProbeArgs, probe } from "../src/snapshot/probe.ts";
	8	import { dirStore } from "../src/snapshot/store.ts";
	9
	10	const root = mkdtempSync(join(tmpdir(), "probe-"));
	11	afterAll(() => rmSync(root, { recursive: true, force: true }));
	12
	13	const NOW = Date.parse("2026-11-02T12:00:00Z");
	14	async function bucketWith(finishedAt: string \| undefined) {
	15	const dir = mkdtempSync(join(root, "b-"));
	16	const store = dirStore(dir);
	17	if (finishedAt)
	18	await store.put("snapshots/s1/manifest.json", JSON.stringify({ name: "s1", finishedAt, files: [] }));
	19	return store;
	20	}
	21	const up = async () => ({ status: 200 });
	22	const FORGE = "https://forge.example";
	23
	24	describe("probe", () => {
	25	test("green: a fresh snapshot and a forge that answers", async () => {
	26	const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
	27	expect(r.ok).toBe(true);
	28	});
	29	test("red: the forge is silent while the bucket is fresh", async () => {
	30	const r = await probe({
	31	store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW,
	32	fetch: async () => { throw new Error("refused"); },
	33	});
	34	expect(r.ok).toBe(false);
	35	expect(r.findings.find((f) => f.what === "forge")?.ok).toBe(false);
	36	expect(r.findings.find((f) => f.what === "bucket")?.ok).toBe(true);
	37	});
	38	test("red: the forge answers 503", async () => {
	39	const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: async () => ({ status: 503 }) });
	40	expect(r.ok).toBe(false);
	41	});
	42	test("red: the bucket's newest snapshot is older than 36 h, forge fine", async () => {
	43	const r = await probe({ store: await bucketWith("2026-10-31T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
	44	expect(r.ok).toBe(false);
	45	expect(r.findings.find((f) => f.what === "bucket")?.detail).toContain("52 h old");
	46	});
	47	test("red: an empty bucket, no store, no URL, a non-http URL", async () => {
	48	expect((await probe({ store: await bucketWith(undefined), forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(false);
	49	expect((await probe({ store: undefined, forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(false);
	50	expect((await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: undefined, now: NOW, fetch: up })).ok).toBe(false);
	51	expect((await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: "file:///etc", now: NOW, fetch: up })).ok).toBe(false);
	52	});
	53	test("red: a bucket that cannot be read", async () => {
	54	const broken = { ...dirStore(root), names: async () => { throw new Error("403"); } };
	55	const r = await probe({ store: broken, forgeUrl: FORGE, now: NOW, fetch: up });
	56	expect(r.findings[1]?.detail).toContain("unreadable");
	57	});
	58	test("red: a forge that never answers, within the timeout", async () => {
	59	const hang = (_u: string, i: { signal: AbortSignal }) =>
	60	new Promise<{ status: number }>((_, rej) => i.signal.addEventListener("abort", () => rej(new Error("aborted"))));
	61	const r = await probe({ store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: hang, timeoutMs: 20 });
	62	expect(r.ok).toBe(false);
	63	expect(r.findings[0]?.detail).toContain("did not answer");
	64	});
	65	test("red: a redirect is not followed and is not green", async () => {
	66	let init: { redirect?: string } \| undefined;
	67	const r = await probe({
	68	store: await bucketWith("2026-11-02T08:00:00Z"), forgeUrl: FORGE, now: NOW,
	69	fetch: async (_u, i) => { init = i; return { status: 301 }; },
	70	});
	71	expect(init?.redirect).toBe("manual");
	72	expect(r.ok).toBe(false);
	73	expect(r.findings[0]?.detail).toContain("301");
	74	});
	75	test("red: a snapshot dated in the future is not fresh; a few minutes of skew is", async () => {
	76	const far = await probe({ store: await bucketWith("2026-11-03T12:00:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
	77	expect(far.ok).toBe(false);
	78	expect(far.findings[1]?.detail).toContain("future");
	79	const skew = await probe({ store: await bucketWith("2026-11-02T12:02:00Z"), forgeUrl: FORGE, now: NOW, fetch: up });
	80	expect(skew.ok).toBe(true);
	81	});
	82	test("the newest snapshot decides, whatever order the store lists them or they were written", async () => {
	83	const store = dirStore(mkdtempSync(join(root, "two-")));
	84	const put = (name: string, finishedAt: string) =>
	85	store.put(`snapshots/${name}/manifest.json`, JSON.stringify({ name, finishedAt, files: [] }));
	86	await put("2026-11-02T08-00-00Z", "2026-11-02T08:00:00Z");
	87	await put("2026-10-01T08-00-00Z", "2026-10-01T08:00:00Z");
	88	expect((await probe({ store, forgeUrl: FORGE, now: NOW, fetch: up })).ok).toBe(true);
	89	const stale = dirStore(mkdtempSync(join(root, "stale-")));
	90	await stale.put("snapshots/b/manifest.json", JSON.stringify({ name: "b", finishedAt: "2026-10-01T08:00:00Z", files: [] }));
	91	await stale.put("snapshots/a/manifest.json", JSON.stringify({ name: "a", finishedAt: "2026-09-01T08:00:00Z", files: [] }));
	92	expect((await probe({ store: stale, forgeUrl: FORGE, now: NOW, fetch: up })).findings[1]?.detail).toContain("snapshot b ");
	93	});
	94	});
	95
	96	describe("parseProbeArgs", () => {
	97	test("reads flags and defaults to 36 h and FORGE_URL", () => {
	98	expect(parseProbeArgs(["--forge", "https://f", "--dir", "/d"])).toEqual({ forgeUrl: "https://f", dir: "/d", maxAgeMs: 36 * 3_600_000 });
	99	expect(parseProbeArgs([], { FORGE_URL: "https://e" })).toMatchObject({ forgeUrl: "https://e" });
	100	expect(parseProbeArgs(["--max-age-hours", "2"])).toMatchObject({ maxAgeMs: 7_200_000 });
	101	});
	102	test("a flag with no value, an unknown flag, and a bad age are errors", () => {
	103	expect(parseProbeArgs(["--forge"])).toEqual({ error: "--forge needs a value" });
	104	expect(parseProbeArgs(["--forge", "--dir", "x"])).toEqual({ error: "--forge needs a value" });
	105	expect(parseProbeArgs(["--nope", "1"])).toHaveProperty("error");
	106	for (const bad of ["0", "-1", "abc", ""]) expect(parseProbeArgs(["--max-age-hours", bad])).toHaveProperty("error");
	107	});
	108	});
	109
	110	describe("scripts/backup-probe.ts", () => {
	111	const run = async (args: string[]) => {
	112	const p = Bun.spawn([process.execPath, join(import.meta.dir, "../scripts/backup-probe.ts"), ...args], { stdout: "pipe", stderr: "pipe", env: { PATH: process.env["PATH"] ?? "" } });
	113	return { code: await p.exited, out: await new Response(p.stdout).text() };
	114	};
	115	test("exit 2 on bad arguments; exit 1 when red", async () => {
	116	expect((await run(["--max-age-hours", "0"])).code).toBe(2);
	117	expect((await run(["--forge"])).code).toBe(2);
	118	const dir = mkdtempSync(join(root, "cli-"));
	119	const r = await run(["--forge", "http://127.0.0.1:1", "--dir", dir]);
	120	expect(r.code).toBe(1);
	121	expect(r.out).toContain("RED forge");
	122	});
	123	test("exit 0 when both are green", async () => {
	124	const dir = mkdtempSync(join(root, "cli-"));
	125	await dirStore(dir).put("snapshots/s/manifest.json", JSON.stringify({ name: "s", finishedAt: new Date().toISOString(), files: [] }));
	126	const server = Bun.serve({ port: 0, fetch: () => new Response("ok") });
	127	try {
	128	expect((await run(["--forge", `http://127.0.0.1:${server.port}`, "--dir", dir])).code).toBe(0);
	129	} finally {
	130	await server.stop(true);
	131	}
	132	});
	133	});