66dfd57 · 462f487 · 09e80c3 · 0c7c63b · fa7587e · 087b6c6
@@ -103,6 +103,7 @@
down.
<!-- footing:start — written by footingTable() in src/system/footing.ts; test/footing.test.ts holds it equal -->
- **pg-grand**: DigitalOcean. The k3s server, a single control plane with no HA. The owner decided on 2026-11-12 to drop it.
- **vxrail**: A k3s agent joined to pg-grand. The forge runs here.
- **off-site**: Object storage on neither machine, at a provider of its own: the off-site backup lane's bucket.
| Needed to | Piece | Runs on | If that machine dies | Breaks | Removed by |
|---|---|---|---|---|---|
@@ -111,7 +112,7 @@
down.
| (a) answer a request | The vxrail's own tunnel (standby/cloudflared-forge-home, 2 replicas): it too sends *.open-platform.sh to Traefik; only forge-home-test.open-platform.sh and arm--forge-trevato-apps.open-platform.sh go straight to the forge | vxrail | It goes with the forge, which runs beside it. | — | — |
| (a) answer a request | Traefik (kube-system/traefik): every public request goes tunnel → Traefik → the forge's Service | pg-grand | The forge is off the internet; it keeps running inside. | 1 | #11686, #8946 |
| (a) answer a request | Child and app hosts, Traefik IngressRoutes forge-children (&lt;name&gt;--forge.open-platform.sh), forge-apps (*--forge-trevato-apps.open-platform.sh) and forge-old-host, and game.cheffryjames.com and www.cheffryjames.com in the forge's namespace | pg-grand | No child forge, app or custom domain answers. | 1 | #11686, #8946 |
| (a) answer a request | forge-ssh, a host socket proxy (:2222 → the forge's ssh service) | pg-grand | git over ssh and the terminal stop answering. | 7 | #11686 |
| (a) answer a request | forge-ssh, a host socket proxy (:2222 → the forge's ssh service) | pg-grand | git over ssh and the terminal stop answering. | 6 | #11686 |
| (a) answer a request | The k3s API server (the vxrail is an agent joined to it) | pg-grand | Running pods keep running, but nothing reschedules, Services stop updating, and no release rolls or Job runs. | 5 | #11686 |
| (b) sign a person in | Sign-in edge: a forwardAuth middleware on the forge's Ingress, answered by mcp/plat-auth | pg-grand | Every request through the edge answers 500, /healthz too. | 2 | #11686 |
| (b) sign a person in | forge-signup-gate, the pod /signup is routed to | pg-grand | No one can sign up. | 3 | #11686 |
@@ -120,11 +121,11 @@
down.
| (c) build | forgejo-runner-vxrail, a Forgejo Actions runner | vxrail | It goes with the forge, which runs beside it. | — | — |
| (d) release | Forgejo's registry (git.open-platform.sh), which the forge's image is pulled from | pg-grand | A pod that restarts on a node without the image cached cannot start; nothing new is released. | 4 | #11686, #8941, #8942 |
| (d) release | registry-node, a registry on the vxrail | vxrail | It goes with the forge, which runs beside it. | — | — |
| (d) release | forge-release-tick, a host timer every 5 min | pg-grand | Nothing is released; production keeps the release it has. | 7 | #11686, #8942 |
| (d) release | standby-reconcile, a host timer every 2 min that applies the forge's manifests from Forgejo; Flux's Kustomization for the forge is suspended | pg-grand | A changed manifest is not applied; the running pod is untouched. | 7 | #8948, #11686 |
| (d) release | Host timers forge-watchdog, forge-github-mirror and forge-prune-images | pg-grand | The watchdog, the GitHub mirror and image pruning stop. | 7 | #11686 |
| (e) back up | The backup namespace's CronJobs (FORGE_BACKUP_LANES) | pg-grand | No new backup is taken. | 6 | #11686 |
| (e) back up | MinIO, where the backup lanes store the forge's backups | pg-grand | No new backup is stored, and those it holds go with the machine. | 6 | #11686 |
| (d) release | forge-release-tick, a host timer every 5 min | pg-grand | Nothing is released; production keeps the release it has. | 6 | #11686, #8942 |
| (d) release | standby-reconcile, a host timer every 2 min that applies the forge's manifests from Forgejo; Flux's Kustomization for the forge is suspended | pg-grand | A changed manifest is not applied; the running pod is untouched. | 6 | #8948, #11686 |
| (d) release | Host timers forge-watchdog, forge-github-mirror and forge-prune-images | pg-grand | The watchdog, the GitHub mirror and image pruning stop. | 6 | #11686 |
| (e) back up | The forge's own backup lane (src/snapshot): the nightly snapshot and the hourly off-site pass, run in the forge's process, every object sealed under FORGE_SNAPSHOT_PASSPHRASE before it leaves; the nightly lands on the forge's own disk (FORGE_SNAPSHOT_DIR on its volume), not the platform's MinIO | vxrail | No new backup is taken; the sealed copies wait off both machines. | — | — |
| (e) back up | The off-site lane's bucket (FORGE_MIRROR_S3_*, or the mirror sealed on the Backups page), on neither machine | off-site | Neither machine holds it: losing either leaves every sealed copy in place. | — | — |
If pg-grand were switched off now, the forge breaks in this order (the Breaks column):
@@ -133,10 +134,9 @@
If pg-grand were switched off now, the forge breaks in this order (the Breaks co
3. the signup gate: no one signs up
4. image pulls and releases
5. the control plane: nothing reschedules, Services stop updating
6. local backups
7. the host timers
6. the host timers
Also on pg-grand, not in the forge's path: Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and MinIO.
Also on pg-grand, not in the forge's path: Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and the platform's backup CronJobs (the backup namespace, FORGE_BACKUP_LANES) with the MinIO they write to; the forge's own backup no longer needs them.
<!-- footing:end -->
The same list is read from the live cluster on `/system`, under "What the
@@ -35,10 +35,10 @@
A machine with `git`, `bun` and a clone of the forge (`bun install`).
## Route 1 — the restore command
    bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N]
    bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite]
`<data-dir>` must be empty or new. `--clones N` (1–32) sets how many
repositories clone at once (default: cores less one, 2–16).
`<data-dir>` must be empty or new. `--clones N` (1–32): repositories cloned at once
(default: cores less one, 2–16). `--offsite`: from the sealed lane, [features/offsite-lane.md](features/offsite-lane.md).
Then boot the forge on that directory, on a side port, off the public one:
@@ -56,4 +56,9 @@
refuses `/data`, and leaves the restored copy in place.
`test/restore-drill-script.test.ts` runs it against a tiny snapshot and a
stub bucket.
## From the off-site lane
The forge's own sealed backup lane (#11805), how to restore from it and how
it is drilled: [features/offsite-lane.md](features/offsite-lane.md).
With the platform unreachable (the B2 mirror only): [backup-truth.md](features/backup-truth.md), _Leaving the platform_.
@@ -0,0 +1,57 @@
# The off-site backup lane
The forge's own backup lane (#11805) is the mirror pass that runs in the
forge's process every hour and after every snapshot. It copies the newest
snapshots to a bucket on neither machine (`FORGE_MIRROR_S3_*`, or the mirror
sealed on the Backups page), and every object it writes is sealed first:
AES-256-GCM under a fresh IV, the key derived from `FORGE_SNAPSHOT_PASSPHRASE`
by scrypt (`src/snapshot/sealed_store.ts`). Paths inside a snapshot are hidden
(an HMAC of each path), so the bucket does not even name a repository.
- **The salt is the lane's own.** The first pass writes one unsealed header,
  `sealed/lane.json`: a random 16-byte salt and a check value (an HMAC under
  the derived key, never the key). A guess at the passphrase costs one scrypt
  for that lane alone, and a wrong passphrase is said as such before any
  object is read.
- **The key is in the passphrase alone**: not in this repository, not on the
  bucket. Keep it with the operator, apart from the bucket's keys.
- **Without the passphrase the lane writes nothing.** Every pass fails saying
  `FORGE_SNAPSHOT_PASSPHRASE is not set`, and `/system` shows the lane red;
  not one unsealed byte leaves the forge.
For the lane to stand without pg-grand, the nightly snapshot itself must land
on the forge's own disk (`FORGE_SNAPSHOT_DIR` on its volume) rather than the
platform's MinIO: the lane copies from there.
## Restore from it
On a fresh machine, follow [RESTORE.md](../RESTORE.md), exporting the lane's
bucket in place of line 3, keeping line 4 (the passphrase), and adding
`--offsite` to line 5:
    export FORGE_MIRROR_S3_ENDPOINT=… FORGE_MIRROR_S3_BUCKET=… FORGE_MIRROR_S3_ACCESS_KEY=… FORGE_MIRROR_S3_SECRET_KEY=…
    bun run src/snapshot/restore.ts latest <data-dir> --offsite
`bun scripts/restore-drill.ts <empty-data-dir> --offsite` drills it the same
way. Without the passphrase both refuse before writing anything; a restore
never starts a lane on an empty bucket.
## The drill
The weekly drill, the Backups page's button and the MCP `restore_drill`
restore the newest snapshot from the off-site lane when the forge has one
(the destination otherwise) into a scratch forge, boot it, and check that
`/api/v1/health` answers and `/` answers below 400. A drill waits for a mirror
pass in flight, at most the drill's floor budget (10 min), then reads the lane
as it stands. Each drill is kept with the store it restored from.
## Where to see it
- **Web**: `/system`, under _Off-site backup_: one sentence, and behind it
  where the lane writes, whether it is encrypted, its last success, and the
  last drill with where it restored from.
- **Terminal**: the palette's **Off-site backup**, the same rows.
- **MCP**: `system` with `{ kind: "offsite" }`, the same reading as
  `{ health, says, where, sealed, lastSuccess, newest, error, drill }`.
All three are a forge admin's alone.
@@ -6,7 +6,10 @@
 * writes only into the empty data directory it is given, and reads
 * production through its public API, nothing else of the platform.
 *
 *   bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh]
 *   bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh] [--offsite]
 *
 * `--offsite` reads the sealed off-site lane (`FORGE_MIRROR_S3_*` and the
 * passphrase, #11805) in place of the destination.
 *
 * It restores, boots the restored forge on a free side port, checks `/`
 * answers, counts the repositories and apps anyone can see there and on
@@ -17,6 +20,7 @@
import { resolve } from "node:path";
import { bootRestored } from "../src/snapshot/boot.ts";
import { resolveDestination } from "../src/snapshot/destination.ts";
import { offsiteFromEnv } from "../src/snapshot/mirror.ts";
import { publicCounts, type PublicCounts } from "../src/snapshot/public_counts.ts";
import { restoreSnapshot } from "../src/snapshot/restore.ts";
import { listSnapshots } from "../src/snapshot/snapshot.ts";
@@ -28,8 +32,10 @@
const PRODUCTION_DATA = "/data";
const BOOT_MS = 60_000;
/** The arguments as plain data, or the reason they are not usable. */
export function parseArgs(args: readonly string[]): { dataDir: string; production: string } | string {
export function parseArgs(args: readonly string[]): { dataDir: string; production: string; offsite?: true } | string {
  const rest = [...args];
  const off = rest.indexOf("--offsite");
  if (off >= 0) rest.splice(off, 1);
  let production = PRODUCTION;
  const at = rest.indexOf("--production");
  if (at >= 0) {
@@ -46,10 +52,10 @@
export function parseArgs(args: readonly string[]): { dataDir: string; productio
  if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback))
    return "--production takes an https URL";
  if (rest.length !== 1 || !rest[0] || rest[0].startsWith("-"))
    return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL]";
    return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL] [--offsite]";
  const dataDir = resolve(rest[0]);
  if (dataDir === PRODUCTION_DATA) return `${dataDir} is production's data directory; restore into the box's own`;
  return { dataDir, production: url.origin };
  return { dataDir, production: url.origin, ...(off >= 0 ? { offsite: true as const } : {}) };
}
/** The two sides' counts as lines a person reads, and which differ. */
@@ -69,7 +75,12 @@
async function main(): Promise<number> {
    console.error(args);
    return 2;
  }
  const store = resolveDestination()?.store;
  const lane = args.offsite ? offsiteFromEnv() : undefined;
  if (typeof lane === "string") {
    console.error(lane);
    return 2;
  }
  const store = lane ?? resolveDestination()?.store;
  if (store === undefined) {
    console.error("no store: export the S3_* lines of the restore card, or FORGE_SNAPSHOT_DIR");
    return 2;
@@ -4,7 +4,7 @@
import { good, bad, type Tool, mayBoards } from "../shared.ts";
export const tool: Tool = {
  name: "restore_drill",
  doc: "no input: restore the newest snapshot as a drill and answer `{ verdict, failedStep?, says }`, worded as the Backups page's button; no store, a failure or a red drill is an answer, not an error; a forge admin's, with an admin token",
  doc: "no input: restore the newest snapshot as a drill and answer `{ verdict, failedStep?, says, from?, offSite? }` (from: the store it restored, the off-site lane when there is one), worded as the Backups page's button; no store, a failure or a red drill is an answer, not an error; a forge admin's, with an admin token",
  description: "Drill-restore the newest snapshot.",
  inputSchema: { type: "object", properties: {} },
  writes: true,
@@ -18,10 +18,12 @@
export const tool: Tool = {
      return good({ verdict: "none", says: "This forge has no snapshots to drill." });
    try {
      const d = await svc.drill();
      // Where it restored from (the off-site lane when the forge has one, #11805), never a key.
      const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true };
      const answer =
        d.verdict === "green"
          ? good({ verdict: "green", says: "The drill restored green." })
          : good({ verdict: "red", failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}.` });
          ? good({ verdict: "green", says: "The drill restored green.", ...from })
          : good({ verdict: "red", failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}.`, ...from });
      return { ...answer, card: { at: d.at } };
    } catch (error) {
      return good({ verdict: "none", says: `The drill could not run: ${String(error)}` });
@@ -8,14 +8,15 @@
import { lagMinutes, loopWatch, perfRecord } from "../../../perf/live.ts";
import { LAG_BAR_MS, overBar } from "../../../perf/lag_minutes.ts";
import { slowestRoutes } from "../../../perf/watch.ts";
import { footing, riskLines } from "../../../system/footing.ts";
import { laneWords, readOffsiteLane } from "../../../snapshot/offsite_lane.ts";
import { good, bad, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "system",
  doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast, `{ kind: \"routes\" }`: the 10 slowest budgeted routes as {route, p95Ms, budgetMs, n, slowestPhase}, over budget first; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only. `{ kind: \"footing\" }`: what the forge stands on (#11685), as /system's \"What the forge stands on\": `forgeNode`, and per piece `{ id, need, what, machine, seen, risk, ifDies, issues }` read from the live cluster, with `risks`, one line per piece not on the forge's machine; a forge admin only",
  doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast, `{ kind: \"routes\" }`: the 10 slowest budgeted routes as {route, p95Ms, budgetMs, n, slowestPhase}, over budget first; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only. `{ kind: \"footing\" }`: what the forge stands on (#11685), as /system's \"What the forge stands on\": `forgeNode`, and per piece `{ id, need, what, machine, seen, risk, ifDies, issues }` read from the live cluster, with `risks`, one line per piece not on the forge's machine; a forge admin only. `{ kind: \"offsite\" }`: the off-site backup lane (#11805), as /system's \"Off-site backup\" and the terminal's: `{ health, says, where, sealed, lastSuccess, newest, error, drill: { at, verdict, from, offSite, reason } | null }`; a forge admin only",
  description:
    "The forge machine now: CPU, memory, disk, network. Admin or telemetry:read; `kind: csp`: 24 h of CSP violations (admin).",
  inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp", "routes", "footing"] } } },
  inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp", "routes", "footing", "offsite"] } } },
  writes: false,
  async run(ctx, user, args) {
    if (args["kind"] !== undefined) {
@@ -30,7 +31,12 @@
export const tool: Tool = {
        const f = await footing.read();
        return good({ ...f, risks: riskLines(f) });
      }
      if (args["kind"] !== "csp") return bad("invalid", "kind is csp, routes, footing, or absent for the machine");
      if (args["kind"] === "offsite") {
        if (!user.isAdmin) return bad("forbidden", "only an admin of this forge reads its off-site backup lane");
        const lane = readOffsiteLane(ctx.app.config.dataDir, ctx.app.db);
        return good({ ...lane, says: laneWords(lane) });
      }
      if (args["kind"] !== "csp") return bad("invalid", "kind is csp, routes, footing, offsite, or absent for the machine");
      if (!user.isAdmin) return bad("forbidden", "only an admin of this forge reads its CSP violations");
      const now = new Date();
      return good({ hours: 24, since: windowStart(24, now).toISOString(), directives: violationsByDirective(ctx.app.db, 24, now), scriptErrors: scriptErrorCounts(ctx.app.db, 24, now) });
@@ -527,8 +527,14 @@
const snapshots = startSnapshots(
    })?.store,
  {
    version: running().version,
    // The off-site lane (#11805): sealed under the passphrase before it leaves.
    mirror: () =>
      resolveMirror({ db: app.db, dataDir: app.config.dataDir, bus: app.bus })?.store,
      resolveMirror({
        db: app.db,
        dataDir: app.config.dataDir,
        bus: app.bus,
        passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined,
      })?.store,
    passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined,
  },
);
@@ -3,7 +3,7 @@
 * takes the newest snapshot, restores it into a fresh directory, opens the
 * database read-only, counts what is in it, opens one sealed value with the
 * restored key, boots the restored directory on a side port and reads
 * `/api/v1/health`, and removes the directory whatever happened.
 * `/api/v1/health` and `/`, and removes the directory whatever happened.
 *
 * Plain data out: when it ran, which snapshot, how long, what it counted,
 * and a verdict. A red verdict names the step that failed; a step that
@@ -19,6 +19,7 @@
import { bootRestored } from "./boot.ts";
import { forgetKey, open } from "../kernel/secret.ts";
import { countRows, mismatched, type DrillCounts } from "./counts.ts";
import { restoreSnapshot, type Progress } from "./restore.ts";
import { isOffBox } from "./off_box.ts";
import { listSnapshots } from "./snapshot.ts";
import type { Store } from "./store.ts";
@@ -155,6 +156,17 @@
export interface DrillResult {
  readonly budgetMs: number;
  /** Why it is red, for the person reading; never a value. */
  readonly reason: string | null;
  /** The store it restored from (`Store.where`, never a key); absent on drills kept before #11805. */
  readonly from?: string | undefined;
  /** Whether that store is off the forge's machine (`isOffBox`). */
  readonly offSite?: boolean | undefined;
}
/** The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or this throws. */
export async function answersRoot(base: string): Promise<void> {
  const res = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
  await res.body?.cancel();
  if (res.status >= 400) throw new Error(`the restored forge answered / with ${res.status}`);
}
class StepTimeout extends Error {}
@@ -276,7 +288,7 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
    step = "health";
    bootStart = performance.now();
    const booted = await within(
      (deps.boot ?? bootRestored)(target, left()),
      (deps.boot ?? bootRestored)(target, left(), undefined, answersRoot),
      left(),
      budget,
    );
@@ -323,6 +335,8 @@
export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
      phases,
      budgetMs,
      reason,
      from: deps.store.where,
      offSite: isOffBox(deps.store),
    };
  }
}
@@ -24,6 +24,17 @@
interface Row {
const PHASES = new Set(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
/** Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is not ours. */
export function parseFrom(raw: string | null): { from?: string; offSite?: boolean } {
  try {
    const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown } | null;
    if (typeof v?.from !== "string" || v.from.length > 300) return {};
    return { from: v.from, offSite: v.offSite === true };
  } catch {
    return {};
  }
}
/** The stored JSON as phases; anything that is not a list of known phases with whole ms reads as none. */
export function parsePhases(raw: string | null): { phases: PhaseTime[]; budgetMs: number } {
  const none = { phases: [], budgetMs: 0 };
@@ -61,7 +72,12 @@
export function recordDrill(db: Database, result: DrillResult): void {
    result.verdict,
    result.failedStep,
    result.reason,
    JSON.stringify({ phases: result.phases, budgetMs: result.budgetMs }),
    // Where it restored from rides the same column: no migration for two plain fields (#11805).
    JSON.stringify({
      phases: result.phases,
      budgetMs: result.budgetMs,
      ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }),
    }),
  );
}
@@ -91,5 +107,6 @@
export function lastDrill(db: Database): DrillResult | undefined {
    reason: row.reason,
    phases: kept.phases,
    budgetMs: kept.budgetMs,
    ...parseFrom(row.phases),
  };
}
@@ -1,7 +1,9 @@
/**
 * The off-box mirror: a second S3-compatible bucket (Backblaze B2, say) that
 * every snapshot is copied to, so losing the machine does not lose the
 * snapshots with it. Its keys are sealed in the forge like the Backups
 * snapshots with it. It is the forge's off-site backup lane (#11805): the
 * pass runs in the forge's own process, and with the operator's passphrase
 * every object it writes is sealed (`sealed_store.ts`). Its keys are sealed in the forge like the Backups
 * destination (`SNAPSHOT_MIRROR_S3_*`), or given as `FORGE_MIRROR_S3_*`.
 *
 * A pass copies each of the newest snapshots the mirror lacks, the manifest
@@ -18,6 +20,7 @@
import { join } from "node:path";
import type { EventBus } from "../core/events.ts";
import { deleteSecret, resolveSecrets } from "../core/secret_store.ts";
import { KEEP, listSnapshots, pruneSnapshots, SNAPSHOT_PREFIX } from "./snapshot.ts";
import { refusingStore, sealedStore } from "./sealed_store.ts";
import { s3Store, type S3Target, type Store } from "./store.ts";
/** The sealed names, one per field of the form. */
@@ -42,11 +45,15 @@
export interface MirrorSources {
  readonly dataDir?: string | undefined;
  readonly bus?: EventBus | undefined;
  readonly env?: Record<string, string | undefined>;
  /** The operator's passphrase: given, every object the mirror holds is sealed under it (#11805). */
  readonly passphrase?: string | undefined;
}
export interface Mirror {
  readonly source: "sealed" | "env";
  readonly store: Store;
  /** Whether what lands there is encrypted; without a passphrase nothing lands, and `/system` says why. */
  readonly encrypted: boolean;
}
/** The mirror's target: sealed in this forge, else `FORGE_MIRROR_S3_*`; undefined when neither is whole. */
@@ -75,9 +82,21 @@
export function mirrorTarget(s: MirrorSources = {}): { source: Mirror["source"];
  };
}
export function resolveMirror(s: MirrorSources = {}): Mirror | undefined {
/** Said by every call of a lane with no passphrase: the board shows it, and nothing is written. */
export const NO_PASSPHRASE = "FORGE_SNAPSHOT_PASSPHRASE is not set, so the off-site lane writes nothing";
/**
 * The lane as the forge writes it: sealed under the passphrase. Without one
 * it fails closed (#11805): a store that refuses every call, so not one
 * unsealed byte leaves and each pass fails saying why.
 */
export function resolveMirror(s: MirrorSources = {}, options: { create?: boolean } = {}): Mirror | undefined {
  const t = mirrorTarget(s);
  return t === undefined ? undefined : { source: t.source, store: s3Store(t.target) };
  if (t === undefined) return undefined;
  const plain = s3Store(t.target);
  return s.passphrase
    ? { source: t.source, store: sealedStore(plain, s.passphrase, options), encrypted: true }
    : { source: t.source, store: refusingStore(plain.where, NO_PASSPHRASE), encrypted: false };
}
/** Which of the sealed mirror names are set, without opening any. */
@@ -110,6 +129,8 @@
export interface MirrorState {
  readonly failingSince?: string;
  readonly failed?: string;
  readonly error?: string;
  /** Whether the mirror's objects are encrypted (`Store.sealed`); absent in states kept before #11805. */
  readonly sealed?: boolean;
  /** Set (and nothing recorded) when the primary, not the mirror, could not be read. */
  readonly sourceError?: string;
}
@@ -158,7 +179,9 @@
async function copyOne(src: Store, dst: Store, name: string): Promise<void> {
  const prefix = `${SNAPSHOT_PREFIX}${name}/`;
  const manifest = `${prefix}manifest.json`;
  const have = new Set(await dst.list(prefix));
  const todo = (await src.list(prefix)).filter((k) => k !== manifest && !have.has(k));
  const listed = (await src.list(prefix)).filter((k) => k !== manifest);
  const stored = await Promise.all(listed.map((k) => dst.storedKey?.(k) ?? k));
  const todo = listed.filter((_, i) => !have.has(stored[i]!));
  for (let i = 0; i < todo.length; i += COPY_AT_ONCE)
    await Promise.all(
      todo.slice(i, i + COPY_AT_ONCE).map(async (key) => {
@@ -207,6 +230,7 @@
export async function mirrorPass(
      where: dst.where,
      attemptAt: at,
      reachable: true,
      sealed: dst.sealed === true,
      syncedAt: at,
      ...(newest === undefined ? {} : { newest: newest.name, newestAt: newest.finishedAt }),
    };
@@ -221,6 +245,7 @@
export async function mirrorPass(
      where: dst.where,
      attemptAt: at,
      reachable,
      sealed: dst.sealed === true,
      ...(prev?.syncedAt === undefined ? {} : { syncedAt: prev.syncedAt }),
      ...(prev?.newest === undefined ? {} : { newest: prev.newest }),
      ...(prev?.newestAt === undefined ? {} : { newestAt: prev.newestAt }),
@@ -254,7 +279,9 @@
export function mirrorNeed(s: MirrorState | undefined, now: number): MirrorNeed
  if (s.failingSince !== undefined)
    return {
      key: `fail:${s.failingSince}`,
      sentence: s.reachable
      sentence: s.error === NO_PASSPHRASE
        ? `The off-box mirror ${s.where} writes nothing: set FORGE_SNAPSHOT_PASSPHRASE.`
        : s.reachable
        ? `Snapshot ${s.failed ?? "(unknown)"} did not land on the off-box mirror ${s.where}: ${s.error ?? "unknown reason"}.`
        : `The off-box mirror ${s.where} is unreachable: ${s.error ?? "unknown reason"}.`,
      since: s.failingSince,
@@ -267,3 +294,18 @@
export function mirrorNeed(s: MirrorState | undefined, now: number): MirrorNeed
    since: s.syncedAt ? new Date(at + MIRROR_STALE_MS).toISOString() : s.attemptAt,
  };
}
/**
 * The off-site lane as a restore on a fresh machine reads it (#11805): from
 * the environment alone (`FORGE_MIRROR_S3_*`), and sealed, so the
 * passphrase is required. A sentence when either is missing.
 */
export function offsiteFromEnv(env: Record<string, string | undefined> = process.env): Store | string {
  const passphrase = env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined;
  // A restore reads: an empty bucket is said, never started as a new lane.
  const m = resolveMirror({ env, passphrase }, { create: false });
  if (m === undefined)
    return "no off-site lane: export FORGE_MIRROR_S3_ENDPOINT, FORGE_MIRROR_S3_BUCKET, FORGE_MIRROR_S3_ACCESS_KEY and FORGE_MIRROR_S3_SECRET_KEY";
  if (passphrase === undefined) return "the off-site lane is sealed: export FORGE_SNAPSHOT_PASSPHRASE";
  return m.store;
}
@@ -45,3 +45,4 @@
export function lastOffBox(dataDir: string): string | null {
  if (times.length === 0) return null;
  return times.reduce((a, b) => (Date.parse(b) > Date.parse(a) ? b : a));
}
@@ -0,0 +1,85 @@
/**
 * The forge's off-site backup lane in one reading (#11805): the mirror pass
 * the forge runs in its own process, where it writes, whether what it writes
 * is sealed, when it last succeeded, and the last restore drill and where
 * that drill restored from. `/system` shows it.
 *
 * `/system`, its terminal screen (`src/tui/screens/offsite.ts`) and the MCP
 * `system` tool (`kind: offsite`) all read it through `readOffsiteLane`.
 *
 * Plain data in (the mirror's kept state and the last drill), plain data
 * out; no network, no clock but the one given.
 */
import type { Database } from "bun:sqlite";
import type { DrillResult } from "./drill.ts";
import { lastDrill } from "./drills.ts";
import { lastMirror, MIRROR_STALE_MS, type MirrorState } from "./mirror.ts";
export type LaneHealth = "none" | "ok" | "late" | "failing" | "unsealed";
export interface OffsiteLane {
  readonly health: LaneHealth;
  /** The lane's bucket (`host/bucket`), never a key; null when the forge has no off-site lane. */
  readonly where: string | null;
  readonly sealed: boolean;
  /** The last pass that left the lane level with the newest snapshot. */
  readonly lastSuccess: string | null;
  /** The newest snapshot the lane holds. */
  readonly newest: string | null;
  /** Why passes are failing now, one line. */
  readonly error: string | null;
  readonly drill: {
    readonly at: string;
    readonly verdict: "green" | "red";
    readonly from: string | null;
    readonly offSite: boolean;
    readonly reason: string | null;
  } | null;
}
export function offsiteLane(state: MirrorState | undefined, drill: DrillResult | undefined, now: number): OffsiteLane {
  const drillSeen = drill === undefined
    ? null
    : { at: drill.at, verdict: drill.verdict, from: drill.from ?? null, offSite: drill.offSite === true, reason: drill.reason };
  if (state === undefined)
    return { health: "none", where: null, sealed: false, lastSuccess: null, newest: null, error: null, drill: drillSeen };
  const sealed = state.sealed === true;
  const synced = Date.parse(state.syncedAt ?? "");
  // A lane without the passphrase refuses every write (#11805); that, not its failing passes, is what to fix.
  const health: LaneHealth =
    !sealed ? "unsealed"
    : state.failingSince !== undefined ? "failing"
    : !Number.isFinite(synced) || now - synced > MIRROR_STALE_MS ? "late"
    : "ok";
  return {
    health,
    where: state.where,
    sealed,
    lastSuccess: state.syncedAt ?? null,
    newest: state.newest ?? null,
    error: state.failingSince === undefined ? null : (state.error ?? "unknown reason"),
    drill: drillSeen,
  };
}
/** The lane now, from what the last pass and the last drill kept: one small file and one row, no network. */
export function readOffsiteLane(dataDir: string, db: Database, now = Date.now()): OffsiteLane {
  return offsiteLane(lastMirror(dataDir), lastDrill(db), now);
}
/** One sentence for the lane, as `/system` heads it. */
export function laneWords(l: OffsiteLane): string {
  switch (l.health) {
    case "none":
      return "No off-site lane: set FORGE_MIRROR_S3_* (or seal a mirror on the Backups page) and FORGE_SNAPSHOT_PASSPHRASE.";
    case "failing":
      return `The off-site lane is failing: ${l.error}.`;
    case "late":
      return "The off-site lane has not succeeded for over a day.";
    case "unsealed":
      return "The off-site lane writes nothing until FORGE_SNAPSHOT_PASSPHRASE is set: it never writes unsealed.";
    case "ok":
      return "The off-site lane is level and sealed.";
  }
}
@@ -48,6 +48,7 @@
import {
import { unwrapKey, WRAPPED_KEY } from "./wrap.ts";
import { isPackedFile, partPaths, unpacker, type PackedFile, type Unpacker } from "./parts.ts";
import { resolveDestination } from "./destination.ts";
import { offsiteFromEnv } from "./mirror.ts";
import type { Store } from "./store.ts";
export interface Restored {
@@ -393,17 +394,26 @@
if (import.meta.main) {
    }
    args.splice(at, 2);
  }
  // `--offsite`: read the sealed off-site lane (FORGE_MIRROR_S3_*) in place of the destination (#11805).
  const offsite = args.indexOf("--offsite");
  if (offsite >= 0) args.splice(offsite, 1);
  const [name, dataDir] = args;
  if (!name || !dataDir || args.length > 2) {
    // CLI usage on stderr for the person at the terminal, not a forge event.
    console.error(
      "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N]",
      "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite]",
    );
    process.exit(2);
  }
  // A restore starts from an empty directory, so nothing is sealed yet: the
  // same resolver answers from the environment alone.
  const store = resolveDestination()?.store;
  const lane = offsite >= 0 ? offsiteFromEnv() : undefined;
  if (typeof lane === "string") {
    // CLI message on stderr for the person at the terminal.
    console.error(lane);
    process.exit(2);
  }
  const store = lane ?? resolveDestination()?.store;
  if (store === undefined) {
    // CLI message on stderr for the person at the terminal.
    console.error(
@@ -0,0 +1,177 @@
/**
 * The off-site lane's store (#11805): a store whose every object is sealed
 * before it leaves the forge, so the bucket on neither machine holds only
 * ciphertext. The key is derived from the operator's passphrase
 * (`FORGE_SNAPSHOT_PASSPHRASE`) with scrypt under a random salt of the
 * lane's own, kept beside the objects in one unsealed header
 * (`sealed/lane.json`: the salt and a check value, never the key), so a
 * guess at the passphrase costs a scrypt per lane and nothing computed for
 * one lane opens another. The passphrase lives in the forge's environment
 * and with the operator, never in the repository and never on the bucket.
 *
 * Each body is AES-256-GCM under a fresh 12-byte IV (`FSO1`, IV, ciphertext,
 * tag), so a tampered object fails the tag and says so. Object names are
 * hidden too: everything after `snapshots/<name>/` is an HMAC of the path,
 * since a path names a repository. The snapshot's name (a UTC moment) stays
 * readable, so a listing still sorts, prunes and resumes. Everything sits
 * under `sealed/`, apart from anything written there before the lane sealed.
 *
 * Plain data in, plain data out; the inner store does the I/O.
 */
import { createCipheriv, createDecipheriv, createHmac, randomBytes, scrypt, timingSafeEqual } from "node:crypto";
import type { Store } from "./store.ts";
export const SEALED_PREFIX = "sealed/";
const MAGIC = Buffer.from("FSO1");
const IV_BYTES = 12;
const TAG_BYTES = 16;
/** The lane's header, beside the sealed objects and unsealed: the salt and a check value, never the key. */
export const LANE_HEADER = `${SEALED_PREFIX}lane.json`;
const SALT_BYTES = 16;
const SALT_HEX = /^[0-9a-f]{32}$/;
const CHECK_HEX = /^[0-9a-f]{64}$/;
const SCRYPT = { N: 1 << 15, r: 8, p: 1, maxmem: 64 * 1024 * 1024 } as const;
const HASHED = /^[0-9a-f]{64}$/;
const IN_SNAPSHOT = /^(snapshots\/[^/]+\/)(.+)$/;
export interface SealKeys {
  readonly enc: Buffer;
  readonly mac: Buffer;
}
/** The two keys a passphrase gives under a lane's salt: one seals bodies, one hides names. One scrypt, about 100 ms, off the main thread. */
export function sealKeys(passphrase: string, salt: Uint8Array): Promise<SealKeys> {
  return new Promise((resolve, reject) => {
    scrypt(passphrase, salt, 64, SCRYPT, (error, k) => {
      if (error) reject(error);
      else resolve({ enc: k.subarray(0, 32), mac: k.subarray(32) });
    });
  });
}
/** What the header keeps to tell the right passphrase from a wrong one; an HMAC, so it gives no shortcut past scrypt. */
const checkOf = (keys: SealKeys): string => createHmac("sha256", keys.mac).update("forge off-site lane header").digest("hex");
/** A header's salt and check, or a sentence when the object is not one the lane wrote. */
export function parseHeader(body: Uint8Array): { salt: Buffer; check: string } | string {
  try {
    const v = JSON.parse(new TextDecoder().decode(body)) as { v?: unknown; salt?: unknown; check?: unknown };
    if (v.v !== 1 || typeof v.salt !== "string" || !SALT_HEX.test(v.salt) || typeof v.check !== "string" || !CHECK_HEX.test(v.check))
      return `${LANE_HEADER} is not a lane header`;
    return { salt: Buffer.from(v.salt, "hex"), check: v.check };
  } catch {
    return `${LANE_HEADER} is not a lane header`;
  }
}
/**
 * The lane's keys: from its header's salt, checked, when the lane has one;
 * else, when `create`, a new random salt kept as the header first. A wrong
 * passphrase throws here, before any object is read or written.
 */
export async function laneKeys(inner: Store, passphrase: string, create: boolean): Promise<SealKeys> {
  const body = await inner.get(LANE_HEADER);
  if (body === undefined) {
    if (!create) throw new Error(`${inner.where} holds no sealed lane (no ${LANE_HEADER})`);
    const salt = randomBytes(SALT_BYTES);
    const keys = await sealKeys(passphrase, salt);
    await inner.put(LANE_HEADER, JSON.stringify({ v: 1, salt: salt.toString("hex"), check: checkOf(keys) }));
    return keys;
  }
  const h = parseHeader(body);
  if (typeof h === "string") throw new Error(h);
  const keys = await sealKeys(passphrase, h.salt);
  if (!timingSafeEqual(Buffer.from(checkOf(keys), "hex"), Buffer.from(h.check, "hex")))
    throw new Error(`the passphrase does not open the lane at ${inner.where}`);
  return keys;
}
export function sealBody(keys: SealKeys, body: Uint8Array | string): Uint8Array {
  const iv = randomBytes(IV_BYTES);
  const cipher = createCipheriv("aes-256-gcm", keys.enc, iv, { authTagLength: TAG_BYTES });
  const plain = typeof body === "string" ? Buffer.from(body) : body;
  return Buffer.concat([MAGIC, iv, cipher.update(plain), cipher.final(), cipher.getAuthTag()]);
}
/** The plain body; throws, naming the key, when the passphrase is wrong or the object is not one the lane sealed. */
export function openBody(keys: SealKeys, sealed: Uint8Array, key: string): Uint8Array {
  const b = Buffer.from(sealed.buffer, sealed.byteOffset, sealed.byteLength);
  if (b.length < MAGIC.length + IV_BYTES + TAG_BYTES || !b.subarray(0, MAGIC.length).equals(MAGIC))
    throw new Error(`${key} is not a sealed object`);
  const iv = b.subarray(MAGIC.length, MAGIC.length + IV_BYTES);
  const decipher = createDecipheriv("aes-256-gcm", keys.enc, iv, { authTagLength: TAG_BYTES });
  decipher.setAuthTag(b.subarray(b.length - TAG_BYTES));
  try {
    return new Uint8Array(Buffer.concat([decipher.update(b.subarray(MAGIC.length + IV_BYTES, b.length - TAG_BYTES)), decipher.final()]));
  } catch {
    throw new Error(`${key} did not open: wrong passphrase, or the object was changed`);
  }
}
/**
 * A key as the lane sees it: `snapshots/<name>/<hmac>`. A key already in
 * that form (as `list` returns it) is left as it is, so a listed key can be
 * read and deleted.
 */
export function hiddenKey(keys: SealKeys, key: string): string {
  const hide = (path: string): string => (HASHED.test(path) ? path : createHmac("sha256", keys.mac).update(path).digest("hex"));
  const m = IN_SNAPSHOT.exec(key);
  return m ? `${m[1]}${hide(m[2]!)}` : hide(key);
}
export interface SealedOptions {
  /** Whether a lane with no header yet starts one (the mirror pass); a restore reads, so it does not. */
  readonly create?: boolean;
}
/** `inner`, every object sealed under `passphrase` and named so a path never shows. The keys are derived on first use. */
export function sealedStore(inner: Store, passphrase: string, options: SealedOptions = {}): Store {
  let pending: Promise<SealKeys> | undefined;
  // One derivation for the store and every bounded view of it; a failed read of the header is tried again next time.
  const keys = (): Promise<SealKeys> =>
    (pending ??= laneKeys(inner, passphrase, options.create ?? true).catch((error: unknown) => {
      pending = undefined;
      throw error;
    }));
  return over(inner, keys);
}
function over(inner: Store, keys: () => Promise<SealKeys>): Store {
  const at = async (key: string): Promise<string> => SEALED_PREFIX + hiddenKey(await keys(), key);
  const store: Store = {
    where: `${inner.where} (sealed)`,
    sealed: true,
    async put(key, body) {
      const k = await keys();
      await inner.put(SEALED_PREFIX + hiddenKey(k, key), sealBody(k, body));
    },
    async get(key) {
      const k = await keys();
      const body = await inner.get(SEALED_PREFIX + hiddenKey(k, key));
      return body === undefined ? undefined : openBody(k, body, key);
    },
    async list(prefix) {
      await keys();
      return (await inner.list(SEALED_PREFIX + prefix)).map((k) => k.slice(SEALED_PREFIX.length));
    },
    async names(prefix) {
      await keys();
      return inner.names(SEALED_PREFIX + prefix);
    },
    async delete(key) {
      await inner.delete(await at(key));
    },
    storedKey: async (key) => hiddenKey(await keys(), key),
  };
  const bounded = inner.bounded;
  return bounded === undefined ? store : { ...store, bounded: (leftMs) => over(bounded.call(inner, leftMs), keys) };
}
/** A lane that may not write: every call fails with `reason`, so the board says why and nothing leaves unsealed. */
export function refusingStore(where: string, reason: string): Store {
  const no = async (): Promise<never> => {
    throw new Error(reason);
  };
  return { where, put: no, get: no, list: no, names: no, delete: no };
}
@@ -26,7 +26,7 @@
import {
  type Manifest,
  type Progress,
} from "./snapshot.ts";
import { describeTimes, runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { describeTimes, DRILL_TIMEOUT_MS, runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS, snapshotDue } from "./due.ts";
import { sealedTarget } from "./destination.ts";
import { watchBucket } from "./bucket.ts";
@@ -141,6 +141,8 @@
export function startSnapshots(
    /** Test seam: how soon a failed snapshot is tried again. */
    readonly retryMs?: number | undefined;
    readonly passphrase?: string | undefined;
    /** Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it stands. */
    readonly drillWaitMs?: number | undefined;
    /** Test seam: the drill itself. */
    readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
    /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */
@@ -346,13 +348,32 @@
export function startSnapshots(
    return mirrorNow();
  };
  /** Wait for the passes in flight, `ms` at most. */
  const mirrorSettled = async (ms: number): Promise<void> => {
    const end = Date.now() + ms;
    while (mirroring !== undefined) {
      const left = end - Date.now();
      if (left <= 0) {
        log.warn("restore drill: the mirror pass is still running; drilling the lane as it stands");
        return;
      }
      let timer: ReturnType<typeof setTimeout> | undefined;
      await Promise.race([mirroring.catch(() => {}), new Promise((r) => (timer = setTimeout(r, left)))]);
      clearTimeout(timer);
    }
  };
  // Single-flight: a restore of every repository is heavy, so a second call
  // while one runs joins it instead of filling the disk with another.
  let drilling: Promise<DrillResult> | undefined;
  const drill = (): Promise<DrillResult> => {
    drilling ??= (async (): Promise<DrillResult> => {
      await (inFlight ?? Promise.resolve()).catch(() => {});
      const store = current();
      // The off-site copy is the one a lost machine leaves (#11805): drill it,
      // once the pass that follows every snapshot has brought it level, or
      // the drill's floor budget has gone by, so a stuck pass cannot hang it.
      await mirrorSettled(options.drillWaitMs ?? DRILL_TIMEOUT_MS);
      const store = currentMirror() ?? current();
      if (store === undefined)
        throw new Error("this forge has nowhere to drill from");
      // A drill that could not run is not a drill that passed: it is kept red,
@@ -27,6 +27,14 @@
export interface Store {
   * that does not retry has nothing to bound and leaves this out.
   */
  bounded?(leftMs: () => number): Store;
  /**
   * The form `list` gives `key` in, when the store renames what it holds
   * (the sealed lane hides paths); a store that keeps keys as given leaves
   * this out.
   */
  storedKey?(key: string): Promise<string>;
  /** True when every object is encrypted before it reaches the store (the off-site lane, #11805). */
  readonly sealed?: boolean;
}
/** The S3 credentials as the platform hands them to an app. */
@@ -30,8 +30,8 @@
import { serviceAccount, type Credentials } from "../cluster/jobs.ts";
export type Need = "answer" | "sign-in" | "build" | "release" | "backup";
/** The machines in the forge's path, by the name the steward's inventory gives them. */
export type Machine = "pg-grand" | "vxrail";
/** The machines in the forge's path, by the name the steward's inventory gives them; `off-site` is neither (#11805). */
export type Machine = "pg-grand" | "vxrail" | "off-site";
/** The machine the forge itself runs on, as declared. */
export const FORGE_MACHINE: Machine = "vxrail";
@@ -40,8 +40,12 @@
export const FORGE_MACHINE: Machine = "vxrail";
export const MACHINES: Record<Machine, string> = {
  "pg-grand": "DigitalOcean. The k3s server, a single control plane with no HA. The owner decided on 2026-11-12 to drop it.",
  vxrail: "A k3s agent joined to pg-grand. The forge runs here.",
  "off-site": "Object storage on neither machine, at a provider of its own: the off-site backup lane's bucket.",
};
/** A piece on the forge's own machine or on neither machine: losing pg-grand does not touch it. */
const besideTheForge = (m: Machine): boolean => m === FORGE_MACHINE || m === "off-site";
export type Find =
  /** Pods of one workload (Deployment, StatefulSet, DaemonSet) in a namespace, or every pod (`*`); namespace `""` is the forge's own. */
  | { readonly kind: "pods"; readonly namespace: string; readonly workload: string }
@@ -49,6 +53,8 @@
export type Find =
  | { readonly kind: "self" }
  /** The Kubernetes API server: the nodes labelled control-plane. */
  | { readonly kind: "control-plane" }
  /** Object storage outside the cluster and off both machines; nothing to look for. */
  | { readonly kind: "offsite" }
  /** Not in the cluster: a systemd unit on a host; the cluster cannot show it, so the declared machine stands. */
  | { readonly kind: "host"; readonly unit: string }
  /** A pod whose namespace is not recorded: not looked for, since that would list every pod in the cluster. */
@@ -78,7 +84,6 @@
export const BREAK_STEPS: readonly string[] = [
  "the signup gate: no one signs up",
  "image pulls and releases",
  "the control plane: nothing reschedules, Services stop updating",
  "local backups",
  "the host timers",
];
@@ -140,7 +145,7 @@
export const FOOTING: readonly Piece[] = [
    machine: "pg-grand",
    ifDies: "git over ssh and the terminal stop answering.",
    issues: [11686],
    breaks: 7,
    breaks: 6,
  },
  {
    id: "control-plane",
@@ -229,7 +234,7 @@
export const FOOTING: readonly Piece[] = [
    machine: "pg-grand",
    ifDies: "Nothing is released; production keeps the release it has.",
    issues: [11686, 8942],
    breaks: 7,
    breaks: 6,
  },
  {
    id: "standby-reconcile",
@@ -239,7 +244,7 @@
export const FOOTING: readonly Piece[] = [
    machine: "pg-grand",
    ifDies: "A changed manifest is not applied; the running pod is untouched.",
    issues: [8948, 11686],
    breaks: 7,
    breaks: 6,
  },
  {
    id: "host-timers",
@@ -249,33 +254,31 @@
export const FOOTING: readonly Piece[] = [
    machine: "pg-grand",
    ifDies: "The watchdog, the GitHub mirror and image pruning stop.",
    issues: [11686],
    breaks: 7,
  },
  {
    id: "backup-lanes",
    need: "backup",
    what: "The backup namespace's CronJobs (FORGE_BACKUP_LANES)",
    find: { kind: "pods", namespace: "backup", workload: "*" },
    machine: "pg-grand",
    ifDies: "No new backup is taken.",
    issues: [11686],
    breaks: 6,
  },
  {
    id: "minio",
    id: "backup-lane",
    need: "backup",
    what: "MinIO, where the backup lanes store the forge's backups",
    find: { kind: "pods", namespace: "minio", workload: "*" },
    machine: "pg-grand",
    ifDies: "No new backup is stored, and those it holds go with the machine.",
    issues: [11686],
    breaks: 6,
    what: "The forge's own backup lane (src/snapshot): the nightly snapshot and the hourly off-site pass, run in the forge's process, every object sealed under FORGE_SNAPSHOT_PASSPHRASE before it leaves; the nightly lands on the forge's own disk (FORGE_SNAPSHOT_DIR on its volume), not the platform's MinIO",
    find: { kind: "self" },
    machine: "vxrail",
    ifDies: "No new backup is taken; the sealed copies wait off both machines.",
    issues: [],
  },
  {
    id: "offsite-bucket",
    need: "backup",
    what: "The off-site lane's bucket (FORGE_MIRROR_S3_*, or the mirror sealed on the Backups page), on neither machine",
    find: { kind: "offsite" },
    machine: "off-site",
    ifDies: "Neither machine holds it: losing either leaves every sealed copy in place.",
    issues: [],
  },
];
/** On pg-grand, not in the forge's path: what dropping the machine takes besides. */
export const ALSO_ON_PG_GRAND =
  "Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and MinIO.";
  "Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and the platform's backup CronJobs (the backup namespace, FORGE_BACKUP_LANES) with the MinIO they write to; the forge's own backup no longer needs them.";
/** How each piece was placed: seen on nodes, or the declared machine kept and why. */
export type Seen =
@@ -311,7 +314,7 @@
export function declaredFooting(why: string, at = new Date()): Footing {
    forgeNode: null,
    unread: why,
    at: at.toISOString(),
    rows: FOOTING.map((p) => ({ ...p, seen: { by: "declared", why }, risk: p.machine !== FORGE_MACHINE })),
    rows: FOOTING.map((p) => ({ ...p, seen: { by: "declared", why }, risk: !besideTheForge(p.machine) })),
  };
}
@@ -387,6 +390,7 @@
export async function probeFooting(opts: ProbeOptions = {}): Promise<Footing> {
    }
    // Nothing in the cluster sees a systemd unit: the declared machine stands, said as such.
    if (f.kind === "host") return { by: "declared", why: `${f.unit} is a systemd unit on a host; the cluster cannot show it` };
    if (f.kind === "offsite") return { by: "declared", why: "object storage outside the cluster, on neither machine" };
    if (f.kind === "unlocated") return { by: "declared", why: "its namespace is not recorded, and the forge does not list every pod in the cluster" };
    const where = nsOf(f.namespace);
    const list = pods.get(where)!;
@@ -403,7 +407,7 @@
export async function probeFooting(opts: ProbeOptions = {}): Promise<Footing> {
    const risk =
      seen.by === "cluster" && forgeNode !== null
        ? seen.nodes.some((n) => n !== forgeNode)
        : p.machine !== FORGE_MACHINE;
        : !besideTheForge(p.machine);
    return { ...p, seen, risk };
  });
  const unread =
@@ -60,6 +60,7 @@
import { entry as economics } from "./economics.ts";
import { entry as money } from "./money.ts";
import { entry as loopStalls } from "./loopstalls.ts";
import { entry as footing } from "./footing.ts";
import { entry as offsite } from "./offsite.ts";
import { entry as waiting } from "./waiting.ts";
import { entry as slowPagesEntry } from "./slowpages.ts";
import { entry as maintenance } from "./maintenance.ts";
@@ -126,6 +127,7 @@
export const ENTRIES: readonly PaletteEntry[] = [
  system,
  loopStalls,
  footing,
  offsite,
  waiting,
  slowPagesEntry,
  telemetry,
@@ -0,0 +1,78 @@
/**
 * The off-site backup lane, in the terminal (#11805): what /system shows
 * under "Off-site backup", in the same order. First the lane's one sentence,
 * then where it writes, whether it seals what it writes, when it last
 * succeeded, and the last restore drill with where it restored from. The
 * reading is `readOffsiteLane`'s, the page's and the MCP tool's (one small
 * file and one row; no network). The gate is /system's: a forge admin who
 * is a person. Read-only.
 */
import { laneWords, readOffsiteLane, type OffsiteLane } from "../../snapshot/offsite_lane.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { pad } from "../term/width.ts";
import { Scroll, drawLines, hitScroll } from "./common.ts";
import type { PaletteEntry } from "./entries.ts";
const mayRead = (session: Session): boolean => session.user?.isAdmin === true && session.user.kind !== "agent";
const fact = (label: string, value: string): string => `  ${pad(label, 14)}${value}`;
/** The screen's lines; empty for anyone the gate refuses. */
export function offsiteLines(session: Session, l: OffsiteLane = readOffsiteLane(session.app.config.dataDir, session.app.db)): string[] {
  if (!mayRead(session)) return [];
  const lines = [laneWords(l), ""];
  if (l.where !== null)
    lines.push(
      "Backup lane",
      fact("Writes to", l.where),
      fact("Encrypted", l.sealed ? "yes, under the snapshot passphrase" : "no"),
      fact("Last success", l.lastSuccess ?? "never"),
      ...(l.newest === null ? [] : [fact("Newest", l.newest)]),
      "",
    );
  const d = l.drill;
  lines.push("Restore drill");
  if (d === null) lines.push(fact("Last", "never run"));
  else
    lines.push(
      fact("Last", d.at),
      fact("Verdict", d.verdict === "green" ? "green" : `red${d.reason === null ? "" : `: ${d.reason}`}`),
      fact("From", d.from === null ? "not recorded" : `${d.from}${d.offSite ? " (off-site)" : " (on this machine)"}`),
    );
  return lines;
}
export function offsiteScreen(session: Session, ctx: Ctx): Screen {
  const scroll = new Scroll();
  return {
    title: "Off-site backup",
    keys: [binding(["j", "k"], "scroll"), binding(["esc", "q"], "back")],
    render(area, s) {
      const c = ctx.theme.c;
      const lines = offsiteLines(session);
      drawLines(s, area, c, scroll, lines.map((text) => [{ text, style: style(text.startsWith("  ") ? c.mutedFg : c.fg, c.bg) }]));
      hitScroll(ctx, area, scroll, lines.length);
    },
    key(e) {
      if (scroll.key(e, offsiteLines(session).length)) return true;
      if (e.key === "esc" || e.key === "q") {
        ctx.pop();
        return true;
      }
      return false;
    },
  };
}
export const entry: PaletteEntry = {
  id: "offsite",
  title: "Off-site backup",
  hint: "where the sealed lane writes, when it last succeeded, and the last restore drill",
  after: ["footing"],
  offered: mayRead,
  run: (session, ctx) => ctx.push(offsiteScreen(session, ctx)),
};
@@ -48,6 +48,7 @@
import { COLUMN_BYTES, PARTS_BYTES, keptPartsStats, treeCacheStats } from "./tre
import { featureHealth, type FeatureHealth } from "../../system/feature_health.ts";
import { egressReportedLine } from "../../system/egress_line.ts";
import { BREAK_STEPS, MACHINES, declaredFooting, footing, offMachine, whereWords, type Footing, type Machine } from "../../system/footing.ts";
import { laneWords, readOffsiteLane, type OffsiteLane } from "../../snapshot/offsite_lane.ts";
import { badge, chartBox, disclosure, meter, pill, row, time } from "../ui.ts";
import { childDirOf, listChildren } from "../../actions/spawn.ts";
import { childrenUse, realSelf, selfUse, shareOf, sizeWords, useWords, type ChildUseRow, type SelfUse } from "../../spawn/usage.ts";
@@ -237,6 +238,43 @@
export function footingSection(f: Footing): Html {
    ${(Object.keys(MACHINES) as Machine[]).map((m) => html`<p class="small muted" data-machine="${m}"><span class="mono">${m}</span>: ${MACHINES[m]}</p>`)}`, { id: "sys-footing-more" })}</section>`;
}
/**
 * The off-site backup lane (#11805): where the forge's own mirror pass
 * writes, whether it seals what it writes, when it last succeeded, and the
 * last restore drill with where it restored from. `snapshot/offsite_lane.ts`'s reading.
 */
export function offsiteSection(l: OffsiteLane): Html {
  // A lane that exists and is not level and sealed is red; no lane at all is a setting, said in words (first sight, #11111).
  const tone = l.health === "ok" || l.health === "none" ? null : badge(l.health, { tone: "danger" });
  const d = l.drill;
  // First sight (#11111): the heading and one sentence stay; the rows open behind one disclosure.
  return html`<section class="settings-block" id="sys-offsite" data-health="${l.health}"><h2>Off-site backup</h2>
    ${disclosure(html`${tone} ${laneWords(l)}`, html`<ul class="list" aria-label="Off-site backup">
      ${l.where === null ? null : row({
        title: "Backup lane",
        facts: [
          { label: "Writes to", value: html`<span class="mono">${l.where}</span>` },
          { label: "Encrypted", value: l.sealed ? "yes, under the snapshot passphrase" : "no" },
          { label: "Last success", value: l.lastSuccess === null ? "never" : time(l.lastSuccess) },
          ...(l.newest === null ? [] : [{ label: "Newest", value: html`<span class="mono">${l.newest}</span>` }]),
        ],
        data: { lane: "offsite", sealed: String(l.sealed), ...(l.lastSuccess === null ? {} : { success: l.lastSuccess }) },
      })}
      ${row({
        title: "Restore drill",
        facts: d === null
          ? [{ label: "Last", value: "never run" }]
          : [
              { label: "Last", value: time(d.at) },
              { label: "Verdict", value: d.verdict === "green" ? "green" : html`${badge("red", { tone: "danger" })} ${d.reason ?? ""}` },
              { label: "From", value: d.from === null ? "not recorded" : html`<span class="mono">${d.from}</span>${d.offSite ? " (off-site)" : " (on this machine)"}` },
            ],
        data: { drill: d?.verdict ?? "none", ...(d === null ? {} : { "off-site": String(d.offSite) }) },
      })}
    </ul>
    <p class="small"><a href="/settings/backups">Backups</a> runs a drill; so does the MCP <code>restore_drill</code>.</p>`, { id: "sys-offsite-more" })}</section>`;
}
/** The last hour's stalls by route, most milliseconds first; the rows are `stallHour`'s. */
export function stallSection(rows: readonly RouteStall[], worst: Html = html``): Html {
  return html`<section class="settings-block"><h2>Main-thread stalls, last hour</h2>${
@@ -439,6 +477,7 @@
export async function systemPage(ctx: Ctx): Promise<Response> {
        <button class="btn" data-size="sm" type="button" data-sort="mem" aria-pressed="false"><kbd>m</kbd> memory</button></div>
      <div id="sys-procs" data-live-hold>${at("sys-procs")}</div></section>
    ${footingSection(standsOn)}
    ${offsiteSection(timePhase(ctx.request, "offsite", () => readOffsiteLane(ctx.app.config.dataDir, ctx.app.db)))}
    ${featuresSection(timePhase(ctx.request, "features", () => featureHealth(ctx.app, reader)))}
    ${lagSection(lagMinutes.readings())}
    <div id="sys-stalls">${stallSection(stalls, worstStallsTable(worst, nowMono, Date.now()))}</div>
@@ -148,14 +148,14 @@
describe("probeFooting", () => {
    expect(byId(f, "standby-reconcile")).toMatchObject({ seen: { by: "declared" }, risk: true });
    for (const id of ["forge-ssh", "release-tick", "standby-reconcile", "host-timers"]) expect(byId(f, id).seen.by).toBe("declared");
    expect(byId(f, "plat-auth").risk).toBe(true);
    // Forbidden: the declared machine is kept and the reason named.
    expect(byId(f, "minio").seen).toEqual({ by: "declared", why: "the forge's service account may not read pods in minio" });
    expect(byId(f, "minio").risk).toBe(true);
    // A CronJob between runs: only a finished pod, so declared.
    expect(byId(f, "backup-lanes").seen).toEqual({ by: "declared", why: "no pod running in backup now" });
    // The backup lane is the forge's own process, and its bucket is on neither machine (#11805).
    expect(byId(f, "backup-lane")).toMatchObject({ seen: { by: "cluster", nodes: ["vxrail-1"] }, risk: false });
    expect(byId(f, "offsite-bucket")).toMatchObject({ machine: "off-site", seen: { by: "declared" }, risk: false });
    // Neither the platform's backup namespace nor MinIO is asked for any more.
    expect(kube.seen.some((p) => p.includes("/namespaces/backup/") || p.includes("/namespaces/minio/"))).toBe(false);
    // Each namespace is listed once although two pieces live in forgejo.
    expect(kube.seen.filter((p) => p === "/api/v1/namespaces/forgejo/pods").length).toBe(1);
    const off = ["forge-ssh", "control-plane", "plat-auth", "signup-gate", "forgejo", "cnpg", "registry", "release-tick", "standby-reconcile", "host-timers", "backup-lanes", "minio"];
    const off = ["forge-ssh", "control-plane", "plat-auth", "signup-gate", "forgejo", "cnpg", "registry", "release-tick", "standby-reconcile", "host-timers"];
    expect(f.rows.filter((r) => r.risk).map((r) => r.id).sort()).toEqual([...off].sort());
    // In the order pg-grand switched off would break the forge: sign-in first, host timers last.
    const lines = riskLines(f);
@@ -203,7 +203,7 @@
describe("probeFooting", () => {
    expect(fetched).toBe(0);
    expect(f.forgeNode).toBeNull();
    expect(f.unread).toBe("not read: not running in a cluster");
    expect(f.rows.every((r) => r.seen.by === "declared" && r.risk === (r.machine !== "vxrail"))).toBe(true);
    expect(f.rows.every((r) => r.seen.by === "declared" && r.risk === (r.machine === "pg-grand"))).toBe(true);
  });
  test("the forge's own pod unreadable: its node is unknown and the declared machines decide risk", async () => {
@@ -0,0 +1,334 @@
/**
 * The off-site backup lane (#11805): the mirror pass writes only ciphertext
 * under names that hide every path, a restore from it with the passphrase
 * brings the forge back, the drill restores from it and keeps where, and
 * `/system` says when the lane last succeeded.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, readdirSync, readFileSync, statSync } from "node:fs";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, relative } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { answersRoot, runDrill, type DrillResult } from "../src/snapshot/drill.ts";
import { lastDrill } from "../src/snapshot/drills.ts";
import { lastMirror, mirrorPass, NO_PASSPHRASE, offsiteFromEnv, resolveMirror } from "../src/snapshot/mirror.ts";
import { parseArgs } from "../scripts/restore-drill.ts";
import { offsiteLane, readOffsiteLane } from "../src/snapshot/offsite_lane.ts";
import { restoreSnapshot } from "../src/snapshot/restore.ts";
import { hiddenKey, LANE_HEADER, openBody, parseHeader, sealBody, sealedStore, sealKeys, SEALED_PREFIX } from "../src/snapshot/sealed_store.ts";
import { startSnapshots } from "../src/snapshot/service.ts";
import { listSnapshots, SNAPSHOT_PREFIX } from "../src/snapshot/snapshot.ts";
import { dirStore, type Store } from "../src/snapshot/store.ts";
import { tool as systemTool } from "../src/http/mcp/tools/system.ts";
import { Session } from "../src/tui/forge.ts";
import { offsiteLines } from "../src/tui/screens/offsite.ts";
import { FOOTING } from "../src/system/footing.ts";
import { offsiteSection } from "../src/web/pages/system.ts";
const PASS = "a long operator passphrase";
const SALT = new Uint8Array(16).fill(7);
function files(dir: string, out: string[] = []): string[] {
  for (const n of readdirSync(dir)) {
    const p = join(dir, n);
    if (statSync(p).isDirectory()) files(p, out);
    else out.push(p);
  }
  return out;
}
describe("the sealed store", () => {
  test("a body goes in as ciphertext and comes back whole; the wrong passphrase or a changed byte does not open", async () => {
    const k = await sealKeys(PASS, SALT);
    const sealed = sealBody(k, "SQLite format 3 and a secret");
    expect(Buffer.from(sealed).toString("latin1")).not.toContain("SQLite");
    expect(Buffer.from(openBody(k, sealed, "x")).toString()).toBe("SQLite format 3 and a secret");
    expect(() => openBody(k, sealed, "x")).not.toThrow();
    const other = await sealKeys("not the passphrase", SALT);
    expect(() => openBody(other, sealed, "x")).toThrow("wrong passphrase");
    // The same passphrase under another lane's salt is another key.
    const otherSalt = await sealKeys(PASS, new Uint8Array(16).fill(8));
    expect(() => openBody(otherSalt, sealed, "x")).toThrow("did not open");
    const changed = new Uint8Array(sealed);
    changed[20] = changed[20]! ^ 1;
    expect(() => openBody(k, changed, "x")).toThrow("did not open");
    expect(() => openBody(k, new TextEncoder().encode("plain"), "x")).toThrow("not a sealed object");
  });
  test("a path inside a snapshot is hidden; the snapshot's name stays; a listed key reads back", async () => {
    const k = await sealKeys(PASS, SALT);
    const hidden = hiddenKey(k, "snapshots/2026-11-12T03-00-00Z/repos/alice/private-plans.bundle");
    expect(hidden).toMatch(/^snapshots\/2026-11-12T03-00-00Z\/[0-9a-f]{64}$/);
    expect(hiddenKey(k, hidden)).toBe(hidden);
  });
  test("each lane keeps a random salt of its own in an unsealed header; a wrong passphrase is said before any object is read", async () => {
    const root = mkdtempSync(join(tmpdir(), "forge-offsite-salt-"));
    try {
      const one = sealedStore(dirStore(join(root, "one")), PASS);
      const two = sealedStore(dirStore(join(root, "two")), PASS);
      await one.put("snapshots/2026-11-12T03-00-00Z/x", "body");
      await two.put("snapshots/2026-11-12T03-00-00Z/x", "body");
      const h1 = parseHeader(readFileSync(join(root, "one", LANE_HEADER)));
      const h2 = parseHeader(readFileSync(join(root, "two", LANE_HEADER)));
      if (typeof h1 === "string" || typeof h2 === "string") throw new Error("no header");
      expect(h1.salt.length).toBe(16);
      expect(h1.salt.equals(h2.salt)).toBe(false);
      expect(readFileSync(join(root, "one", LANE_HEADER), "utf8")).not.toContain(PASS);
      // Same passphrase, same path, other salt: the hidden names differ, so nothing computed for one lane serves the other.
      expect(await one.list("snapshots/")).not.toEqual(await two.list("snapshots/"));
      // A reopened lane reads its own salt back.
      expect(new TextDecoder().decode(await sealedStore(dirStore(join(root, "one")), PASS).get("snapshots/2026-11-12T03-00-00Z/x"))).toBe("body");
      await expect(sealedStore(dirStore(join(root, "one")), "not it").get("snapshots/2026-11-12T03-00-00Z/x")).rejects.toThrow("does not open the lane");
      // A reader (a restore) never starts a lane on an empty bucket.
      await expect(sealedStore(dirStore(join(root, "empty")), PASS, { create: false }).list("snapshots/")).rejects.toThrow("holds no sealed lane");
      expect(parseHeader(new TextEncoder().encode('{"v":1,"salt":"zz","check":"00"}'))).toBeTypeOf("string");
    } finally {
      await rm(root, { recursive: true, force: true });
    }
  });
  test("without a passphrase the lane fails closed: no request reaches the bucket, and the board says why", async () => {
    const seen: string[] = [];
    const bucket = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: (req) => (seen.push(`${req.method} ${new URL(req.url).pathname}`), new Response("", { status: 200 })) });
    const root = mkdtempSync(join(tmpdir(), "forge-offsite-closed-"));
    try {
      const env = {
        FORGE_MIRROR_S3_ENDPOINT: `http://127.0.0.1:${String(bucket.port)}`,
        FORGE_MIRROR_S3_BUCKET: "offsite",
        FORGE_MIRROR_S3_ACCESS_KEY: "a",
        FORGE_MIRROR_S3_SECRET_KEY: "s",
      };
      const plain = resolveMirror({ env });
      expect(plain?.encrypted).toBe(false);
      expect(plain?.store.sealed).toBeUndefined();
      await expect(plain!.store.put("snapshots/x/forge.db", "SQLite format 3")).rejects.toThrow(NO_PASSPHRASE);
      const primary = dirStore(join(root, "primary"));
      await primary.put("snapshots/2026-11-12T03-00-00Z/forge.db", "SQLite format 3");
      await primary.put("snapshots/2026-11-12T03-00-00Z/manifest.json", JSON.stringify({ name: "2026-11-12T03-00-00Z", startedAt: "2026-11-12T03:00:00Z", finishedAt: "2026-11-12T03:01:00Z", files: [] }));
      const state = await mirrorPass(root, primary, plain!.store);
      expect(seen).toEqual([]);
      expect(state.error).toBe(NO_PASSPHRASE);
      const lane = offsiteLane(state, undefined, Date.now());
      expect(lane.health).toBe("unsealed");
      expect(String(offsiteSection(lane))).toContain("writes nothing until FORGE_SNAPSHOT_PASSPHRASE is set");
      const sealed = resolveMirror({ env, passphrase: PASS });
      expect(sealed?.encrypted).toBe(true);
      expect(sealed?.store.sealed).toBe(true);
      expect(sealed?.store.where).toBe(`127.0.0.1:${String(bucket.port)}/offsite (sealed)`);
    } finally {
      await bucket.stop(true);
      await rm(root, { recursive: true, force: true });
    }
  });
  test("a restore on a fresh machine reads the lane from the environment, and only with the passphrase", async () => {
    const lane = {
      FORGE_MIRROR_S3_ENDPOINT: "https://s3.b2.example.com",
      FORGE_MIRROR_S3_BUCKET: "offsite",
      FORGE_MIRROR_S3_ACCESS_KEY: "a",
      FORGE_MIRROR_S3_SECRET_KEY: "s",
    };
    expect(offsiteFromEnv({})).toStartWith("no off-site lane");
    expect(offsiteFromEnv(lane)).toBe("the off-site lane is sealed: export FORGE_SNAPSHOT_PASSPHRASE");
    const store = offsiteFromEnv({ ...lane, FORGE_SNAPSHOT_PASSPHRASE: PASS });
    expect(typeof store === "string" ? store : store.sealed).toBe(true);
    expect(parseArgs(["/tmp/x", "--offsite"])).toMatchObject({ dataDir: "/tmp/x", offsite: true });
    expect(parseArgs(["/tmp/x"])).not.toHaveProperty("offsite");
    // The card's command refuses before it touches the network or the directory.
    const env: Record<string, string> = { PATH: process.env["PATH"] ?? "", ...lane };
    const run = Bun.spawnSync([process.execPath, "run", "src/snapshot/restore.ts", "latest", join(tmpdir(), "never-written"), "--offsite"], {
      cwd: join(import.meta.dir, ".."),
      env,
    });
    expect(run.exitCode).toBe(2);
    expect(run.stderr.toString()).toContain("the off-site lane is sealed");
  });
});
describe("the off-site lane end to end", () => {
  let root: string;
  let app: App;
  let offDir: string;
  beforeAll(async () => {
    root = mkdtempSync(join(tmpdir(), "forge-offsite-"));
    offDir = join(root, "offsite-bucket");
    app = createApp({ dataDir: join(root, "data"), port: 0 });
    const u = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!u.ok) throw new Error(u.error);
  });
  afterAll(async () => {
    await rm(root, { recursive: true, force: true });
  });
  test("the pass seals every object, a restore with the passphrase brings the forge back, and the drill restores from it", async () => {
    const mirror = sealedStore(dirStore(offDir), PASS);
    const svc = startSnapshots(app, dirStore(join(root, "primary")), {
      version: "t",
      nightly: false,
      passphrase: PASS,
      mirror,
      drill: (deps) => runDrill({ ...deps, boot: async () => ({ ok: true, reason: "" }) }),
    });
    const m = await svc.now();
    const state = await svc.mirrorFresh();
    expect(state?.failingSince).toBeUndefined();
    expect(state?.sealed).toBe(true);
    expect(lastMirror(app.config.dataDir)?.sealed).toBe(true);
    // On the bucket: only `sealed/snapshots/<name>/<hmac>`, and not one plain byte of the database or the manifest.
    const onBucket = files(offDir);
    // Every object of the snapshot and its manifest, and the lane's header.
    expect(onBucket.length).toBe(m.files.length + 2);
    expect(onBucket.map((f) => relative(offDir, f))).toContain(LANE_HEADER);
    for (const f of onBucket.filter((f) => relative(offDir, f) !== LANE_HEADER)) {
      expect(relative(offDir, f)).toMatch(new RegExp(`^${SEALED_PREFIX}${SNAPSHOT_PREFIX}${m.name}/[0-9a-f]{64}$`));
      const raw = readFileSync(f).toString("latin1");
      expect(raw.startsWith("FSO1")).toBe(true);
      expect(raw).not.toContain("SQLite format");
      expect(raw).not.toContain('"startedAt"');
    }
    // The lane lists and restores like any store, given the passphrase; another passphrase opens nothing.
    expect((await listSnapshots(mirror)).map((x) => x.name)).toEqual([m.name]);
    const target = join(root, "restored");
    const done = await restoreSnapshot(sealedStore(dirStore(offDir), PASS), m.name, target, { passphrase: PASS });
    expect(done.manifest.name).toBe(m.name);
    expect(statSync(join(target, "forge.db")).size).toBeGreaterThan(0);
    // Another passphrase is refused at the header, and the bytes there do not open under it.
    await expect(sealedStore(dirStore(offDir), "wrong").get(`${SNAPSHOT_PREFIX}${m.name}/manifest.json`)).rejects.toThrow("does not open the lane");
    const header = parseHeader(readFileSync(join(offDir, LANE_HEADER)));
    if (typeof header === "string") throw new Error(header);
    const sealedFile = onBucket.find((f) => relative(offDir, f) !== LANE_HEADER)!;
    const wrong = await sealKeys("wrong", header.salt);
    expect(() => openBody(wrong, readFileSync(sealedFile), "x")).toThrow("wrong passphrase");
    // The drill restores the newest off-site snapshot, green, and keeps where it came from.
    const d = await svc.drill();
    expect(d.verdict).toBe("green");
    expect(d.snapshotId).toBe(m.name);
    expect(d.from).toBe(mirror.where);
    const kept = lastDrill(app.db);
    expect(kept?.from).toBe(mirror.where);
    expect(kept?.verdict).toBe("green");
    await svc.stop();
  }, 60_000);
  test("/system's section says when the lane last succeeded and what the drill restored from", () => {
    const lane = offsiteLane(lastMirror(app.config.dataDir), lastDrill(app.db), Date.now());
    expect(lane.health).toBe("ok");
    const out = String(offsiteSection(lane));
    expect(out).toContain('id="sys-offsite"');
    expect(out).toContain('data-health="ok"');
    expect(out).toContain('data-sealed="true"');
    expect(out).toContain(`data-success="${lane.lastSuccess}"`);
    expect(out).toContain("Last success");
    expect(out).toContain('data-drill="green"');
  });
  test("the terminal screen and the MCP read say what /system says, to a forge admin alone", async () => {
    const lane = readOffsiteLane(app.config.dataDir, app.db);
    const ident = (u: Awaited<ReturnType<typeof person>>) => ({ sshUsername: u.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:t", user: u });
    const admin = await person("lane-keeper", true);
    const member = await person("lane-reader", false);
    const lines = offsiteLines(new Session(app, ident(admin)));
    expect(lines[0]).toBe("The off-site lane is level and sealed.");
    expect(lines.join("\n")).toContain(`Writes to     ${lane.where!}`);
    expect(lines.join("\n")).toContain(`Last success  ${lane.lastSuccess!}`);
    expect(lines.join("\n")).toContain(`From          ${lane.drill!.from!} (off-site)`);
    expect(offsiteLines(new Session(app, ident(member)))).toEqual([]);
    const read = await systemTool.run({ app } as never, admin, { kind: "offsite" });
    expect(JSON.stringify(read)).toContain(`"lastSuccess":"${lane.lastSuccess!}"`);
    expect(JSON.stringify(read)).toContain('"says":"The off-site lane is level and sealed."');
    expect(JSON.stringify(await systemTool.run({ app } as never, member, { kind: "offsite" }))).toContain("forbidden");
  });
  test("a mirror pass that never ends holds a drill no longer than the drill's wait", async () => {
    let release = (): void => {};
    const held = new Promise<void>((r) => (release = r));
    const stuck: Store = { ...dirStore(join(root, "stuck")), where: "stuck", names: async () => (await held, []) };
    let drilled = false;
    const svc = startSnapshots(app, dirStore(join(root, "primary")), {
      version: "t",
      nightly: false,
      mirror: stuck,
      drillWaitMs: 50,
      drill: async (): Promise<DrillResult> => {
        drilled = true;
        return { at: new Date().toISOString(), snapshotId: null, ms: 0, counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 }, health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 0, reason: null };
      },
    });
    const pass = svc.mirrorFresh();
    const started = performance.now();
    await svc.drill();
    expect(drilled).toBe(true);
    expect(performance.now() - started).toBeLessThan(5_000);
    release();
    await pass;
    await svc.stop();
  });
  async function person(username: string, isAdmin: boolean) {
    const r = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", isAdmin });
    if (!r.ok) throw new Error(r.error);
    return r.value;
  }
});
describe("answersRoot", () => {
  test("a restored forge's / answers below 400, a redirect included; a 5xx is red", async () => {
    const statuses: Record<string, number> = { "/ok/": 200, "/moved/": 302, "/broken/": 500 };
    const server = Bun.serve({
      port: 0,
      hostname: "127.0.0.1",
      fetch: (req) => {
        const status = statuses[new URL(req.url).pathname] ?? 404;
        return new Response("", { status, headers: status === 302 ? { location: "/signin" } : {} });
      },
    });
    try {
      const base = `http://127.0.0.1:${String(server.port)}`;
      await answersRoot(`${base}/ok`);
      await answersRoot(`${base}/moved`);
      await expect(answersRoot(`${base}/broken`)).rejects.toThrow("answered / with 500");
    } finally {
      await server.stop(true);
    }
  });
});
describe("the lane's health", () => {
  const now = Date.parse("2026-11-12T12:00:00Z");
  const base = { where: "b2.example.com/offsite (sealed)", attemptAt: "2026-11-12T11:00:00Z", reachable: true };
  test("none, ok, unsealed, late and failing", () => {
    expect(offsiteLane(undefined, undefined, now).health).toBe("none");
    expect(offsiteLane({ ...base, sealed: true, syncedAt: "2026-11-12T11:00:00Z" }, undefined, now).health).toBe("ok");
    expect(offsiteLane({ ...base, sealed: false, syncedAt: "2026-11-12T11:00:00Z" }, undefined, now).health).toBe("unsealed");
    expect(offsiteLane({ ...base, sealed: true, syncedAt: "2026-11-10T11:00:00Z" }, undefined, now).health).toBe("late");
    const failing = offsiteLane({ ...base, sealed: true, failingSince: "2026-11-12T10:00:00Z", error: "refused" }, undefined, now);
    expect(failing.health).toBe("failing");
    expect(String(offsiteSection(failing))).toContain("The off-site lane is failing: refused.");
  });
});
describe("FOOTING", () => {
  test("no backup piece is on pg-grand", () => {
    const backup = FOOTING.filter((p) => p.need === "backup");
    expect(backup.length).toBeGreaterThan(0);
    expect(backup.every((p) => p.machine !== "pg-grand")).toBe(true);
    expect(backup.map((p) => p.machine).sort()).toEqual(["off-site", "vxrail"]);
  });
});
@@ -119,6 +119,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["telemetry", "Telemetry", "which agents may read flow and system; enter flips the grant"],
    ["stalls", "Main-thread stalls", "the last hour's stall ms by route"],
    ["footing", "What the forge stands on", "each piece on another machine, in the order it would break the forge"],
    ["offsite", "Off-site backup", "where the sealed lane writes, when it last succeeded, and the last restore drill"],
    ["capacity", "Capacity", "concurrency and guards, live; e changes a settable one"],
    ["resilience", "Resilience", "where snapshots go and the mirror; each protection green, red or untested"],
    ["maintenance", "Maintenance", "prepare the forge for a power-off, check the drain, or end maintenance"],
@@ -53,6 +53,7 @@
test("one <h2> per part, each in a settings-block, and no card", async () => {
    "Disks and network",
    "Processes",
    "What the forge stands on",
    "Off-site backup",
    "Loop lag, last 20 minutes",
    "Main-thread stalls, last hour",
    "Telemetry readers",
		@@ -103,6 +103,7 @@ down.
103	103	<!-- footing:start — written by footingTable() in src/system/footing.ts; test/footing.test.ts holds it equal -->
104	104	- pg-grand: DigitalOcean. The k3s server, a single control plane with no HA. The owner decided on 2026-11-12 to drop it.
105	105	- vxrail: A k3s agent joined to pg-grand. The forge runs here.
	106	- off-site: Object storage on neither machine, at a provider of its own: the off-site backup lane's bucket.
106	107
107	108	\| Needed to \| Piece \| Runs on \| If that machine dies \| Breaks \| Removed by \|
108	109	\|---\|---\|---\|---\|---\|---\|
		@@ -111,7 +112,7 @@ down.
111	112	\| (a) answer a request \| The vxrail's own tunnel (standby/cloudflared-forge-home, 2 replicas): it too sends *.open-platform.sh to Traefik; only forge-home-test.open-platform.sh and arm--forge-trevato-apps.open-platform.sh go straight to the forge \| vxrail \| It goes with the forge, which runs beside it. \| — \| — \|
112	113	\| (a) answer a request \| Traefik (kube-system/traefik): every public request goes tunnel → Traefik → the forge's Service \| pg-grand \| The forge is off the internet; it keeps running inside. \| 1 \| #11686, #8946 \|
113	114	\| (a) answer a request \| Child and app hosts, Traefik IngressRoutes forge-children (<name>--forge.open-platform.sh), forge-apps (*--forge-trevato-apps.open-platform.sh) and forge-old-host, and game.cheffryjames.com and www.cheffryjames.com in the forge's namespace \| pg-grand \| No child forge, app or custom domain answers. \| 1 \| #11686, #8946 \|
114		\| (a) answer a request \| forge-ssh, a host socket proxy (:2222 → the forge's ssh service) \| pg-grand \| git over ssh and the terminal stop answering. \| 7 \| #11686 \|
	115	\| (a) answer a request \| forge-ssh, a host socket proxy (:2222 → the forge's ssh service) \| pg-grand \| git over ssh and the terminal stop answering. \| 6 \| #11686 \|
115	116	\| (a) answer a request \| The k3s API server (the vxrail is an agent joined to it) \| pg-grand \| Running pods keep running, but nothing reschedules, Services stop updating, and no release rolls or Job runs. \| 5 \| #11686 \|
116	117	\| (b) sign a person in \| Sign-in edge: a forwardAuth middleware on the forge's Ingress, answered by mcp/plat-auth \| pg-grand \| Every request through the edge answers 500, /healthz too. \| 2 \| #11686 \|
117	118	\| (b) sign a person in \| forge-signup-gate, the pod /signup is routed to \| pg-grand \| No one can sign up. \| 3 \| #11686 \|
		@@ -120,11 +121,11 @@ down.
120	121	\| (c) build \| forgejo-runner-vxrail, a Forgejo Actions runner \| vxrail \| It goes with the forge, which runs beside it. \| — \| — \|
121	122	\| (d) release \| Forgejo's registry (git.open-platform.sh), which the forge's image is pulled from \| pg-grand \| A pod that restarts on a node without the image cached cannot start; nothing new is released. \| 4 \| #11686, #8941, #8942 \|
122	123	\| (d) release \| registry-node, a registry on the vxrail \| vxrail \| It goes with the forge, which runs beside it. \| — \| — \|
123		\| (d) release \| forge-release-tick, a host timer every 5 min \| pg-grand \| Nothing is released; production keeps the release it has. \| 7 \| #11686, #8942 \|
124		\| (d) release \| standby-reconcile, a host timer every 2 min that applies the forge's manifests from Forgejo; Flux's Kustomization for the forge is suspended \| pg-grand \| A changed manifest is not applied; the running pod is untouched. \| 7 \| #8948, #11686 \|
125		\| (d) release \| Host timers forge-watchdog, forge-github-mirror and forge-prune-images \| pg-grand \| The watchdog, the GitHub mirror and image pruning stop. \| 7 \| #11686 \|
126		\| (e) back up \| The backup namespace's CronJobs (FORGE_BACKUP_LANES) \| pg-grand \| No new backup is taken. \| 6 \| #11686 \|
127		\| (e) back up \| MinIO, where the backup lanes store the forge's backups \| pg-grand \| No new backup is stored, and those it holds go with the machine. \| 6 \| #11686 \|
	124	\| (d) release \| forge-release-tick, a host timer every 5 min \| pg-grand \| Nothing is released; production keeps the release it has. \| 6 \| #11686, #8942 \|
	125	\| (d) release \| standby-reconcile, a host timer every 2 min that applies the forge's manifests from Forgejo; Flux's Kustomization for the forge is suspended \| pg-grand \| A changed manifest is not applied; the running pod is untouched. \| 6 \| #8948, #11686 \|
	126	\| (d) release \| Host timers forge-watchdog, forge-github-mirror and forge-prune-images \| pg-grand \| The watchdog, the GitHub mirror and image pruning stop. \| 6 \| #11686 \|
	127	\| (e) back up \| The forge's own backup lane (src/snapshot): the nightly snapshot and the hourly off-site pass, run in the forge's process, every object sealed under FORGE_SNAPSHOT_PASSPHRASE before it leaves; the nightly lands on the forge's own disk (FORGE_SNAPSHOT_DIR on its volume), not the platform's MinIO \| vxrail \| No new backup is taken; the sealed copies wait off both machines. \| — \| — \|
	128	\| (e) back up \| The off-site lane's bucket (FORGE_MIRROR_S3_*, or the mirror sealed on the Backups page), on neither machine \| off-site \| Neither machine holds it: losing either leaves every sealed copy in place. \| — \| — \|
128	129
129	130	If pg-grand were switched off now, the forge breaks in this order (the Breaks column):
130	131
		@@ -133,10 +134,9 @@ If pg-grand were switched off now, the forge breaks in this order (the Breaks co
133	134	3. the signup gate: no one signs up
134	135	4. image pulls and releases
135	136	5. the control plane: nothing reschedules, Services stop updating
136		6. local backups
137		7. the host timers
	137	6. the host timers
138	138
139		Also on pg-grand, not in the forge's path: Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and MinIO.
	139	Also on pg-grand, not in the forge's path: Forgejo and its database, the platform's mcp/plat-auth and www, the CloudNativePG operator with the tenant apps' databases, their local-path volumes pinned to that node, and the platform's backup CronJobs (the backup namespace, FORGE_BACKUP_LANES) with the MinIO they write to; the forge's own backup no longer needs them.
140	140	<!-- footing:end -->
141	141
142	142	The same list is read from the live cluster on `/system`, under "What the
		@@ -35,10 +35,10 @@ A machine with `git`, `bun` and a clone of the forge (`bun install`).
35	35
36	36	## Route 1 — the restore command
37	37
38		bun run src/snapshot/restore.ts <snapshot-name\|latest> <data-dir> [--clones N]
	38	bun run src/snapshot/restore.ts <snapshot-name\|latest> <data-dir> [--clones N] [--offsite]
39	39
40		`<data-dir>` must be empty or new. `--clones N` (1–32) sets how many
41		repositories clone at once (default: cores less one, 2–16).
	40	`<data-dir>` must be empty or new. `--clones N` (1–32): repositories cloned at once
	41	(default: cores less one, 2–16). `--offsite`: from the sealed lane, [features/offsite-lane.md](features/offsite-lane.md).
42	42
43	43	Then boot the forge on that directory, on a side port, off the public one:
44	44
		@@ -56,4 +56,9 @@ refuses `/data`, and leaves the restored copy in place.
56	56	`test/restore-drill-script.test.ts` runs it against a tiny snapshot and a
57	57	stub bucket.
58	58
	59	## From the off-site lane
	60
	61	The forge's own sealed backup lane (#11805), how to restore from it and how
	62	it is drilled: [features/offsite-lane.md](features/offsite-lane.md).
	63
59	64	With the platform unreachable (the B2 mirror only): [backup-truth.md](features/backup-truth.md), _Leaving the platform_.
		@@ -0,0 +1,57 @@
	1	# The off-site backup lane
	2
	3	The forge's own backup lane (#11805) is the mirror pass that runs in the
	4	forge's process every hour and after every snapshot. It copies the newest
	5	snapshots to a bucket on neither machine (`FORGE_MIRROR_S3_*`, or the mirror
	6	sealed on the Backups page), and every object it writes is sealed first:
	7	AES-256-GCM under a fresh IV, the key derived from `FORGE_SNAPSHOT_PASSPHRASE`
	8	by scrypt (`src/snapshot/sealed_store.ts`). Paths inside a snapshot are hidden
	9	(an HMAC of each path), so the bucket does not even name a repository.
	10
	11	- The salt is the lane's own. The first pass writes one unsealed header,
	12	`sealed/lane.json`: a random 16-byte salt and a check value (an HMAC under
	13	the derived key, never the key). A guess at the passphrase costs one scrypt
	14	for that lane alone, and a wrong passphrase is said as such before any
	15	object is read.
	16	- The key is in the passphrase alone: not in this repository, not on the
	17	bucket. Keep it with the operator, apart from the bucket's keys.
	18	- Without the passphrase the lane writes nothing. Every pass fails saying
	19	`FORGE_SNAPSHOT_PASSPHRASE is not set`, and `/system` shows the lane red;
	20	not one unsealed byte leaves the forge.
	21
	22	For the lane to stand without pg-grand, the nightly snapshot itself must land
	23	on the forge's own disk (`FORGE_SNAPSHOT_DIR` on its volume) rather than the
	24	platform's MinIO: the lane copies from there.
	25
	26	## Restore from it
	27
	28	On a fresh machine, follow [RESTORE.md](../RESTORE.md), exporting the lane's
	29	bucket in place of line 3, keeping line 4 (the passphrase), and adding
	30	`--offsite` to line 5:
	31
	32	export FORGE_MIRROR_S3_ENDPOINT=… FORGE_MIRROR_S3_BUCKET=… FORGE_MIRROR_S3_ACCESS_KEY=… FORGE_MIRROR_S3_SECRET_KEY=…
	33	bun run src/snapshot/restore.ts latest <data-dir> --offsite
	34
	35	`bun scripts/restore-drill.ts <empty-data-dir> --offsite` drills it the same
	36	way. Without the passphrase both refuse before writing anything; a restore
	37	never starts a lane on an empty bucket.
	38
	39	## The drill
	40
	41	The weekly drill, the Backups page's button and the MCP `restore_drill`
	42	restore the newest snapshot from the off-site lane when the forge has one
	43	(the destination otherwise) into a scratch forge, boot it, and check that
	44	`/api/v1/health` answers and `/` answers below 400. A drill waits for a mirror
	45	pass in flight, at most the drill's floor budget (10 min), then reads the lane
	46	as it stands. Each drill is kept with the store it restored from.
	47
	48	## Where to see it
	49
	50	- Web: `/system`, under _Off-site backup_: one sentence, and behind it
	51	where the lane writes, whether it is encrypted, its last success, and the
	52	last drill with where it restored from.
	53	- Terminal: the palette's Off-site backup, the same rows.
	54	- MCP: `system` with `{ kind: "offsite" }`, the same reading as
	55	`{ health, says, where, sealed, lastSuccess, newest, error, drill }`.
	56
	57	All three are a forge admin's alone.
		@@ -6,7 +6,10 @@
6	6	* writes only into the empty data directory it is given, and reads
7	7	* production through its public API, nothing else of the platform.
8	8	*
9		* bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh]
	9	* bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh] [--offsite]
	10	*
	11	* `--offsite` reads the sealed off-site lane (`FORGE_MIRROR_S3_*` and the
	12	* passphrase, #11805) in place of the destination.
10	13	*
11	14	* It restores, boots the restored forge on a free side port, checks `/`
12	15	* answers, counts the repositories and apps anyone can see there and on
		@@ -17,6 +20,7 @@
17	20	import { resolve } from "node:path";
18	21	import { bootRestored } from "../src/snapshot/boot.ts";
19	22	import { resolveDestination } from "../src/snapshot/destination.ts";
	23	import { offsiteFromEnv } from "../src/snapshot/mirror.ts";
20	24	import { publicCounts, type PublicCounts } from "../src/snapshot/public_counts.ts";
21	25	import { restoreSnapshot } from "../src/snapshot/restore.ts";
22	26	import { listSnapshots } from "../src/snapshot/snapshot.ts";
		@@ -28,8 +32,10 @@ const PRODUCTION_DATA = "/data";
28	32	const BOOT_MS = 60_000;
29	33
30	34	/** The arguments as plain data, or the reason they are not usable. */
31		export function parseArgs(args: readonly string[]): { dataDir: string; production: string } \| string {
	35	export function parseArgs(args: readonly string[]): { dataDir: string; production: string; offsite?: true } \| string {
32	36	const rest = [...args];
	37	const off = rest.indexOf("--offsite");
	38	if (off >= 0) rest.splice(off, 1);
33	39	let production = PRODUCTION;
34	40	const at = rest.indexOf("--production");
35	41	if (at >= 0) {
		@@ -46,10 +52,10 @@ export function parseArgs(args: readonly string[]): { dataDir: string; productio
46	52	if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback))
47	53	return "--production takes an https URL";
48	54	if (rest.length !== 1 \|\| !rest[0] \|\| rest[0].startsWith("-"))
49		return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL]";
	55	return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL] [--offsite]";
50	56	const dataDir = resolve(rest[0]);
51	57	if (dataDir === PRODUCTION_DATA) return `${dataDir} is production's data directory; restore into the box's own`;
52		return { dataDir, production: url.origin };
	58	return { dataDir, production: url.origin, ...(off >= 0 ? { offsite: true as const } : {}) };
53	59	}
54	60
55	61	/** The two sides' counts as lines a person reads, and which differ. */
		@@ -69,7 +75,12 @@ async function main(): Promise<number> {
69	75	console.error(args);
70	76	return 2;
71	77	}
72		const store = resolveDestination()?.store;
	78	const lane = args.offsite ? offsiteFromEnv() : undefined;
	79	if (typeof lane === "string") {
	80	console.error(lane);
	81	return 2;
	82	}
	83	const store = lane ?? resolveDestination()?.store;
73	84	if (store === undefined) {
74	85	console.error("no store: export the S3_* lines of the restore card, or FORGE_SNAPSHOT_DIR");
75	86	return 2;
		@@ -4,7 +4,7 @@ import { good, bad, type Tool, mayBoards } from "../shared.ts";
4	4
5	5	export const tool: Tool = {
6	6	name: "restore_drill",
7		doc: "no input: restore the newest snapshot as a drill and answer `{ verdict, failedStep?, says }`, worded as the Backups page's button; no store, a failure or a red drill is an answer, not an error; a forge admin's, with an admin token",
	7	doc: "no input: restore the newest snapshot as a drill and answer `{ verdict, failedStep?, says, from?, offSite? }` (from: the store it restored, the off-site lane when there is one), worded as the Backups page's button; no store, a failure or a red drill is an answer, not an error; a forge admin's, with an admin token",
8	8	description: "Drill-restore the newest snapshot.",
9	9	inputSchema: { type: "object", properties: {} },
10	10	writes: true,
		@@ -18,10 +18,12 @@ export const tool: Tool = {
18	18	return good({ verdict: "none", says: "This forge has no snapshots to drill." });
19	19	try {
20	20	const d = await svc.drill();
	21	// Where it restored from (the off-site lane when the forge has one, #11805), never a key.
	22	const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true };
21	23	const answer =
22	24	d.verdict === "green"
23		? good({ verdict: "green", says: "The drill restored green." })
24		: good({ verdict: "red", failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}.` });
	25	? good({ verdict: "green", says: "The drill restored green.", ...from })
	26	: good({ verdict: "red", failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}.`, ...from });
25	27	return { ...answer, card: { at: d.at } };
26	28	} catch (error) {
27	29	return good({ verdict: "none", says: `The drill could not run: ${String(error)}` });
		@@ -8,14 +8,15 @@ import { lagMinutes, loopWatch, perfRecord } from "../../../perf/live.ts";
8	8	import { LAG_BAR_MS, overBar } from "../../../perf/lag_minutes.ts";
9	9	import { slowestRoutes } from "../../../perf/watch.ts";
10	10	import { footing, riskLines } from "../../../system/footing.ts";
	11	import { laneWords, readOffsiteLane } from "../../../snapshot/offsite_lane.ts";
11	12	import { good, bad, type Tool } from "../shared.ts";
12	13
13	14	export const tool: Tool = {
14	15	name: "system",
15		doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast, `{ kind: \"routes\" }`: the 10 slowest budgeted routes as {route, p95Ms, budgetMs, n, slowestPhase}, over budget first; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only. `{ kind: \"footing\" }`: what the forge stands on (#11685), as /system's \"What the forge stands on\": `forgeNode`, and per piece `{ id, need, what, machine, seen, risk, ifDies, issues }` read from the live cluster, with `risks`, one line per piece not on the forge's machine; a forge admin only",
	16	doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast, `{ kind: \"routes\" }`: the 10 slowest budgeted routes as {route, p95Ms, budgetMs, n, slowestPhase}, over budget first; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only. `{ kind: \"footing\" }`: what the forge stands on (#11685), as /system's \"What the forge stands on\": `forgeNode`, and per piece `{ id, need, what, machine, seen, risk, ifDies, issues }` read from the live cluster, with `risks`, one line per piece not on the forge's machine; a forge admin only. `{ kind: \"offsite\" }`: the off-site backup lane (#11805), as /system's \"Off-site backup\" and the terminal's: `{ health, says, where, sealed, lastSuccess, newest, error, drill: { at, verdict, from, offSite, reason } \| null }`; a forge admin only",
16	17	description:
17	18	"The forge machine now: CPU, memory, disk, network. Admin or telemetry:read; `kind: csp`: 24 h of CSP violations (admin).",
18		inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp", "routes", "footing"] } } },
	19	inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp", "routes", "footing", "offsite"] } } },
19	20	writes: false,
20	21	async run(ctx, user, args) {
21	22	if (args["kind"] !== undefined) {
		@@ -30,7 +31,12 @@ export const tool: Tool = {
30	31	const f = await footing.read();
31	32	return good({ ...f, risks: riskLines(f) });
32	33	}
33		if (args["kind"] !== "csp") return bad("invalid", "kind is csp, routes, footing, or absent for the machine");
	34	if (args["kind"] === "offsite") {
	35	if (!user.isAdmin) return bad("forbidden", "only an admin of this forge reads its off-site backup lane");
	36	const lane = readOffsiteLane(ctx.app.config.dataDir, ctx.app.db);
	37	return good({ ...lane, says: laneWords(lane) });
	38	}
	39	if (args["kind"] !== "csp") return bad("invalid", "kind is csp, routes, footing, offsite, or absent for the machine");
34	40	if (!user.isAdmin) return bad("forbidden", "only an admin of this forge reads its CSP violations");
35	41	const now = new Date();
36	42	return good({ hours: 24, since: windowStart(24, now).toISOString(), directives: violationsByDirective(ctx.app.db, 24, now), scriptErrors: scriptErrorCounts(ctx.app.db, 24, now) });
		@@ -527,8 +527,14 @@ const snapshots = startSnapshots(
527	527	})?.store,
528	528	{
529	529	version: running().version,
	530	// The off-site lane (#11805): sealed under the passphrase before it leaves.
530	531	mirror: () =>
531		resolveMirror({ db: app.db, dataDir: app.config.dataDir, bus: app.bus })?.store,
	532	resolveMirror({
	533	db: app.db,
	534	dataDir: app.config.dataDir,
	535	bus: app.bus,
	536	passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] \|\| undefined,
	537	})?.store,
532	538	passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] \|\| undefined,
533	539	},
534	540	);
		@@ -3,7 +3,7 @@
3	3	* takes the newest snapshot, restores it into a fresh directory, opens the
4	4	* database read-only, counts what is in it, opens one sealed value with the
5	5	* restored key, boots the restored directory on a side port and reads
6		* `/api/v1/health`, and removes the directory whatever happened.
	6	* `/api/v1/health` and `/`, and removes the directory whatever happened.
7	7	*
8	8	* Plain data out: when it ran, which snapshot, how long, what it counted,
9	9	* and a verdict. A red verdict names the step that failed; a step that
		@@ -19,6 +19,7 @@ import { bootRestored } from "./boot.ts";
19	19	import { forgetKey, open } from "../kernel/secret.ts";
20	20	import { countRows, mismatched, type DrillCounts } from "./counts.ts";
21	21	import { restoreSnapshot, type Progress } from "./restore.ts";
	22	import { isOffBox } from "./off_box.ts";
22	23	import { listSnapshots } from "./snapshot.ts";
23	24	import type { Store } from "./store.ts";
24	25
		@@ -155,6 +156,17 @@ export interface DrillResult {
155	156	readonly budgetMs: number;
156	157	/** Why it is red, for the person reading; never a value. */
157	158	readonly reason: string \| null;
	159	/** The store it restored from (`Store.where`, never a key); absent on drills kept before #11805. */
	160	readonly from?: string \| undefined;
	161	/** Whether that store is off the forge's machine (`isOffBox`). */
	162	readonly offSite?: boolean \| undefined;
	163	}
	164
	165	/** The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or this throws. */
	166	export async function answersRoot(base: string): Promise<void> {
	167	const res = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
	168	await res.body?.cancel();
	169	if (res.status >= 400) throw new Error(`the restored forge answered / with ${res.status}`);
158	170	}
159	171
160	172	class StepTimeout extends Error {}
		@@ -276,7 +288,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
276	288	step = "health";
277	289	bootStart = performance.now();
278	290	const booted = await within(
279		(deps.boot ?? bootRestored)(target, left()),
	291	(deps.boot ?? bootRestored)(target, left(), undefined, answersRoot),
280	292	left(),
281	293	budget,
282	294	);
		@@ -323,6 +335,8 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
323	335	phases,
324	336	budgetMs,
325	337	reason,
	338	from: deps.store.where,
	339	offSite: isOffBox(deps.store),
326	340	};
327	341	}
328	342	}
		@@ -24,6 +24,17 @@ interface Row {
24	24
25	25	const PHASES = new Set(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
26	26
	27	/** Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is not ours. */
	28	export function parseFrom(raw: string \| null): { from?: string; offSite?: boolean } {
	29	try {
	30	const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown } \| null;
	31	if (typeof v?.from !== "string" \|\| v.from.length > 300) return {};
	32	return { from: v.from, offSite: v.offSite === true };
	33	} catch {
	34	return {};
	35	}
	36	}
	37
27	38	/** The stored JSON as phases; anything that is not a list of known phases with whole ms reads as none. */
28	39	export function parsePhases(raw: string \| null): { phases: PhaseTime[]; budgetMs: number } {
29	40	const none = { phases: [], budgetMs: 0 };
		@@ -61,7 +72,12 @@ export function recordDrill(db: Database, result: DrillResult): void {
61	72	result.verdict,
62	73	result.failedStep,
63	74	result.reason,
64		JSON.stringify({ phases: result.phases, budgetMs: result.budgetMs }),
	75	// Where it restored from rides the same column: no migration for two plain fields (#11805).
	76	JSON.stringify({
	77	phases: result.phases,
	78	budgetMs: result.budgetMs,
	79	...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }),
	80	}),
65	81	);
66	82	}
67	83
		@@ -91,5 +107,6 @@ export function lastDrill(db: Database): DrillResult \| undefined {
91	107	reason: row.reason,
92	108	phases: kept.phases,
93	109	budgetMs: kept.budgetMs,
	110	...parseFrom(row.phases),
94	111	};
95	112	}