Public: anyone can read this; the crew can change it.
The forge can be restored when the forge is gone: the bucket's credentials are not only inside the database being backed up #14246
Check · Review · Land · Live — Changes were asked on this head.
mason/forge:mason/14242-the-forge-can-be-restored-when-the-forge → main 4 commits fork 3 threads · 3 unresolved
docs/RECOVERY.md +2 −2
| @@ -35,10 +35,10 @@ A machine with `git`, `bun` and a clone of the forge (`bun install`). | ||
| 35 | 35 | |
| 36 | 36 | ## Route 1 — the restore command |
| 37 | 37 | |
| 38 | bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME] | |
| 38 | bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME] [--kit FILE] | |
| 39 | 39 | |
| 40 | 40 | `<data-dir>` must be empty or new. `--clones N` (1–32): clones at once (default: cores less one, 2–16). `--offsite`: [](features/offsite-lane.md). |
| 41 | `--forge NAME`: one hosted forge, [](features/hosted-backup.md). | |
| 41 | `--forge NAME`: one hosted forge, [](features/hosted-backup.md). `--kit FILE`: the bucket from the recovery kit (Backups page), [](RESTORE.md). | |
| 42 | 42 | |
| 43 | 43 | Then boot the forge on that directory, on a side port, off the public one: |
| 44 | 44 |
docs/RESTORE.md +22
| @@ -38,6 +38,28 @@ On a box that stays: `scripts/box-install.sh`, then lines 3–6 as the `forge` | ||
| 38 | 38 | user with `<data-dir>` `/var/lib/forge`, then `systemctl enable --now forge` |
| 39 | 39 | in place of line 7 ([](EXTRACT.md), _The box beside production_). |
| 40 | 40 | |
| 41 | ## When the forge is gone: the recovery kit | |
| 42 | ||
| 43 | The bucket's address and keys are sealed inside the forge's own database, so a lost machine | |
| 44 | takes them with it. The Backups page's **Recovery kit** keeps a copy that does not live there: | |
| 45 | one JSON file with the endpoint, bucket, a **read-only** key (made in the bucket's console and | |
| 46 | sealed on that page; the forge's own read-write key never goes in it), whether the snapshots | |
| 47 | are sealed, and the passphrase's fingerprint (never the passphrase). Download it, keep it off | |
| 48 | the machine, and keep the passphrase somewhere else again. The page says when it was last | |
| 49 | downloaded and whether it still matches the destination. | |
| 50 | ||
| 51 | On a clean machine with the release checkout and an empty data directory, the kit and the | |
| 52 | passphrase are all it takes: | |
| 53 | ||
| 54 | FORGE_SNAPSHOT_PASSPHRASE=<passphrase> bun run src/snapshot/restore.ts latest <data-dir> --kit <kit-file> | |
| 55 | FORGE_DATA=<data-dir> bun run src/main.ts | |
| 56 | ||
| 57 | A wrong passphrase is refused before the bucket is read. After each green inside drill the | |
| 58 | forge runs the same restore from the kit alone, in a fresh process with no `S3_*`, no `FORGE_*` | |
| 59 | and no data directory of its own, then boots what came back: the **outside drill**. The | |
| 60 | Backups verdict is not green while only the inside drill has passed, and says which modes have | |
| 61 | (`restore drill … passed, inside and outside`). `test/recovery-kit.test.ts` runs both. | |
| 62 | ||
| 41 | 63 | ## The drill on the box |
| 42 | 64 | |
| 43 | 65 | `scripts/restore-drill.ts` runs lines 3–7 and checks them, from the bucket |
src/actions/kit.ts added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | /** | |
| 2 | * The recovery kit, as an admin asks for it (#14242): save the bucket's read-only key, and take the | |
| 3 | * kit to keep off the machine. Nothing sealed is read back to a page; the kit is the one place a | |
| 4 | * key leaves, and only the read-only one, and only to a person who is a site admin. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { App } from "../app.ts"; | |
| 8 | import { putSecret } from "./secrets.ts"; | |
| 9 | import { mayChangeDestination } from "./destination.ts"; | |
| 10 | import { readOnlyTarget, SEALED_READ, testTarget } from "../snapshot/destination.ts"; | |
| 11 | import { kitIdentity, kitText, makeKit, recordDownload } from "../snapshot/kit.ts"; | |
| 12 | import type { User } from "../core/users.ts"; | |
| 13 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 14 | import { NOT_SEALED } from "../kernel/seal_words.ts"; | |
| 15 | ||
| 16 | const KEY = /^[\x21-\x7e]{1,256}$/; | |
| 17 | ||
| 18 | /** | |
| 19 | * Seal the read-only key pair, then say whether it lists the bucket (an error is `not_found` or a | |
| 20 | * sentence). | |
| 21 | */ | |
| 22 | export async function saveReadKey(app: App, actor: User | undefined, accessKey: string, secretKey: string): Promise<Result<string, string>> { | |
| 23 | if (!mayChangeDestination(actor)) return err("not_found"); | |
| 24 | if (!KEY.test(accessKey) || !KEY.test(secretKey)) return err("Both keys are needed: printable characters without spaces."); | |
| 25 | for (const [name, value] of [ | |
| 26 | [SEALED_READ.accessKey, accessKey], | |
| 27 | [SEALED_READ.secretKey, secretKey], | |
| 28 | ] as const) { | |
| 29 | if (!putSecret(app, actor, "forge", "", name, value).ok) return err(NOT_SEALED); | |
| 30 | } | |
| 31 | const target = readOnlyTarget({ db: app.db, dataDir: app.config.dataDir, bus: app.bus }); | |
| 32 | if (target === undefined) return ok("Read-only key sealed; it is used once the destination is sealed here too."); | |
| 33 | return ok(`Read-only key sealed. ${await testTarget(target)}`); | |
| 34 | } | |
| 35 | ||
| 36 | /** | |
| 37 | * The kit's file text and name, and the download recorded; an error is `not_found` or the sentence | |
| 38 | * on what is missing. | |
| 39 | */ | |
| 40 | export function takeKit(app: App, actor: User | undefined, passphrase: string | undefined): Result<{ name: string; text: string }, string> { | |
| 41 | if (!mayChangeDestination(actor)) return err("not_found"); | |
| 42 | const kit = makeKit({ db: app.db, dataDir: app.config.dataDir, bus: app.bus }, passphrase); | |
| 43 | if (typeof kit === "string") return err(`No kit: ${kit}.`); | |
| 44 | recordDownload(app.config.dataDir, kit); | |
| 45 | app.bus.append({ kind: "backup.kit_downloaded", actorId: actor!.id, payload: { identity: kitIdentity(kit) } }); | |
| 46 | return ok({ name: `forge-recovery-kit-${kit.madeAt.slice(0, 10)}.json`, text: kitText(kit) }); | |
| 47 | } |
src/http/route_marks.ts +2
| @@ -369,6 +369,8 @@ export const ROUTES: Readonly<Record<string, Mark>> = { | ||
| 369 | 369 | "/settings/backups/destination": "admin", |
| 370 | 370 | "/settings/backups/test": "admin", |
| 371 | 371 | "/settings/backups/clear": "admin", |
| 372 | "/settings/backups/kit": "admin", | |
| 373 | "/settings/backups/kit/key": "admin", | |
| 372 | 374 | "/settings/backups/mirror": "admin", |
| 373 | 375 | "/settings/backups/mirror/clear": "admin", |
| 374 | 376 | "/settings/backups/mirror/now": "admin", |
src/http/settings_routes.ts +3
| @@ -62,6 +62,7 @@ import { | ||
| 62 | 62 | backupsTest, |
| 63 | 63 | domainsMoved, |
| 64 | 64 | } from "../web/pages/backups.ts"; |
| 65 | import { backupsKit, backupsKitKey } from "../web/pages/backups_kit.ts"; | |
| 65 | 66 | import { |
| 66 | 67 | cloudAdd, |
| 67 | 68 | cloudAudit, |
| @@ -281,6 +282,8 @@ export function settingsRoutes(router: Router<Handler>, knownPatterns: () => rea | ||
| 281 | 282 | router.add("POST", "/settings/backups/destination", backupsDestination); |
| 282 | 283 | router.add("POST", "/settings/backups/test", backupsTest); |
| 283 | 284 | router.add("POST", "/settings/backups/clear", backupsClear); |
| 285 | router.add("POST", "/settings/backups/kit", backupsKit); | |
| 286 | router.add("POST", "/settings/backups/kit/key", backupsKitKey); | |
| 284 | 287 | router.add("POST", "/settings/backups/mirror", backupsMirror); |
| 285 | 288 | router.add("POST", "/settings/backups/mirror/clear", backupsMirrorClear); |
| 286 | 289 | router.add("POST", "/settings/backups/mirror/now", backupsMirrorNow); |
src/snapshot/bucket.ts +35 −2
| @@ -21,6 +21,7 @@ import { isOffBox } from "./off_box.ts"; | ||
| 21 | 21 | import { listSnapshots, notInSnapshot } from "./snapshot.ts"; |
| 22 | 22 | import type { HostedEntry } from "./hosted.ts"; |
| 23 | 23 | import { scrubReason } from "./destination.ts"; |
| 24 | import { lastOutside, type OutsideDrill } from "./kit.ts"; | |
| 24 | 25 | import { bodiesOf, type S3Target, type Store } from "./store.ts"; |
| 25 | 26 | |
| 26 | 27 | /** A newest snapshot older than this is red: the nightly is 24 h apart, plus a slow one's slack. */ |
| @@ -168,6 +169,18 @@ export function recordRun(dataDir: string, at: string, lanes: readonly LaneRead[ | ||
| 168 | 169 | const reads = new Map<string, { lanes: LaneRead[]; checkedAt: string; since: string }>(); |
| 169 | 170 | const watching = new Map<string, string>(); |
| 170 | 171 | |
| 172 | /** | |
| 173 | * The data directories whose forge owes an outside drill (#14242): a forge with a destination of | |
| 174 | * its own. Only such a forge is held from green until a restore from the recovery kit has passed. | |
| 175 | */ | |
| 176 | const outsideOwed = new Set<string>(); | |
| 177 | export const owesOutsideDrill = (dataDir: string, owed = true): void => void (owed ? outsideOwed.add(dataDir) : outsideOwed.delete(dataDir)); | |
| 178 | ||
| 179 | /** What `judge` is told of the outside drill; absent for a forge that owes none. */ | |
| 180 | export interface OutsideSaid { | |
| 181 | readonly last: OutsideDrill | undefined; | |
| 182 | } | |
| 183 | ||
| 171 | 184 | /** Whether a bucket watch has been started for this data directory. */ |
| 172 | 185 | export const isWatched = (dataDir: string): boolean => watching.has(dataDir); |
| 173 | 186 | |
| @@ -273,6 +286,7 @@ export function judge( | ||
| 273 | 286 | drill: DrillResult | undefined, |
| 274 | 287 | now: number, |
| 275 | 288 | booted: number = now, |
| 289 | outside?: OutsideSaid, | |
| 276 | 290 | ): Omit<BackupVerdict, "runs" | "checkLine"> { |
| 277 | 291 | const reasons: string[] = []; |
| 278 | 292 | /** A few words for each reason, for a tile's note. */ |
| @@ -352,6 +366,7 @@ export function judge( | ||
| 352 | 366 | reasons.push("No place is given for snapshots; none is being taken."); |
| 353 | 367 | } |
| 354 | 368 | let drillSays: BackupVerdict["drill"] = null; |
| 369 | let outsideNote = false; | |
| 355 | 370 | if (drill !== undefined) { |
| 356 | 371 | const ok = drillCounts(drill); |
| 357 | 372 | let says = ok |
| @@ -364,6 +379,22 @@ export function judge( | ||
| 364 | 379 | if (ok) says = `${says} (${proofAge} ago)`; |
| 365 | 380 | // And the hosted forge it restored alone, by name: one of them has come back (#13819). |
| 366 | 381 | if (ok && drill.hosted !== undefined) says = `${says}, with hosted forge ${printable(drill.hosted.name)} (1 of ${drill.hosted.of})`; |
| 382 | // A forge that owes an outside drill is green only when the restore from the kit has passed | |
| 383 | // too, and says which modes have (#14242). | |
| 384 | let outsideUnproved = false; | |
| 385 | if (ok && outside !== undefined) { | |
| 386 | if (outside.last?.verdict === "green") says = `${says}, inside and outside`; | |
| 387 | else if (outside.last === undefined) { | |
| 388 | says = `${says}, inside only`; | |
| 389 | outsideUnproved = true; | |
| 390 | unknowns.push("Only the inside restore drill has passed: nothing has restored this forge from the recovery kit and the passphrase alone. Save a read-only key on the Backups page so the outside drill can run."); | |
| 391 | } else { | |
| 392 | says = `${says}, inside only; the outside drill failed`; | |
| 393 | causes.push("outside drill failed"); | |
| 394 | reasons.push(`The last outside restore drill (${outside.last.at.slice(0, 10)}) failed: ${printable(outside.last.reason ?? "no reason kept")}.`); | |
| 395 | } | |
| 396 | } | |
| 397 | outsideNote = outsideUnproved; | |
| 367 | 398 | drillSays = { at: drill.at, ok, says }; |
| 368 | 399 | if (!ok) { |
| 369 | 400 | causes.push(drill.verdict === "green" ? "drill restored nothing" : "drill failed"); |
| @@ -389,7 +420,9 @@ export function judge( | ||
| 389 | 420 | : unknown |
| 390 | 421 | ? checkedAt === null |
| 391 | 422 | ? "not checked yet" |
| 392 | : "no restore drill yet" | |
| 423 | : outsideNote | |
| 424 | ? "inside drill only" | |
| 425 | : "no restore drill yet" | |
| 393 | 426 | : `${sub} · drill passed`; |
| 394 | 427 | return { |
| 395 | 428 | state: red ? "red" : unknown ? "unknown" : "green", |
| @@ -407,7 +440,7 @@ export function judge( | ||
| 407 | 440 | /** The verdict now, from what the watch last read. */ |
| 408 | 441 | export function backupVerdict(dataDir: string, drill: DrillResult | undefined, now = Date.now()): BackupVerdict { |
| 409 | 442 | const booted = Date.parse(watching.get(dataDir) ?? "") || now; |
| 410 | const v = judge(reads.get(dataDir), drill, now, booted); | |
| 443 | const v = judge(reads.get(dataDir), drill, now, booted, outsideOwed.has(dataDir) ? { last: lastOutside(dataDir) } : undefined); | |
| 411 | 444 | const runs = runCounts.get(dataDir) ?? 0; |
| 412 | 445 | const checkLine = |
| 413 | 446 | v.checkedAt === null |
src/snapshot/destination.ts +26 −1
| @@ -41,6 +41,17 @@ export const SEALED = { | ||
| 41 | 41 | } as const; |
| 42 | 42 | export const SEALED_NAMES: readonly string[] = Object.values(SEALED); |
| 43 | 43 | |
| 44 | /** | |
| 45 | * The read-only key the recovery kit carries (`kit.ts`): a second key pair for the same bucket, | |
| 46 | * made in the bucket's own console and sealed here. The forge never writes with it, so the kit an | |
| 47 | * owner keeps off the machine cannot overwrite or delete a snapshot. | |
| 48 | */ | |
| 49 | export const SEALED_READ = { | |
| 50 | accessKey: "SNAPSHOT_S3_READ_ACCESS_KEY", | |
| 51 | secretKey: "SNAPSHOT_S3_READ_SECRET_KEY", | |
| 52 | } as const; | |
| 53 | const SEALED_READ_NAMES: readonly string[] = Object.values(SEALED_READ); | |
| 54 | ||
| 44 | 55 | export interface Sources { |
| 45 | 56 | readonly db?: Database | undefined; |
| 46 | 57 | readonly dataDir?: string | undefined; |
| @@ -69,6 +80,20 @@ export function sealedTarget(s: Sources): S3Target | undefined { | ||
| 69 | 80 | }; |
| 70 | 81 | } |
| 71 | 82 | |
| 83 | /** | |
| 84 | * The sealed destination with its read-only key in place of the read-write one; undefined when | |
| 85 | * either is not sealed. | |
| 86 | */ | |
| 87 | export function readOnlyTarget(s: Sources): S3Target | undefined { | |
| 88 | const target = sealedTarget(s); | |
| 89 | if (!target || !s.db || s.dataDir === undefined || !s.bus) return undefined; | |
| 90 | const v = resolveSecrets(s.db, s.dataDir, s.bus, SEALED_READ_NAMES, {}); | |
| 91 | const accessKeyId = v[SEALED_READ.accessKey]; | |
| 92 | const secretAccessKey = v[SEALED_READ.secretKey]; | |
| 93 | if (!accessKeyId || !secretAccessKey) return undefined; | |
| 94 | return { ...target, accessKeyId, secretAccessKey }; | |
| 95 | } | |
| 96 | ||
| 72 | 97 | export function resolveDestination(s: Sources = {}): Destination | undefined { |
| 73 | 98 | const env = s.env ?? process.env; |
| 74 | 99 | const sealed = sealedTarget(s); |
| @@ -139,7 +164,7 @@ export function sealedSet(db: Database): string[] { | ||
| 139 | 164 | |
| 140 | 165 | /** Forget the sealed destination; the environment's, if any, takes over. */ |
| 141 | 166 | export function clearSealed(db: Database): void { |
| 142 | for (const name of SEALED_NAMES) deleteSecret(db, "forge", "", name); | |
| 167 | for (const name of [...SEALED_NAMES, ...SEALED_READ_NAMES]) deleteSecret(db, "forge", "", name); | |
| 143 | 168 | } |
| 144 | 169 | |
| 145 | 170 | /** An error as one short sentence: the target's keys struck out, at most 200 characters. */ |
src/snapshot/drill_outside.ts added +89
| @@ -0,0 +1,89 @@ | ||
| 1 | /** | |
| 2 | * The drill's second mode (#14242), run from OUTSIDE the forge's data: the newest snapshot is | |
| 3 | * restored by `restore.ts --kit` in a fresh process that is given the kit file and the passphrase | |
| 4 | * and nothing else (no `S3_*`, no `FORGE_*`, no data directory, an empty home), into an empty | |
| 5 | * directory, and the restored forge is booted and asked for `/`. The inside drill (`drill.ts`) | |
| 6 | * proves the snapshot is good with the credentials the running forge reads from its own database; | |
| 7 | * this proves the disaster case, where that database is gone. | |
| 8 | * | |
| 9 | * A fresh process on the release's own checkout stands in for a container from the release image: | |
| 10 | * it shares the machine's disk and network, and nothing of the forge's environment or data. | |
| 11 | */ | |
| 12 | ||
| 13 | import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 14 | import { join } from "node:path"; | |
| 15 | import { bootRestored } from "./boot.ts"; | |
| 16 | import { answersRoot, DRILL_TIMEOUT_MS } from "./drill.ts"; | |
| 17 | import { kitIdentity, kitText, type Kit, type OutsideDrill } from "./kit.ts"; | |
| 18 | ||
| 19 | const RESTORE = join(import.meta.dir, "restore.ts"); | |
| 20 | ||
| 21 | export interface OutsideDeps { | |
| 22 | readonly dataDir: string; | |
| 23 | readonly kit: Kit; | |
| 24 | readonly passphrase: string; | |
| 25 | readonly budgetMs?: number | undefined; | |
| 26 | /** Test seam: the restore script the process runs. */ | |
| 27 | readonly script?: string | undefined; | |
| 28 | /** Test seam: boot a restored directory and say whether it is healthy. */ | |
| 29 | readonly boot?: typeof bootRestored | undefined; | |
| 30 | } | |
| 31 | ||
| 32 | /** Run it; the answer is always a record, red with the reason when anything went wrong. */ | |
| 33 | export async function runOutsideDrill(deps: OutsideDeps): Promise<OutsideDrill> { | |
| 34 | const began = performance.now(); | |
| 35 | const at = new Date().toISOString(); | |
| 36 | const identity = kitIdentity(deps.kit); | |
| 37 | const budget = deps.budgetMs ?? DRILL_TIMEOUT_MS * 3; | |
| 38 | const left = (): number => Math.max(1, budget - (performance.now() - began)); | |
| 39 | const done = (verdict: "green" | "red", snapshotId: string | null, reason: string | null): OutsideDrill => ({ | |
| 40 | at, | |
| 41 | verdict, | |
| 42 | snapshotId, | |
| 43 | ms: Math.round(performance.now() - began), | |
| 44 | reason, | |
| 45 | identity, | |
| 46 | }); | |
| 47 | const parent = join(deps.dataDir, "drill"); | |
| 48 | mkdirSync(parent, { recursive: true }); | |
| 49 | // `restore-` so a restart's sweep (`sweepDrills`) removes what a cut drill left. | |
| 50 | const scratch = mkdtempSync(join(parent, "restore-outside-")); | |
| 51 | let snapshotId: string | null = null; | |
| 52 | try { | |
| 53 | const kitFile = join(scratch, "kit.json"); | |
| 54 | writeFileSync(kitFile, kitText(deps.kit)); | |
| 55 | chmodSync(kitFile, 0o600); | |
| 56 | const home = join(scratch, "home"); | |
| 57 | const target = join(scratch, "data"); | |
| 58 | mkdirSync(home); | |
| 59 | const child = Bun.spawn([process.execPath, deps.script ?? RESTORE, "latest", target, "--kit", kitFile], { | |
| 60 | cwd: home, | |
| 61 | stdin: "ignore", | |
| 62 | stdout: "pipe", | |
| 63 | stderr: "pipe", | |
| 64 | env: { | |
| 65 | PATH: process.env["PATH"] ?? "", | |
| 66 | HOME: home, | |
| 67 | FORGE_SNAPSHOT_PASSPHRASE: deps.passphrase, | |
| 68 | ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}), | |
| 69 | }, | |
| 70 | }); | |
| 71 | const timer = setTimeout(() => child.kill("SIGKILL"), left()); | |
| 72 | const [out, err, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]).finally(() => | |
| 73 | clearTimeout(timer), | |
| 74 | ); | |
| 75 | snapshotId = /^restored (\S+) /m.exec(out)?.[1] ?? null; | |
| 76 | if (code !== 0) { | |
| 77 | const said = (err.trim() || out.trim()).split("\n").pop() ?? ""; | |
| 78 | // The kit's keys are never in what a restore prints; strike them anyway. | |
| 79 | const clean = said.split(deps.kit.secretAccessKey).join("…").split(deps.kit.accessKeyId).join("…"); | |
| 80 | return done("red", snapshotId, `restore from the kit exited ${code ?? "without a code"}: ${clean.replace(/\S*\/\S*/g, "<path>").slice(0, 200)}`); | |
| 81 | } | |
| 82 | const booted = await (deps.boot ?? bootRestored)(target, left(), undefined, answersRoot); | |
| 83 | return booted.ok ? done("green", snapshotId, null) : done("red", snapshotId, `the forge restored from the kit did not boot: ${booted.reason}`.slice(0, 300)); | |
| 84 | } catch (error) { | |
| 85 | return done("red", snapshotId, `the outside drill did not run: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)); | |
| 86 | } finally { | |
| 87 | rmSync(scratch, { recursive: true, force: true }); | |
| 88 | } | |
| 89 | } |
src/snapshot/kit.ts added +209
| @@ -0,0 +1,209 @@ | ||
| 1 | /** | |
| 2 | * The recovery kit (#14242): what a person needs, on a machine that has never seen this forge, to | |
| 3 | * bring it back from its bucket. The bucket's address and keys are sealed inside the database the | |
| 4 | * snapshots back up, so a lost machine takes them with it; the kit is the copy kept elsewhere. | |
| 5 | * | |
| 6 | * One JSON file: the endpoint, bucket and region, a READ-ONLY key for it (never the forge's own | |
| 7 | * read-write key), whether the snapshots are sealed, the passphrase's fingerprint (never the | |
| 8 | * passphrase) and the exact command. `restore.ts --kit <file>` reads it. Plain data in, plain | |
| 9 | * data out; the files kept beside it (when it was last downloaded, how the outside drill went) | |
| 10 | * live under `<data>/drill/`. | |
| 11 | */ | |
| 12 | ||
| 13 | import { scryptSync, timingSafeEqual, createHash } from "node:crypto"; | |
| 14 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 15 | import { join } from "node:path"; | |
| 16 | import { readOnlyTarget, sealsSnapshots, type Sources } from "./destination.ts"; | |
| 17 | import { sealedStore } from "./sealed_store.ts"; | |
| 18 | import { s3Store, type S3Target, type Store } from "./store.ts"; | |
| 19 | ||
| 20 | export interface Kit { | |
| 21 | readonly v: 1; | |
| 22 | readonly madeAt: string; | |
| 23 | readonly endpoint: string; | |
| 24 | readonly bucket: string; | |
| 25 | readonly region?: string | undefined; | |
| 26 | /** A read-only key pair for the bucket. */ | |
| 27 | readonly accessKeyId: string; | |
| 28 | readonly secretAccessKey: string; | |
| 29 | /** Whether the snapshots are sealed under the passphrase (`FORGE_SNAPSHOT_SEAL`). */ | |
| 30 | readonly sealed: boolean; | |
| 31 | /** | |
| 32 | * `fingerprintOf` the passphrase, so the right one can be told from a wrong one on a clean | |
| 33 | * machine. | |
| 34 | */ | |
| 35 | readonly passphraseFingerprint: string; | |
| 36 | readonly command: string; | |
| 37 | } | |
| 38 | ||
| 39 | export const KIT_COMMAND = "FORGE_SNAPSHOT_PASSPHRASE=<passphrase> bun run src/snapshot/restore.ts latest <empty-data-dir> --kit <this-file>"; | |
| 40 | ||
| 41 | /** | |
| 42 | * Eight bytes of a scrypt of the passphrase: it tells a typo from the passphrase and gives no | |
| 43 | * shortcut to it. | |
| 44 | */ | |
| 45 | export function fingerprintOf(passphrase: string): string { | |
| 46 | return scryptSync(passphrase, "forge recovery kit passphrase fingerprint", 8, { N: 1 << 15, r: 8, p: 1, maxmem: 64 * 1024 * 1024 }).toString("hex"); | |
| 47 | } | |
| 48 | ||
| 49 | /** The kit for this forge, or the sentence on what stands in the way. */ | |
| 50 | export function makeKit(s: Sources, passphrase: string | undefined, now: Date = new Date()): Kit | string { | |
| 51 | const target = readOnlyTarget(s); | |
| 52 | if (target === undefined) | |
| 53 | return "no read-only key is sealed for the bucket: make one in the bucket's console (read and list only) and save it on this page"; | |
| 54 | if (!passphrase) return "FORGE_SNAPSHOT_PASSPHRASE is not set, so there is no passphrase to fingerprint"; | |
| 55 | return { | |
| 56 | v: 1, | |
| 57 | madeAt: now.toISOString(), | |
| 58 | endpoint: target.endpoint, | |
| 59 | bucket: target.bucket, | |
| 60 | ...(target.region === undefined ? {} : { region: target.region }), | |
| 61 | accessKeyId: target.accessKeyId, | |
| 62 | secretAccessKey: target.secretAccessKey, | |
| 63 | sealed: sealsSnapshots(s.env), | |
| 64 | passphraseFingerprint: fingerprintOf(passphrase), | |
| 65 | command: KIT_COMMAND, | |
| 66 | }; | |
| 67 | } | |
| 68 | ||
| 69 | export const kitText = (kit: Kit): string => `${JSON.stringify(kit, null, 2)}\n`; | |
| 70 | ||
| 71 | /** | |
| 72 | * What the kit says the destination is, as a short stable name: changes when any part of it does. | |
| 73 | */ | |
| 74 | export function kitIdentity(kit: Kit): string { | |
| 75 | return createHash("sha256") | |
| 76 | .update([kit.endpoint, kit.bucket, kit.region ?? "", kit.accessKeyId, String(kit.sealed), kit.passphraseFingerprint].join("\n")) | |
| 77 | .digest("hex") | |
| 78 | .slice(0, 16); | |
| 79 | } | |
| 80 | ||
| 81 | const short = (v: unknown, max: number): v is string => typeof v === "string" && v.length > 0 && v.length <= max; | |
| 82 | ||
| 83 | /** A kit from a file's text, or the sentence on why it is not one. */ | |
| 84 | export function parseKit(text: string): Kit | string { | |
| 85 | let v: Record<string, unknown>; | |
| 86 | try { | |
| 87 | v = JSON.parse(text) as Record<string, unknown>; | |
| 88 | } catch { | |
| 89 | return "the kit is not JSON"; | |
| 90 | } | |
| 91 | if (typeof v !== "object" || v === null || v["v"] !== 1) return "the kit is not a recovery kit this forge wrote"; | |
| 92 | const region = v["region"]; | |
| 93 | if ( | |
| 94 | !short(v["endpoint"], 300) || | |
| 95 | !/^https?:\/\//.test(v["endpoint"]) || | |
| 96 | !short(v["bucket"], 63) || | |
| 97 | !short(v["accessKeyId"], 256) || | |
| 98 | !short(v["secretAccessKey"], 256) || | |
| 99 | typeof v["sealed"] !== "boolean" || | |
| 100 | typeof v["passphraseFingerprint"] !== "string" || | |
| 101 | !/^[0-9a-f]{16}$/.test(v["passphraseFingerprint"]) || | |
| 102 | (region !== undefined && !short(region, 32)) | |
| 103 | ) | |
| 104 | return "the kit is missing a field or one is malformed"; | |
| 105 | return { | |
| 106 | v: 1, | |
| 107 | madeAt: typeof v["madeAt"] === "string" ? v["madeAt"] : "", | |
| 108 | endpoint: v["endpoint"], | |
| 109 | bucket: v["bucket"], | |
| 110 | ...(region === undefined ? {} : { region: region as string }), | |
| 111 | accessKeyId: v["accessKeyId"], | |
| 112 | secretAccessKey: v["secretAccessKey"], | |
| 113 | sealed: v["sealed"], | |
| 114 | passphraseFingerprint: v["passphraseFingerprint"], | |
| 115 | command: KIT_COMMAND, | |
| 116 | }; | |
| 117 | } | |
| 118 | ||
| 119 | /** | |
| 120 | * The store a kit and a passphrase open, for a restore: the bucket with the read-only key, unsealed | |
| 121 | * by the passphrase when the snapshots are sealed. A passphrase that is not the one the kit | |
| 122 | * fingerprints is refused before the bucket is touched. | |
| 123 | */ | |
| 124 | export function kitStore(kit: Kit, passphrase: string | undefined): Store | string { | |
| 125 | if (!passphrase) return "the kit needs the passphrase: set FORGE_SNAPSHOT_PASSPHRASE"; | |
| 126 | const want = Buffer.from(kit.passphraseFingerprint, "hex"); | |
| 127 | const got = Buffer.from(fingerprintOf(passphrase), "hex"); | |
| 128 | if (want.length !== got.length || !timingSafeEqual(want, got)) return "that passphrase is not the one this kit was made with"; | |
| 129 | const target: S3Target = { | |
| 130 | endpoint: kit.endpoint, | |
| 131 | bucket: kit.bucket, | |
| 132 | accessKeyId: kit.accessKeyId, | |
| 133 | secretAccessKey: kit.secretAccessKey, | |
| 134 | region: kit.region, | |
| 135 | }; | |
| 136 | const store = s3Store(target); | |
| 137 | return kit.sealed ? sealedStore(store, passphrase, { create: false }) : store; | |
| 138 | } | |
| 139 | ||
| 140 | // What is kept beside the drills: when the kit was last downloaded, and the outside drill's word. | |
| 141 | const kept = (dataDir: string, name: string): string => join(dataDir, "drill", name); | |
| 142 | ||
| 143 | function readJson(path: string): Record<string, unknown> | undefined { | |
| 144 | try { | |
| 145 | const v = JSON.parse(readFileSync(path, "utf8")) as unknown; | |
| 146 | return typeof v === "object" && v !== null ? (v as Record<string, unknown>) : undefined; | |
| 147 | } catch { | |
| 148 | return undefined; | |
| 149 | } | |
| 150 | } | |
| 151 | ||
| 152 | function writeJson(path: string, value: unknown): void { | |
| 153 | mkdirSync(join(path, ".."), { recursive: true }); | |
| 154 | const tmp = `${path}.tmp`; | |
| 155 | writeFileSync(tmp, JSON.stringify(value)); | |
| 156 | renameSync(tmp, path); | |
| 157 | } | |
| 158 | ||
| 159 | export interface Downloaded { | |
| 160 | readonly at: string; | |
| 161 | readonly identity: string; | |
| 162 | } | |
| 163 | ||
| 164 | export function recordDownload(dataDir: string, kit: Kit, now: Date = new Date()): void { | |
| 165 | writeJson(kept(dataDir, "kit.json"), { at: now.toISOString(), identity: kitIdentity(kit) }); | |
| 166 | } | |
| 167 | ||
| 168 | export function lastDownload(dataDir: string): Downloaded | undefined { | |
| 169 | const v = readJson(kept(dataDir, "kit.json")); | |
| 170 | return typeof v?.["at"] === "string" && typeof v["identity"] === "string" ? { at: v["at"], identity: v["identity"] } : undefined; | |
| 171 | } | |
| 172 | ||
| 173 | /** The outside drill's word: a restore from the kit and the passphrase alone, then a boot. */ | |
| 174 | export interface OutsideDrill { | |
| 175 | readonly at: string; | |
| 176 | readonly verdict: "green" | "red"; | |
| 177 | readonly snapshotId: string | null; | |
| 178 | readonly ms: number; | |
| 179 | /** Why it is red; never a value. */ | |
| 180 | readonly reason: string | null; | |
| 181 | /** The kit it used, `kitIdentity`. */ | |
| 182 | readonly identity: string; | |
| 183 | } | |
| 184 | ||
| 185 | export function recordOutside(dataDir: string, d: OutsideDrill): void { | |
| 186 | writeJson(kept(dataDir, "outside.json"), d); | |
| 187 | } | |
| 188 | ||
| 189 | export function lastOutside(dataDir: string): OutsideDrill | undefined { | |
| 190 | const v = readJson(kept(dataDir, "outside.json")); | |
| 191 | if (v === undefined || typeof v["at"] !== "string" || (v["verdict"] !== "green" && v["verdict"] !== "red")) return undefined; | |
| 192 | return { | |
| 193 | at: v["at"], | |
| 194 | verdict: v["verdict"], | |
| 195 | snapshotId: typeof v["snapshotId"] === "string" ? v["snapshotId"] : null, | |
| 196 | ms: typeof v["ms"] === "number" ? v["ms"] : 0, | |
| 197 | reason: typeof v["reason"] === "string" ? v["reason"].slice(0, 300) : null, | |
| 198 | identity: typeof v["identity"] === "string" ? v["identity"] : "", | |
| 199 | }; | |
| 200 | } | |
| 201 | ||
| 202 | /** What the Backups page says of the kit, as plain words. */ | |
| 203 | export function kitWords(kit: Kit | string, downloaded: Downloaded | undefined): { state: "ok" | "stale" | "never" | "none"; says: string } { | |
| 204 | if (typeof kit === "string") return { state: "none", says: `No kit can be made: ${kit}.` }; | |
| 205 | if (downloaded === undefined) return { state: "never", says: "The kit has never been downloaded. Download it and keep it off this machine." }; | |
| 206 | if (downloaded.identity !== kitIdentity(kit)) | |
| 207 | return { state: "stale", says: `The kit downloaded ${downloaded.at} no longer matches the destination, key or passphrase. Download a new one and replace the old.` }; | |
| 208 | return { state: "ok", says: `The kit downloaded ${downloaded.at} still matches the destination.` }; | |
| 209 | } |
src/snapshot/restore.ts +28 −3
| @@ -8,7 +8,10 @@ | ||
| 8 | 8 | * It refuses a data directory that is not empty: a restore over a live |
| 9 | 9 | * forge is not a restore, it is a mess. |
| 10 | 10 | * |
| 11 | * bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] | |
| 11 | * bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--kit FILE] | |
| 12 | * | |
| 13 | * `--kit FILE` reads the bucket from the recovery kit (`kit.ts`) in place of `S3_*`: on a clean | |
| 14 | * machine the kit and `FORGE_SNAPSHOT_PASSPHRASE` are all it needs. | |
| 12 | 15 | * |
| 13 | 16 | * `--clones N` (1–32) sets how many repositories clone at once; the |
| 14 | 17 | * default follows the machine's cores. |
| @@ -61,6 +64,7 @@ import { unwrapKey, WRAPPED_KEY } from "./wrap.ts"; | ||
| 61 | 64 | import { isPackedFile, partPaths, safePath, unpacker, type PackedFile, type Unpacker } from "./parts.ts"; |
| 62 | 65 | import { resolveSnapshots } from "./destination.ts"; |
| 63 | 66 | import { offsiteFromEnv } from "./mirror.ts"; |
| 67 | import { kitStore, parseKit } from "./kit.ts"; | |
| 64 | 68 | import type { Store } from "./store.ts"; |
| 65 | 69 | |
| 66 | 70 | export interface Restored { |
| @@ -717,14 +721,24 @@ if (import.meta.main) { | ||
| 717 | 721 | const only = args.indexOf("--forge"); |
| 718 | 722 | const forgeName = only >= 0 ? args[only + 1] : undefined; |
| 719 | 723 | if (only >= 0) args.splice(only, 2); |
| 724 | // `--kit FILE`: the recovery kit (`kit.ts`) in place of the environment's store, so a clean | |
| 725 | // machine needs only the kit and the passphrase (#14242). | |
| 726 | const kitAt = args.indexOf("--kit"); | |
| 727 | const kitFile = kitAt >= 0 ? args[kitAt + 1] : undefined; | |
| 728 | if (kitAt >= 0) args.splice(kitAt, 2); | |
| 720 | 729 | const [name, dataDir] = args; |
| 721 | 730 | if (!name || !dataDir || args.length > 2 || (only >= 0 && !forgeName)) { |
| 722 | 731 | // CLI usage on stderr for the person at the terminal, not a forge event. |
| 723 | 732 | console.error( |
| 724 | "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME]", | |
| 733 | "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME] [--kit FILE]", | |
| 725 | 734 | ); |
| 726 | 735 | process.exit(2); |
| 727 | 736 | } |
| 737 | if (kitAt >= 0 && (!kitFile || offsite >= 0)) { | |
| 738 | // CLI usage on stderr for the person at the terminal. | |
| 739 | console.error("--kit takes the kit's file, and does not combine with --offsite"); | |
| 740 | process.exit(2); | |
| 741 | } | |
| 728 | 742 | // A restore starts from an empty directory, so nothing is sealed yet: the |
| 729 | 743 | // same resolver answers from the environment alone. |
| 730 | 744 | const lane = offsite >= 0 ? offsiteFromEnv() : undefined; |
| @@ -733,7 +747,18 @@ if (import.meta.main) { | ||
| 733 | 747 | console.error(lane); |
| 734 | 748 | process.exit(2); |
| 735 | 749 | } |
| 736 | const store = lane ?? resolveSnapshots({ create: false })?.store; | |
| 750 | let kitStored: Store | undefined; | |
| 751 | if (kitFile !== undefined) { | |
| 752 | const kit = parseKit(await Bun.file(kitFile).text().catch(() => "not readable")); | |
| 753 | const opened = typeof kit === "string" ? kit : kitStore(kit, process.env["FORGE_SNAPSHOT_PASSPHRASE"]); | |
| 754 | if (typeof opened === "string") { | |
| 755 | // CLI message on stderr for the person at the terminal. | |
| 756 | console.error(`kit: ${opened}`); | |
| 757 | process.exit(2); | |
| 758 | } | |
| 759 | kitStored = opened; | |
| 760 | } | |
| 761 | const store = kitStored ?? lane ?? resolveSnapshots({ create: false })?.store; | |
| 737 | 762 | if (store === undefined) { |
| 738 | 763 | // CLI message on stderr for the person at the terminal. |
| 739 | 764 | console.error( |
src/snapshot/service.ts +32 −3
| @@ -28,7 +28,9 @@ import { | ||
| 28 | 28 | import { describeTimes, DRILL_TIMEOUT_MS, runDrill, type DrillDeps, type DrillResult } from "./drill.ts"; |
| 29 | 29 | import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS, snapshotDue } from "./due.ts"; |
| 30 | 30 | import { sealedTarget } from "./destination.ts"; |
| 31 | import { watchBucket } from "./bucket.ts"; | |
| 31 | import { owesOutsideDrill, watchBucket } from "./bucket.ts"; | |
| 32 | import { makeKit, recordOutside, type Kit, type OutsideDrill } from "./kit.ts"; | |
| 33 | import { runOutsideDrill } from "./drill_outside.ts"; | |
| 32 | 34 | import { readDeclaredLanes } from "./lanes.ts"; |
| 33 | 35 | import { lastDrill, recordDrill } from "./drills.ts"; |
| 34 | 36 | import { cutDrill, runDrillInChild, sweepDrills, type DrillState } from "./drill_job.ts"; |
| @@ -171,6 +173,8 @@ export function startSnapshots( | ||
| 171 | 173 | readonly drillWaitMs?: number | undefined; |
| 172 | 174 | /** Test seam: the drill itself. */ |
| 173 | 175 | readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined; |
| 176 | /** Test seam: the outside drill (`drill_outside.ts`). */ | |
| 177 | readonly outsideDrill?: ((kit: Kit, passphrase: string) => Promise<OutsideDrill>) | undefined; | |
| 174 | 178 | /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */ |
| 175 | 179 | readonly catchUpMs?: number | undefined; |
| 176 | 180 | readonly redRetryMs?: number | undefined; |
| @@ -451,6 +455,25 @@ export function startSnapshots( | ||
| 451 | 455 | drill().catch((error) => log.error("restore drill failed to run", { error: String(error) })); |
| 452 | 456 | return { started: true, state: drillNow ?? drillWaiting }; |
| 453 | 457 | }; |
| 458 | /** | |
| 459 | * The drill's second mode (#14242), after an inside one passes: a restore from the recovery kit | |
| 460 | * and the passphrase alone. Without a read-only key there is no kit, and it says why in the log. | |
| 461 | */ | |
| 462 | const outside = async (): Promise<void> => { | |
| 463 | const passphrase = options.passphrase; | |
| 464 | const kit = makeKit({ db: app.db, dataDir: app.config.dataDir, bus: app.bus }, passphrase); | |
| 465 | if (typeof kit === "string" || !passphrase) { | |
| 466 | log.info("outside restore drill not run", { why: typeof kit === "string" ? kit : "no passphrase" }); | |
| 467 | return; | |
| 468 | } | |
| 469 | try { | |
| 470 | const said = await (options.outsideDrill ?? ((k, p) => runOutsideDrill({ dataDir: app.config.dataDir, kit: k, passphrase: p })))(kit, passphrase); | |
| 471 | recordOutside(app.config.dataDir, said); | |
| 472 | log.info("outside restore drill", { verdict: said.verdict, ...(said.reason === null ? {} : { reason: said.reason }) }); | |
| 473 | } catch (error) { | |
| 474 | log.warn("outside restore drill failed to run", { error: String(error) }); | |
| 475 | } | |
| 476 | }; | |
| 454 | 477 | const drill = (): Promise<DrillResult> => { |
| 455 | 478 | drilling ??= (async (): Promise<DrillResult> => { |
| 456 | 479 | await (inFlight ?? Promise.resolve()).catch(() => {}); |
| @@ -493,6 +516,7 @@ export function startSnapshots( | ||
| 493 | 516 | }), |
| 494 | 517 | ); |
| 495 | 518 | recordDrill(app.db, result); |
| 519 | if (result.verdict === "green") await outside(); | |
| 496 | 520 | // The card and the recovery issue hear of every drill, whoever asked. |
| 497 | 521 | const posted = await reportVerdict(app.config.dataDir, result); |
| 498 | 522 | app.bus.append({ kind: "drill.finished", payload: { verdict: result.verdict, step: result.failedStep } }); |
| @@ -639,10 +663,14 @@ export function startSnapshots( | ||
| 639 | 663 | // The bucket is read by this process, on its own clock, whatever any job says. |
| 640 | 664 | const bucket = watchBucket( |
| 641 | 665 | app.config.dataDir, |
| 642 | () => [ | |
| 666 | () => { | |
| 667 | // A forge with a destination owes the outside drill; the verdict reads this (#14242). | |
| 668 | owesOutsideDrill(app.config.dataDir, current() !== undefined); | |
| 669 | return [ | |
| 643 | 670 | { lane: "primary", store: current() }, |
| 644 | 671 | { lane: "mirror", store: currentMirror() }, |
| 645 | ], | |
| 672 | ]; | |
| 673 | }, | |
| 646 | 674 | options.bucketEveryMs, |
| 647 | 675 | options.bucketFirstMs, |
| 648 | 676 | () => { |
| @@ -697,6 +725,7 @@ export function startSnapshots( | ||
| 697 | 725 | if (mirrorTimer !== undefined) clearTimeout(mirrorTimer); |
| 698 | 726 | if (catchUpSnapshot !== undefined) clearTimeout(catchUpSnapshot); |
| 699 | 727 | bucket.stop(); |
| 728 | owesOutsideDrill(app.config.dataDir, false); | |
| 700 | 729 | if (mirroring !== undefined) await mirroring.catch(() => {}); |
| 701 | 730 | if (inFlight !== undefined) await inFlight.catch(() => {}); |
| 702 | 731 | if (handles.get(app) === handle) handles.delete(app); |
src/web/pages/backups.ts +2 −1
| @@ -49,6 +49,7 @@ import { liveRegionsJs } from "../client.ts"; | ||
| 49 | 49 | import { flashTo, fragment, page, redirect } from "../layout.ts"; |
| 50 | 50 | import { badge, btn, fileSize, icon, row, time } from "../ui.ts"; |
| 51 | 51 | import { accountSections } from "./origins.ts"; |
| 52 | import { kitCard } from "./backups_kit.ts"; | |
| 52 | 53 | |
| 53 | 54 | function requireAdmin(ctx: Ctx): Response | undefined { |
| 54 | 55 | if (!ctx.auth.user) return signInFirst(ctx); |
| @@ -275,7 +276,7 @@ export async function backupsPage(ctx: Ctx): Promise<Response> { | ||
| 275 | 276 | if (denied) return denied; |
| 276 | 277 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 277 | 278 | const svc = snapshotsOf(ctx.app); |
| 278 | const form = html`${timePhase(ctx.request, "destination", () => destinationCard(ctx))}${timePhase( | |
| 279 | const form = html`${timePhase(ctx.request, "destination", () => destinationCard(ctx))}${timePhase(ctx.request, "kit", () => kitCard(ctx))}${timePhase( | |
| 279 | 280 | ctx.request, |
| 280 | 281 | "mirror", |
| 281 | 282 | () => mirrorCard(ctx), |
src/web/pages/backups_kit.ts added +82
| @@ -0,0 +1,82 @@ | ||
| 1 | /** | |
| 2 | * The recovery kit on the Backups page (#14242): whether a kit can be made, when it was last | |
| 3 | * downloaded and whether it still matches the destination, the outside drill's word, the form that | |
| 4 | * seals the bucket's read-only key, and the download. The kit holds a key, so it is a POST behind | |
| 5 | * the same-origin check, an attachment, and never cached. | |
| 6 | */ | |
| 7 | ||
| 8 | import { saveReadKey, takeKit } from "../../actions/kit.ts"; | |
| 9 | import { SEALED_READ, sealedSet } from "../../snapshot/destination.ts"; | |
| 10 | import { KIT_COMMAND, kitWords, lastDownload, lastOutside, makeKit } from "../../snapshot/kit.ts"; | |
| 11 | import type { Ctx } from "../../http/context.ts"; | |
| 12 | import { notFoundPage, signInFirst } from "../chrome.ts"; | |
| 13 | import { field, readForm } from "../form.ts"; | |
| 14 | import { html, type Html } from "../html.ts"; | |
| 15 | import { flashTo } from "../layout.ts"; | |
| 16 | import { badge, btn, time } from "../ui.ts"; | |
| 17 | ||
| 18 | const passphrase = (): string | undefined => process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined; | |
| 19 | ||
| 20 | function requireAdmin(ctx: Ctx): Response | undefined { | |
| 21 | if (!ctx.auth.user) return signInFirst(ctx); | |
| 22 | if (!ctx.auth.user.isAdmin) return notFoundPage(ctx); | |
| 23 | return undefined; | |
| 24 | } | |
| 25 | ||
| 26 | export function kitCard(ctx: Ctx): Html { | |
| 27 | const dataDir = ctx.app.config.dataDir; | |
| 28 | const sources = { db: ctx.app.db, dataDir, bus: ctx.app.bus }; | |
| 29 | const kit = makeKit(sources, passphrase()); | |
| 30 | const said = kitWords(kit, lastDownload(dataDir)); | |
| 31 | const outside = lastOutside(dataDir); | |
| 32 | const haveKey = sealedSet(ctx.app.db).includes(SEALED_READ.accessKey); | |
| 33 | const ok = said.state === "ok"; | |
| 34 | const drilled = | |
| 35 | outside === undefined | |
| 36 | ? html`<p class="break">${badge("Not proved", { tone: "danger", icon: "alert" })} The outside drill (a restore from the kit and the passphrase alone) has not passed yet; it runs after each inside drill once a kit can be made.</p>` | |
| 37 | : outside.verdict === "green" | |
| 38 | ? html`<p class="break">${badge("Ready", { tone: "success", icon: "check" })} Outside drill ${time(outside.at)}: the kit and the passphrase alone restored <code>${outside.snapshotId ?? ""}</code> and the forge booted.</p>` | |
| 39 | : html`<p class="break">${badge("At risk", { tone: "danger", icon: "alert" })} Outside drill ${time(outside.at)} failed: ${outside.reason ?? "no reason kept"}.</p>`; | |
| 40 | return html`<section class="settings-block" id="kit"> | |
| 41 | <h2>Recovery kit</h2> | |
| 42 | <p class="break">${ok ? badge("Ready", { tone: "success", icon: "check" }) : badge("At risk", { tone: "danger", icon: "alert" })} ${said.says}</p> | |
| 43 | <p class="small">If this machine or its disk is lost, the bucket's address and keys go with it. The kit is one file with the endpoint, the bucket, a read-only key and the passphrase's fingerprint (never the passphrase). Keep it off this machine, with the passphrase kept somewhere else. On a clean machine with the release's checkout: <code>${KIT_COMMAND}</code></p> | |
Principle 1 and 8: the card says what it is for three times (status line, this paragraph, the drill paragraph) and the paragraph is a wall of words with the command inline. One confident sentence, the rest a |
||
| 44 | ${drilled} | |
| 45 | <form method="post" action="/settings/backups/kit/key" class="stack"> | |
| 46 | <label class="field"><span>Read-only access key</span><input name="access_key" autocomplete="off" spellcheck="false" placeholder="${haveKey ? "set — enter both to replace" : ""}" /></label> | |
| 47 | <label class="field"><span>Read-only secret key</span><input name="secret_key" type="password" autocomplete="off" spellcheck="false" /></label> | |
| 48 | <div class="row-actions">${btn("Seal read-only key", { type: "submit", icon: "lock" })}</div> | |
| 49 | </form> | |
| 50 | <form method="post" action="/settings/backups/kit" class="mt"> | |
| 51 | ${btn("Download the kit", { type: "submit", variant: "primary", icon: "download" })} | |
Principle 5: a |
||
| 52 | </form> | |
| 53 | </section>`; | |
| 54 | } | |
| 55 | ||
| 56 | /** POST /settings/backups/kit/key — seal the read-only key and test it. */ | |
| 57 | export async function backupsKitKey(ctx: Ctx): Promise<Response> { | |
| 58 | const denied = requireAdmin(ctx); | |
| 59 | if (denied) return denied; | |
| 60 | const form = await readForm(ctx, "admin"); | |
| 61 | if (form instanceof Response) return form; | |
| 62 | const saved = await saveReadKey(ctx.app, ctx.auth.user, field(form, "access_key").trim(), field(form, "secret_key").trim()); | |
| 63 | return flashTo(ctx, "/settings/backups", saved.ok ? { message: saved.value } : { tone: "danger", message: saved.error }); | |
| 64 | } | |
| 65 | ||
| 66 | /** POST /settings/backups/kit — the kit as a file; the download is recorded. */ | |
| 67 | export async function backupsKit(ctx: Ctx): Promise<Response> { | |
| 68 | const denied = requireAdmin(ctx); | |
| 69 | if (denied) return denied; | |
| 70 | const form = await readForm(ctx, "admin"); | |
| 71 | if (form instanceof Response) return form; | |
| 72 | const kit = takeKit(ctx.app, ctx.auth.user, passphrase()); | |
| 73 | if (!kit.ok) return flashTo(ctx, "/settings/backups", { tone: "danger", message: kit.error }); | |
| 74 | return new Response(kit.value.text, { | |
| 75 | headers: { | |
| 76 | "content-type": "application/json; charset=utf-8", | |
| 77 | "content-disposition": `attachment; filename="${kit.value.name}"`, | |
| 78 | "cache-control": "no-store", | |
| 79 | "x-content-type-options": "nosniff", | |
| 80 | }, | |
| 81 | }); | |
| 82 | } |
test/recovery-kit.test.ts added +240
| @@ -0,0 +1,240 @@ | ||
| 1 | /** | |
| 2 | * The recovery kit (#14242): made only with a read-only key and a passphrase, read by | |
| 3 | * `restore.ts --kit` on a clean machine with nothing else, proved by the outside drill, and held | |
| 4 | * against a green verdict until that drill has passed. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { existsSync, mkdtempSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; | |
| 8 | import { rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join, relative } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 13 | import { saveDestination } from "../src/actions/destination.ts"; | |
| 14 | import { saveReadKey, takeKit } from "../src/actions/kit.ts"; | |
| 15 | import { createUser, type User } from "../src/core/users.ts"; | |
| 16 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 17 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; | |
| 18 | import { judge, type LaneRead } from "../src/snapshot/bucket.ts"; | |
| 19 | import { runOutsideDrill } from "../src/snapshot/drill_outside.ts"; | |
| 20 | import { fingerprintOf, kitIdentity, kitStore, kitText, kitWords, lastDownload, makeKit, parseKit, type Kit, type OutsideDrill } from "../src/snapshot/kit.ts"; | |
| 21 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 22 | ||
| 23 | const PASS = "correct horse battery staple"; | |
| 24 | const BUCKET = "kitbucket"; | |
| 25 | const ESC: Record<string, string> = { "&": "&", "<": "<", ">": ">" }; | |
| 26 | const xml = (s: string): string => s.replace(/[&<>]/g, (c) => ESC[c]!); | |
| 27 | ||
| 28 | let root: string; | |
| 29 | let app: App; | |
| 30 | let admin: User; | |
| 31 | let bucket: ReturnType<typeof Bun.serve>; | |
| 32 | ||
| 33 | function keysUnder(dir: string, out: string[] = []): string[] { | |
| 34 | for (const name of readdirSync(dir)) { | |
| 35 | const path = join(dir, name); | |
| 36 | if (statSync(path).isDirectory()) keysUnder(path, out); | |
| 37 | else out.push(path); | |
| 38 | } | |
| 39 | return out; | |
| 40 | } | |
| 41 | ||
| 42 | /** Path-style S3 over a directory: GET and HEAD of a key, and ListObjectsV2 with a delimiter. */ | |
| 43 | function stubBucket(dir: string): ReturnType<typeof Bun.serve> { | |
| 44 | return Bun.serve({ | |
| 45 | port: 0, | |
| 46 | hostname: "127.0.0.1", | |
| 47 | fetch(req) { | |
| 48 | const url = new URL(req.url); | |
| 49 | const path = decodeURIComponent(url.pathname); | |
| 50 | if (path === `/${BUCKET}` || path === `/${BUCKET}/`) { | |
| 51 | const prefix = url.searchParams.get("prefix") ?? ""; | |
| 52 | const delimiter = url.searchParams.get("delimiter"); | |
| 53 | const keys = keysUnder(dir).map((p) => relative(dir, p)).filter((k) => k.startsWith(prefix)).sort(); | |
| 54 | const contents: string[] = []; | |
| 55 | const prefixes = new Set<string>(); | |
| 56 | for (const key of keys) { | |
| 57 | const cut = delimiter ? key.indexOf(delimiter, prefix.length) : -1; | |
| 58 | if (cut >= 0) prefixes.add(key.slice(0, cut + 1)); | |
| 59 | else contents.push(key); | |
| 60 | } | |
| 61 | const body = | |
| 62 | `<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Name>${BUCKET}</Name><Prefix>${xml(prefix)}</Prefix>` + | |
| 63 | `<KeyCount>${contents.length + prefixes.size}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>` + | |
| 64 | contents.map((k) => `<Contents><Key>${xml(k)}</Key><Size>${statSync(join(dir, k)).size}</Size></Contents>`).join("") + | |
| 65 | [...prefixes].map((p) => `<CommonPrefixes><Prefix>${xml(p)}</Prefix></CommonPrefixes>`).join("") + | |
| 66 | `</ListBucketResult>`; | |
| 67 | return new Response(body, { headers: { "content-type": "application/xml" } }); | |
| 68 | } | |
| 69 | const key = path.slice(`/${BUCKET}/`.length); | |
| 70 | if (!path.startsWith(`/${BUCKET}/`) || key.split("/").includes("..")) return new Response(null, { status: 400 }); | |
| 71 | const file = join(dir, key); | |
| 72 | if (!existsSync(file)) return new Response(null, { status: 404 }); | |
| 73 | return new Response(req.method === "HEAD" ? null : readFileSync(file), { headers: { "content-length": String(statSync(file).size) } }); | |
| 74 | }, | |
| 75 | }); | |
| 76 | } | |
| 77 | ||
| 78 | const kitFor = (over: Partial<Kit> = {}): Kit => ({ | |
| 79 | v: 1, | |
| 80 | madeAt: "2026-11-16T00:00:00.000Z", | |
| 81 | endpoint: `http://127.0.0.1:${bucket.port}`, | |
| 82 | bucket: BUCKET, | |
| 83 | accessKeyId: "READKEY", | |
| 84 | secretAccessKey: "READSECRET", | |
| 85 | sealed: false, | |
| 86 | passphraseFingerprint: fingerprintOf(PASS), | |
| 87 | command: "", | |
| 88 | ...over, | |
| 89 | }); | |
| 90 | ||
| 91 | beforeAll(async () => { | |
| 92 | root = mkdtempSync(join(tmpdir(), "forge-kit-")); | |
| 93 | app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 94 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 95 | if (!u.ok) throw new Error(u.error); | |
| 96 | admin = u.value; | |
| 97 | const repo = await createRepo(app, admin, { name: "notes", visibility: "private" }); | |
| 98 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 99 | const c = await commitToBranch(app, admin, repo.value, { message: "first", files: [{ path: "README.md", content: "# notes\n" }] }); | |
| 100 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 101 | await takeSnapshot({ db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t", passphrase: PASS }, dirStore(join(root, "store")), new Date("2026-11-15T08:00:00Z")); | |
| 102 | bucket = stubBucket(join(root, "store")); | |
| 103 | }); | |
| 104 | ||
| 105 | afterAll(async () => { | |
| 106 | bucket.stop(true); | |
| 107 | app.close(); | |
| 108 | await rm(root, { recursive: true, force: true }); | |
| 109 | }); | |
| 110 | ||
| 111 | test("no kit is made without a read-only key or without the passphrase", async () => { | |
| 112 | const sources = { db: app.db, dataDir: app.config.dataDir, bus: app.bus }; | |
| 113 | expect(makeKit(sources, PASS)).toContain("no read-only key"); | |
| 114 | const dest = saveDestination(app, admin, { endpoint: `http://127.0.0.1:${bucket.port}`, bucket: BUCKET, region: "", accessKey: "RWKEY", secretKey: "RWSECRET" }); | |
| 115 | expect(dest.ok).toBe(true); | |
| 116 | expect(makeKit(sources, PASS)).toContain("no read-only key"); | |
| 117 | const saved = await saveReadKey(app, admin, "READKEY", "READSECRET"); | |
| 118 | expect(saved.ok).toBe(true); | |
| 119 | expect(makeKit(sources, undefined)).toContain("FORGE_SNAPSHOT_PASSPHRASE"); | |
| 120 | }); | |
| 121 | ||
| 122 | test("the kit carries the read-only key and the fingerprint, never the read-write key or the passphrase", () => { | |
| 123 | const sources = { db: app.db, dataDir: app.config.dataDir, bus: app.bus }; | |
| 124 | const kit = makeKit(sources, PASS); | |
| 125 | if (typeof kit === "string") throw new Error(kit); | |
| 126 | const text = kitText(kit); | |
| 127 | expect(text).toContain("READKEY"); | |
| 128 | expect(text).not.toContain("RWKEY"); | |
| 129 | expect(text).not.toContain("RWSECRET"); | |
| 130 | expect(text).not.toContain(PASS); | |
| 131 | expect(kit.passphraseFingerprint).toBe(fingerprintOf(PASS)); | |
| 132 | expect(parseKit(text)).toMatchObject({ bucket: BUCKET, accessKeyId: "READKEY" }); | |
| 133 | }); | |
| 134 | ||
| 135 | test("a person who is not an admin gets no kit, and a download is recorded and matches until the key changes", async () => { | |
| 136 | const stranger = { ...admin, isAdmin: false } as User; | |
| 137 | expect(takeKit(app, stranger, PASS)).toEqual({ ok: false, error: "not_found" }); | |
| 138 | const sources = { db: app.db, dataDir: app.config.dataDir, bus: app.bus }; | |
| 139 | const before = makeKit(sources, PASS); | |
| 140 | if (typeof before === "string") throw new Error(before); | |
| 141 | expect(kitWords(before, lastDownload(app.config.dataDir)).state).toBe("never"); | |
| 142 | const taken = takeKit(app, admin, PASS); | |
| 143 | expect(taken.ok).toBe(true); | |
| 144 | const after = makeKit(sources, PASS); | |
| 145 | if (typeof after === "string") throw new Error(after); | |
| 146 | expect(kitWords(after, lastDownload(app.config.dataDir)).state).toBe("ok"); | |
| 147 | await saveReadKey(app, admin, "READKEY2", "READSECRET2"); | |
| 148 | const changed = makeKit(sources, PASS); | |
| 149 | if (typeof changed === "string") throw new Error(changed); | |
| 150 | expect(kitWords(changed, lastDownload(app.config.dataDir)).state).toBe("stale"); | |
| 151 | }); | |
| 152 | ||
| 153 | test("a malformed kit and a wrong passphrase are refused", () => { | |
| 154 | expect(parseKit("{")).toBe("the kit is not JSON"); | |
| 155 | expect(parseKit(JSON.stringify({ v: 1, endpoint: "ftp://x" }))).toContain("malformed"); | |
| 156 | expect(kitStore(kitFor(), "not it")).toContain("not the one"); | |
| 157 | expect(kitStore(kitFor(), undefined)).toContain("passphrase"); | |
| 158 | }); | |
| 159 | ||
| 160 | const cleanEnv = (extra: Record<string, string>): Record<string, string> => ({ | |
| 161 | PATH: process.env["PATH"] ?? "", | |
| 162 | ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}), | |
| 163 | ...extra, | |
| 164 | }); | |
| 165 | ||
| 166 | test("restore.ts --kit restores the newest snapshot on a clean machine with only the kit and the passphrase", async () => { | |
| 167 | const kitFile = join(root, "kit.json"); | |
| 168 | writeFileSync(kitFile, kitText(kitFor())); | |
| 169 | const target = join(root, "clean-data"); | |
| 170 | const run = Bun.spawn([process.execPath, join(import.meta.dir, "../src/snapshot/restore.ts"), "latest", target, "--kit", kitFile], { | |
| 171 | cwd: root, | |
| 172 | env: cleanEnv({ FORGE_SNAPSHOT_PASSPHRASE: PASS }), | |
| 173 | stdout: "pipe", | |
| 174 | stderr: "pipe", | |
| 175 | }); | |
| 176 | const [out, err, code] = await Promise.all([new Response(run.stdout).text(), new Response(run.stderr).text(), run.exited]); | |
| 177 | expect({ code, err }).toEqual({ code: 0, err: "" }); | |
| 178 | expect(out).toContain("restored 2026-11-15T08-00-00Z"); | |
| 179 | expect(existsSync(join(target, "forge.db"))).toBe(true); | |
| 180 | }); | |
| 181 | ||
| 182 | test("restore.ts --kit with the wrong passphrase refuses before reading the bucket", async () => { | |
| 183 | const kitFile = join(root, "kit-wrong.json"); | |
| 184 | writeFileSync(kitFile, kitText(kitFor())); | |
| 185 | const run = Bun.spawn([process.execPath, join(import.meta.dir, "../src/snapshot/restore.ts"), "latest", join(root, "never"), "--kit", kitFile], { | |
| 186 | cwd: root, | |
| 187 | env: cleanEnv({ FORGE_SNAPSHOT_PASSPHRASE: "wrong" }), | |
| 188 | stdout: "pipe", | |
| 189 | stderr: "pipe", | |
| 190 | }); | |
| 191 | const [err, code] = await Promise.all([new Response(run.stderr).text(), run.exited]); | |
| 192 | expect(code).toBe(2); | |
| 193 | expect(err).toContain("not the one this kit was made with"); | |
| 194 | expect(existsSync(join(root, "never"))).toBe(false); | |
| 195 | }); | |
| 196 | ||
| 197 | test("the outside drill is green when the kit restores and the forge boots, red when it does not", async () => { | |
| 198 | const boot = async (): Promise<{ ok: boolean; reason: string }> => ({ ok: true, reason: "" }); | |
| 199 | const green = await runOutsideDrill({ dataDir: join(root, "live"), kit: kitFor(), passphrase: PASS, boot, budgetMs: 120_000 }); | |
| 200 | expect(green).toMatchObject({ verdict: "green", snapshotId: "2026-11-15T08-00-00Z", reason: null, identity: kitIdentity(kitFor()) }); | |
| 201 | const wrong = await runOutsideDrill({ dataDir: join(root, "live"), kit: kitFor(), passphrase: "wrong", boot, budgetMs: 120_000 }); | |
| 202 | expect(wrong.verdict).toBe("red"); | |
| 203 | expect(wrong.reason).toContain("not the one"); | |
| 204 | const down = await runOutsideDrill({ dataDir: join(root, "live"), kit: kitFor(), passphrase: PASS, boot: async () => ({ ok: false, reason: "no answer" }), budgetMs: 120_000 }); | |
| 205 | expect(down).toMatchObject({ verdict: "red", reason: expect.stringContaining("did not boot") }); | |
| 206 | expect(readdirSync(join(root, "live", "drill")).filter((n) => n.startsWith("restore-"))).toEqual([]); | |
| 207 | }); | |
| 208 | ||
| 209 | // The verdict: green only when the outside mode has passed too. | |
| 210 | const T = Date.parse("2026-11-16T12:00:00Z"); | |
| 211 | const lane = (): LaneRead => ({ lane: "primary", where: "s3/kit", offBox: true, newestAt: "2026-11-16T08:00:00Z", newestName: "2026-11-16T08-00-00Z", newestBytes: 1000, count: 3, readAt: "2026-11-16T11:55:00Z" }); | |
| 212 | const drill = (): DrillResult => ({ | |
| 213 | at: "2026-11-15T09:00:00Z", | |
| 214 | snapshotId: "x", | |
| 215 | ms: 5, | |
| 216 | counts: { repositories: 1, issues: 1, users: 1, pulls: 0, apps: 0 }, | |
| 217 | health: "ok", | |
| 218 | verdict: "green", | |
| 219 | failedStep: null, | |
| 220 | phases: [], | |
| 221 | budgetMs: 1, | |
| 222 | reason: null, | |
| 223 | }); | |
| 224 | const state = { lanes: [lane()], checkedAt: "2026-11-16T11:55:00Z" }; | |
| 225 | const outside = (verdict: "green" | "red"): OutsideDrill => ({ at: "2026-11-15T09:30:00Z", verdict, snapshotId: "x", ms: 5, reason: verdict === "red" ? "bad kit" : null, identity: "i" }); | |
| 226 | ||
| 227 | test("a forge that owes the outside drill is not green while only the inside mode has passed", () => { | |
| 228 | const inside = judge(state, drill(), T, T, { last: undefined }); | |
| 229 | expect(inside.state).toBe("unknown"); | |
| 230 | expect(inside.drill?.says).toContain("inside only"); | |
| 231 | expect(inside.unknowns.join(" ")).toContain("Only the inside restore drill has passed"); | |
| 232 | const both = judge(state, drill(), T, T, { last: outside("green") }); | |
| 233 | expect(both.state).toBe("green"); | |
| 234 | expect(both.drill?.says).toContain("inside and outside"); | |
| 235 | const failed = judge(state, drill(), T, T, { last: outside("red") }); | |
| 236 | expect(failed.state).toBe("red"); | |
| 237 | expect(failed.reasons.join(" ")).toContain("outside restore drill"); | |
| 238 | // A forge that owes none is judged as it always was. | |
| 239 | expect(judge(state, drill(), T, T).state).toBe("green"); | |
| 240 | }); |
test/settings-routes.test.ts +2
| @@ -79,6 +79,8 @@ POST /settings/backups/drill | ||
| 79 | 79 | POST /settings/backups/destination |
| 80 | 80 | POST /settings/backups/test |
| 81 | 81 | POST /settings/backups/clear |
| 82 | POST /settings/backups/kit | |
| 83 | POST /settings/backups/kit/key | |
| 82 | 84 | POST /settings/backups/mirror |
| 83 | 85 | POST /settings/backups/mirror/clear |
| 84 | 86 | POST /settings/backups/mirror/now |
test/spawn-pipes.test.ts +1
| @@ -44,6 +44,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 44 | 44 | "src/release/launch.ts": { sites: ["stderr"], reason: "hear() reads stderr to its end for the release's life, passing every chunk through to ours and keeping the tail until ready; stdout inherited" }, |
| 45 | 45 | "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" }, |
| 46 | 46 | "src/release/small_forge.ts": { sites: ["stderr", "stdout"], reason: "run() reads each git stderr to its end via Response.text() before the exit is awaited, stdout ignored; spawnSync buffers rev-parse stdout, stderr ignored" }, |
| 47 | "src/snapshot/drill_outside.ts": { sites: ["stdout+stderr"], reason: "the restore child's stdout is read to its end for the restored snapshot id and stderr for the failure text, alongside the exit" }, | |
| 47 | 48 | "src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the exit is read for a lost port; stdout ignored" }, |
| 48 | 49 | "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" }, |
| 49 | 50 | "src/checks/install.ts": { sites: ["stdout+stderr"], reason: "prefillInstallCache reads stdout and stderr to the end via Response.text() while the exit is awaited" }, |
test/spawn-sites.test.ts +2
| @@ -89,6 +89,8 @@ const SITES: Record<string, string> = { | ||
| 89 | 89 | "bun (process.execPath) on src/main.ts in the snapshot tree", |
| 90 | 90 | "src/snapshot/drill_job.ts": |
| 91 | 91 | "runs bun on the forge's own drill_child.ts with the job as JSON on stdin; argv holds only the bun path and a fixed script, nothing from a request", |
| 92 | "src/snapshot/drill_outside.ts": | |
| 93 | "bun (process.execPath) on the forge's own restore script with a fixed argv, the kit written to a 0600 file in a scratch directory it just made; the passphrase in the environment only, nothing from a request", | |
| 92 | 94 | "src/snapshot/hosted.ts": |
| 93 | 95 | "bun (process.execPath) on the forge's own hosted_copy.ts with an empty environment; its two arguments are a JSON job of three paths the snapshot built from a validated forge name and its own scratch directory, and the path its answer is written to: nothing from a request", |
| 94 | 96 | "src/spawn/archive.ts": |
test/timers-named.test.ts +1
| @@ -165,6 +165,7 @@ const ALLOWED: readonly Allowed[] = [ | ||
| 165 | 165 | ...allow("setTimeout", KILL, [ |
| 166 | 166 | ["src/checks/install.ts", "stop"], |
| 167 | 167 | ["src/snapshot/drill_job.ts", "() => { killed ="], |
| 168 | ["src/snapshot/drill_outside.ts", "() => child.kill(\"SIGKILL\")"], | |
| 168 | 169 | ["src/apps/process.ts", "() => { if (proc.exitCode === null"], |
| 169 | 170 | ["src/release/launch.ts", "() => child.kill(\"SIGKILL\")"], |
| 170 | 171 | ["src/release/checkout.ts", "() => proc.kill(\"SIGKILL\")"], |
Perf: makeKit runs scrypt and decrypts secrets on every page render. See finding 3.