Public: anyone can read this; the crew can change it.

Hosted forges are in no backup #13839

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13819-hosted-forges-are-in-no-backup → main 7 commits fork

  • Merged. scout/13819-hosted-forges-are-in-no-backup was merged into main as 8282ca7.

Review

Sign in to review this change.
14 files changed +674 −41
docs/RECOVERY.md +3 −3
@@ -35,10 +35,10 @@ A machine with `git`, `bun` and a clone of the forge (`bun install`).
3535
3636 ## Route 1 — the restore command
3737
38 bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite]
38 bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME]
3939
40 `<data-dir>` must be empty or new. `--clones N` (1–32): repositories cloned at once
41 (default: cores less one, 2–16). `--offsite`: from the sealed lane, [features/offsite-lane.md](features/offsite-lane.md).
40 `<data-dir>` must be empty or new. `--clones N` (1–32): clones at once (default: cores less one, 2–16). `--offsite`: [features/offsite-lane.md](features/offsite-lane.md).
41 `--forge NAME`: one hosted forge, [features/hosted-backup.md](features/hosted-backup.md).
4242
4343 Then boot the forge on that directory, on a side port, off the public one:
4444
docs/features/README.md +1
@@ -468,6 +468,7 @@ Read with the guide page [Backups and moving](../guide-backups-moving.md).
468468 - [Taking updates from an origin you follow](follow-an-origin.md)
469469 - [The backups line on Resilience](resilience-backups.md)
470470 - [A backup that finishes on a slow line](slow-line-backups.md)
471 - [Hosted forges in the nightly backup](hosted-backup.md)
471472 - [The off-site backup lane](offsite-lane.md)
472473 - [The operator guard and proposing through your fork](origin-guard.md)
473474 - [Which hosted forges are in use, and archiving one](forge-archive.md)
docs/features/hosted-backup.md added +28
@@ -0,0 +1,28 @@
1 # Hosted forges in the nightly backup
2
3 Each forge this one hosts goes into the nightly snapshot (#13819), under `children/<name>/`.
4
5 | What | Where in the snapshot |
6 | --- | --- |
7 | Database (a consistent copy, taken while it runs) | `forge.db` |
8 | Repositories | `repos/<owner>/<name>.bundle` |
9 | SSH host keys | `ssh/` |
10 | Its key, wrapped under `FORGE_SNAPSHOT_PASSPHRASE` | `secret.key.wrapped` (without a passphrase the key stays home) |
11 | Every app's production data | alongside |
12 | This forge's own record of it | `ledger.json` |
13
14 Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends nothing. A forge that cannot be read is named in the manifest's `hosted` list and the log; the snapshot goes on without it.
15
16 ## Where it shows
17
18 - Backups page: how many forges the newest backup kept, and how many were left out.
19 - Resilience screen (terminal): the same line, then one line per forge: backed up H h ago, or left out and why.
20 - `/forges`: each row, "backed up H h ago".
21
22 ## Restore one forge
23
24 Into an empty data directory; `children/<bodyId>/` and `spawn/<name>.json` are written there:
25
26 bun run src/snapshot/restore.ts latest <data-dir> --forge <name>
27
28 A whole restore of the parent leaves `children/` out; restore each forge this way.
src/snapshot/hosted.ts added +129
@@ -0,0 +1,129 @@
1 /**
2 * Hosted forges in the nightly snapshot (#13819). Each forge the parent hosts goes in under
3 * `children/<name>/`, laid out as its own data directory so one can be restored alone:
4 * `forge.db` (a consistent copy), `repos/<owner>/<name>.bundle`, `ssh/`, the key wrapped
5 * under the passphrase (`secret.key.wrapped`), every app's production data, and `ledger.json`,
6 * the parent's own record of it. The manifest's `hosted` lists each forge with when it was kept.
7 * Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends
8 * nothing.
9 */
10
11 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
12 import { join } from "node:path";
13 import { CHILD_NAME } from "../spawn/placement.ts";
14
15 /** One hosted forge as a snapshot kept it. */
16 export interface HostedEntry {
17 readonly name: string;
18 /** When its copy was taken. */
19 readonly at: string;
20 readonly repos: number;
21 readonly bytes: number;
22 /** `none` when no passphrase wrapped its key: its sealed values are lost on restore. */
23 readonly key: "wrapped" | "none";
24 /** Why it is not in this snapshot, when it is not. */
25 readonly why?: string;
26 }
27
28 export interface HostedSource {
29 readonly name: string;
30 readonly dir: string;
31 readonly ledger: string;
32 }
33
34 const BODY_ID = /^[A-Za-z0-9_-]{1,64}$/;
35
36 /** The data directory name a forge's ledger text names, or undefined when it names none. */
37 export function bodyIdOf(ledger: string): string | undefined {
38 try {
39 const v = JSON.parse(ledger) as { bodyId?: unknown };
40 return typeof v.bodyId === "string" && BODY_ID.test(v.bodyId) ? v.bodyId : undefined;
41 } catch {
42 return undefined;
43 }
44 }
45 export const HOSTED_PREFIX = "children/";
46
47 /** The forges the parent hosts that are finished and have a data directory, by name. */
48 export function hostedSources(dataDir: string): HostedSource[] {
49 let files: string[];
50 try {
51 files = readdirSync(join(dataDir, "spawn"));
52 } catch {
53 return [];
54 }
55 const out: HostedSource[] = [];
56 for (const f of files.sort()) {
57 if (!f.endsWith(".json")) continue;
58 const name = f.slice(0, -5);
59 if (!CHILD_NAME.test(name)) continue;
60 try {
61 const text = readFileSync(join(dataDir, "spawn", f), "utf8");
62 const v = JSON.parse(text) as { name?: unknown; done?: unknown };
63 const bodyId = bodyIdOf(text);
64 if (v.name !== name || v.done !== true || bodyId === undefined) continue;
65 const dir = join(dataDir, "children", bodyId);
66 if (existsSync(join(dir, "forge.db"))) out.push({ name, dir, ledger: text });
67 } catch {
68 // A ledger that does not read is not a forge to keep.
69 }
70 }
71 return out;
72 }
73
74 const hours = (ms: number): string => (ms < 3_600_000 ? "under an hour" : `${Math.floor(ms / 3_600_000)} h`);
75
76 /**
77 * The Backups line and the Resilience screen's head: how many hosted forges the newest snapshot
78 * kept. Each forge's age is `keptWords`.
79 */
80 export function hostedWords(hosted: readonly HostedEntry[] | undefined): string | undefined {
81 if (hosted === undefined) return undefined;
82 if (hosted.length === 0) return "No hosted forges to keep.";
83 const left = hosted.filter((h) => h.why !== undefined).length;
84 const kept = hosted.length - left;
85 return `${kept} hosted forge${kept === 1 ? "" : "s"} in the newest backup${left > 0 ? `; ${left} left out (see the log)` : ""}.`;
86 }
87
88 /** What /forges says beside one forge. */
89 export function keptWords(hosted: readonly HostedEntry[] | undefined, name: string, now: number): string {
90 if (hosted === undefined) return "not in a backup yet";
91 const h = hosted.find((e) => e.name === name);
92 if (h === undefined) return "not in the newest backup";
93 if (h.why !== undefined) return `left out of the newest backup: ${h.why}`;
94 return `backed up ${hours(now - Date.parse(h.at))} ago`;
95 }
96
97 const keptFile = (dataDir: string): string => join(dataDir, "snapshot", "hosted.json");
98
99 /**
100 * What the last whole snapshot kept of each hosted forge, held on disk so /forges reads no store.
101 */
102 export function noteHosted(dataDir: string, hosted: readonly HostedEntry[]): void {
103 mkdirSync(join(dataDir, "snapshot"), { recursive: true });
104 const tmp = `${keptFile(dataDir)}.tmp`;
105 writeFileSync(tmp, JSON.stringify(hosted));
106 renameSync(tmp, keptFile(dataDir));
107 }
108
109 /**
110 * The entries `noteHosted` kept; entries that do not read whole are dropped, a missing or corrupt
111 * file is undefined.
112 */
113 export function readHosted(dataDir: string): HostedEntry[] | undefined {
114 try {
115 const v: unknown = JSON.parse(readFileSync(keptFile(dataDir), "utf8"));
116 if (!Array.isArray(v)) return undefined;
117 const out: HostedEntry[] = [];
118 for (const h of v as Record<string, unknown>[]) {
119 if (typeof h?.["name"] !== "string" || typeof h["at"] !== "string") continue;
120 if (typeof h["repos"] !== "number" || typeof h["bytes"] !== "number") continue;
121 if (h["key"] !== "wrapped" && h["key"] !== "none") continue;
122 if (h["why"] !== undefined && typeof h["why"] !== "string") continue;
123 out.push({ name: h["name"], at: h["at"], repos: h["repos"], bytes: h["bytes"], key: h["key"], ...(h["why"] === undefined ? {} : { why: h["why"] }) });
124 }
125 return out;
126 } catch {
127 return undefined;
128 }
129 }
src/snapshot/restore.ts +111 −7
@@ -38,7 +38,7 @@
3838 * open is reported by name.
3939 */
4040
41 import { chmodSync, existsSync, mkdirSync, readdirSync } from "node:fs";
41 import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
4242 import { open as openFile, type FileHandle } from "node:fs/promises";
4343 import { availableParallelism } from "node:os";
4444 import { dirname, join } from "node:path";
@@ -55,6 +55,7 @@ import {
5555 type Manifest,
5656 type SnapshotFile,
5757 } from "./snapshot.ts";
58 import { bodyIdOf, HOSTED_PREFIX } from "./hosted.ts";
5859 import { chunkKey, isChunkId, openChunk } from "./chunks.ts";
5960 import { unwrapKey, WRAPPED_KEY } from "./wrap.ts";
6061 import { isPackedFile, partPaths, safePath, unpacker, type PackedFile, type Unpacker } from "./parts.ts";
@@ -233,7 +234,7 @@ export async function readWhole(store: Store, name: string, entry: SnapshotFile)
233234 * more than the reads in flight. Each chunk is proved against its name as
234235 * it is opened, each file against its size and hash as it closes.
235236 */
236 async function fetchChunked(
237 export async function fetchChunked(
237238 store: Store,
238239 name: string,
239240 entries: readonly SnapshotFile[],
@@ -392,10 +393,13 @@ export async function restoreSnapshot(
392393 const raw = await store.get(`${prefix}manifest.json`);
393394 if (raw === undefined)
394395 throw new Error(`no snapshot ${name} in ${store.where}`);
395 const manifest = JSON.parse(new TextDecoder().decode(raw)) as Manifest;
396 const parsed = JSON.parse(new TextDecoder().decode(raw)) as Manifest;
396397 say("manifest", 1, 1, raw.byteLength);
397 if (typeof manifest !== "object" || manifest === null || !Array.isArray(manifest.files))
398 if (typeof parsed !== "object" || parsed === null || !Array.isArray(parsed.files))
398399 throw new Error(`snapshot ${name}: its manifest does not read`);
400 // The hosted forges' files (`hosted.ts`) come back one forge at a time (`restoreHosted`), not
401 // into the parent's own data directory.
402 const manifest: Manifest = { ...parsed, files: parsed.files.filter((f) => typeof f.path !== "string" || !f.path.startsWith(HOSTED_PREFIX)) };
399403 const format: unknown = manifest.format;
400404 if (format !== undefined && format !== 2)
401405 throw new Error(`snapshot ${name} is of format ${String(format)}, which this forge does not read`);
@@ -573,7 +577,6 @@ export async function restoreSnapshot(
573577 );
574578 if (failed.length > 0)
575579 throw new Error(failed.sort((a, b) => a.at - b.at)[0]!.error);
576 const { rmSync } = await import("node:fs");
577580 rmSync(join(dataDir, "restore-bundles"), { recursive: true, force: true });
578581 say("key", 0, key === undefined ? 0 : 1, 0);
579582 if (key !== undefined) {
@@ -591,6 +594,90 @@ export async function restoreSnapshot(
591594 return { manifest, files, repos, key: have, sealedMissing };
592595 }
593596
597 /** What one hosted forge's restore brought back. */
598 export interface RestoredHosted {
599 readonly name: string;
600 /** Where its files were written. */
601 readonly dir: string;
602 /** The parent's ledger line for it, to be written to `<data>/spawn/<name>.json`. */
603 readonly ledger: string;
604 readonly files: number;
605 readonly repos: number;
606 readonly key: "restored" | "missing";
607 }
608
609 /**
610 * One hosted forge out of a snapshot, alone (#13819): its files written as its own data directory
611 * into `dir` (empty; a function of the ledger's bodyId when the name is the ledger's to give),
612 * its repositories cloned back from their bundles. The caller writes the ledger and starts it.
613 * Refuses, in a plain sentence, a snapshot that is missing or does not hold the forge.
614 */
615 export async function restoreHosted(
616 store: Store,
617 snapshot: string,
618 forge: string,
619 where: string | ((bodyId: string) => string),
620 options: { readonly passphrase?: string | undefined; readonly git?: typeof runGit | undefined } = {},
621 ): Promise<RestoredHosted> {
622 if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$/.test(snapshot)) throw new Error(`not a snapshot name: ${snapshot}`);
623 if (!/^[a-z0-9-]{1,40}$/.test(forge)) throw new Error(`not a forge name: ${forge}`);
624 const raw = await store.get(`${SNAPSHOT_PREFIX}${snapshot}/manifest.json`);
625 if (raw === undefined) throw new Error(`no snapshot ${snapshot} in ${store.where}`);
626 const manifest = JSON.parse(new TextDecoder().decode(raw)) as Manifest;
627 if (manifest.format !== 2 || !Array.isArray(manifest.files)) throw new Error(`snapshot ${snapshot} holds no hosted forges`);
628 const entry = manifest.hosted?.find((h) => h.name === forge);
629 if (entry === undefined || entry.why !== undefined) throw new Error(`snapshot ${snapshot} does not hold ${forge}${entry?.why === undefined ? "" : ` (${entry.why})`}`);
630 const prefix = `${HOSTED_PREFIX}${forge}/`;
631 const mine = manifest.files.filter((f) => f.path.startsWith(prefix)).map((f) => ({ ...f, path: f.path.slice(prefix.length) }));
632 const ledger = mine.find((f) => f.path === "ledger.json");
633 if (ledger === undefined || !mine.some((f) => f.path === "forge.db")) throw new Error(`snapshot ${snapshot}: ${forge} is not whole in it`);
634 const ledgerText = new TextDecoder().decode(await readWhole(store, snapshot, ledger));
635 const bodyId = bodyIdOf(ledgerText);
636 if (bodyId === undefined) throw new Error(`the ledger of ${forge} in ${snapshot} names no data directory`);
637 const dir = typeof where === "string" ? where : where(bodyId);
638 if (existsSync(dir) && readdirSync(dir).length > 0) throw new Error(`${dir} is not empty; a restore needs an empty data directory`);
639 let key: Buffer | undefined;
640 const keyEntry = mine.find((f) => f.path === WRAPPED_KEY);
641 if (keyEntry !== undefined && options.passphrase) {
642 key = unwrapKey(await readWhole(store, snapshot, keyEntry), options.passphrase);
643 if (key === undefined) throw new Error("the passphrase does not open this forge's wrapped key");
644 }
645 const files = mine.filter((f) => f.path !== "ledger.json" && f.path !== WRAPPED_KEY);
646 const got = await fetchChunked(store, snapshot, files, dir, () => {});
647 const git = options.git ?? runGit;
648 for (const b of got.bundles) {
649 const to = repoDir(join(dir, "repos"), b.owner, b.name);
650 mkdirSync(dirname(to), { recursive: true });
651 const cloned = await git(["clone", "--quiet", "--mirror", b.file, to]);
652 if (!cloned.ok) throw new Error(`could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`);
653 await git(["remote", "remove", "origin"], { cwd: to });
654 await git(["config", "core.logAllRefUpdates", "true"], { cwd: to });
655 await installHooks(to);
656 }
657 rmSync(join(dir, "restore-bundles"), { recursive: true, force: true });
658 if (key !== undefined) {
659 await Bun.write(join(dir, "secret.key"), key);
660 chmodSync(join(dir, "secret.key"), 0o600);
661 }
662 return { name: forge, dir, ledger: ledgerText, files: got.files, repos: got.bundles.length, key: key === undefined ? "missing" : "restored" };
663 }
664
665 /**
666 * The `--forge` restore: the forge into `<data>/children/<bodyId>`, its ledger into `<data>/spawn`.
667 */
668 export async function restoreHostedInto(
669 store: Store,
670 snapshot: string,
671 forge: string,
672 dataDir: string,
673 options: { readonly passphrase?: string | undefined; readonly git?: typeof runGit | undefined } = {},
674 ): Promise<RestoredHosted> {
675 const one = await restoreHosted(store, snapshot, forge, (id) => join(dataDir, "children", id), options);
676 mkdirSync(join(dataDir, "spawn"), { recursive: true });
677 await Bun.write(join(dataDir, "spawn", `${forge}.json`), one.ledger);
678 return one;
679 }
680
594681 if (import.meta.main) {
595682 const args = process.argv.slice(2);
596683 let clones: number | undefined;
@@ -607,11 +694,16 @@ if (import.meta.main) {
607694 // `--offsite`: read the sealed off-site lane (FORGE_MIRROR_S3_*) in place of the destination (#11805).
608695 const offsite = args.indexOf("--offsite");
609696 if (offsite >= 0) args.splice(offsite, 1);
697 // `--forge NAME`: one hosted forge alone, into <data-dir>/children/<bodyId> with its ledger under
698 // spawn/ (#13819).
699 const only = args.indexOf("--forge");
700 const forgeName = only >= 0 ? args[only + 1] : undefined;
701 if (only >= 0) args.splice(only, 2);
610702 const [name, dataDir] = args;
611 if (!name || !dataDir || args.length > 2) {
703 if (!name || !dataDir || args.length > 2 || (only >= 0 && !forgeName)) {
612704 // CLI usage on stderr for the person at the terminal, not a forge event.
613705 console.error(
614 "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite]",
706 "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME]",
615707 );
616708 process.exit(2);
617709 }
@@ -637,6 +729,18 @@ if (import.meta.main) {
637729 console.error(`no snapshot in ${store.where}`);
638730 process.exit(1);
639731 }
732 if (forgeName !== undefined) {
733 const snap = newest?.name ?? name;
734 try {
735 const one = await restoreHostedInto(store, snap, forgeName, dataDir, { passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined });
736 console.log(`restored ${forgeName} from ${snap}: ${one.files} files, ${one.repos} repositories into ${one.dir}${one.key === "missing" ? "; its key was not restored" : ""}`);
737 process.exit(0);
738 } catch (e) {
739 // CLI message on stderr for the person at the terminal.
740 console.error(e instanceof Error ? e.message : String(e));
741 process.exit(1);
742 }
743 }
640744 // One line per phase when it ends: its count, bytes and seconds.
641745 let began = performance.now();
642746 const done = await restoreSnapshot(store, newest?.name ?? name, dataDir, {
src/snapshot/service.ts +3
@@ -15,6 +15,7 @@ import { activeCrewRuns, crewHoldsCore, forgeCpus, holdClock } from "../core/one
1515 import { appendSnapshot } from "../core/ledger.ts";
1616 import { createLog } from "../kernel/log.ts";
1717 import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts";
18 import { noteHosted } from "./hosted.ts";
1819 import {
1920 KEEP,
2021 listSnapshots,
@@ -285,6 +286,8 @@ export function startSnapshots(
285286 running = undefined;
286287 startedAt = undefined;
287288 });
289 // /forges says when each hosted forge was kept (#13819).
290 if (manifest.hosted !== undefined) noteHosted(app.config.dataDir, manifest.hosted);
288291 // No snapshot is being taken now (runs are one after another), so the sweep may run every
289292 // time.
290293 await pruneSnapshots(store, KEEP, Date.now(), true);
src/snapshot/snapshot.ts +86 −28
@@ -46,6 +46,7 @@ import { listEveryRepo } from "../core/repos.ts";
4646 import { keepLentObjects, repoDir } from "../git/repos.ts";
4747 import { createLog } from "../kernel/log.ts";
4848 import { bundleRepo } from "./bundle.ts";
49 import { hostedSources, HOSTED_PREFIX, type HostedEntry } from "./hosted.ts";
4950 import { countRows, type DrillCounts } from "./counts.ts";
5051 import { CHUNK_PREFIX, chunkId, chunkKey, chunkSeed, contentChunker, fixedChunker, isChunkId, packChunk, type Chunker } from "./chunks.ts";
5152 import type { PackedFile } from "./parts.ts";
@@ -122,6 +123,8 @@ export interface Manifest {
122123 readonly absent?: readonly AbsentRepo[];
123124 /** Refs a bundle could not hold because their commits are missing, by repository. */
124125 readonly brokenRefs?: readonly BrokenRefs[];
126 /** The hosted forges this snapshot kept under `children/<name>/` (#13819). */
127 readonly hosted?: readonly HostedEntry[];
125128 /**
126129 * What this snapshot itself put in the store: chunks sent and their bytes as stored, and chunks
127130 * it found there.
@@ -594,35 +597,89 @@ export async function takeSnapshot(
594597
595598 // Every app's production data. SQLite inside is copied the consistent
596599 // way; its side-files are the live process's and not part of a copy.
597 const apps = listApps(forge.db);
598 let kept = 0;
599 for (const app of apps) {
600 const dir = envDataDir(forge.dataDir, app.id, "production");
601 const inside = filesUnder(dir);
602 if (inside.length === 0) continue;
603 kept++;
604 for (const f of inside) {
605 if (SIDE_FILE.test(f)) continue;
606 const source = join(dir, f);
607 if (SQLITE.test(f)) {
608 const copy = join(
609 scratch,
610 `app-${app.id}-${f.replace(/[\/\\]/g, "__")}`,
611 );
612 try {
613 await vacuumFile(source, copy);
614 await putFile(`apps/${app.id}/production/data/${f}`, copy, "fixed");
615 } catch (error) {
616 log.warn("app database not copied consistently; copied as a file", {
617 app: app.slug,
618 file: f,
619 error: String(error),
620 });
621 await putFile(`apps/${app.id}/production/data/${f}`, source, "fixed");
622 } finally {
623 rmSync(copy, { force: true });
600 // SQLite inside is copied the consistent way; side-files are the live process's.
601 const putAppData = async (apps: ReturnType<typeof listApps>, root: string, into: string, label: string): Promise<number> => {
602 let withData = 0;
603 for (const app of apps) {
604 const dir = envDataDir(root, app.id, "production");
605 const inside = filesUnder(dir);
606 if (inside.length === 0) continue;
607 withData++;
608 for (const f of inside) {
609 if (SIDE_FILE.test(f)) continue;
610 const source = join(dir, f);
611 const path = `${into}apps/${app.id}/production/data/${f}`;
612 if (SQLITE.test(f)) {
613 const copy = join(scratch, `app-${label}${app.id}-${f.replace(/[\/\\]/g, "__")}`);
614 try {
615 await vacuumFile(source, copy);
616 await putFile(path, copy, "fixed");
617 } catch (error) {
618 log.warn("app database not copied consistently; copied as a file", { app: `${label}${app.slug}`, file: f, error: String(error) });
619 await putFile(path, source, "fixed");
620 } finally {
621 rmSync(copy, { force: true });
622 }
623 } else await putFile(path, source);
624 }
625 }
626 return withData;
627 };
628 const kept = await putAppData(listApps(forge.db), forge.dataDir, "", "");
629
630 // Every hosted forge (#13819), each under its own prefix so one restores alone. A forge that
631 // cannot be read costs that forge, never the snapshot; the manifest says which.
632 const hosted: HostedEntry[] = [];
633 for (const child of hostedSources(forge.dataDir)) {
634 const into = `${HOSTED_PREFIX}${child.name}/`;
635 const from = files.length;
636 const copy = join(scratch, `hosted-${child.name}.db`);
637 let repoCount = 0;
638 let wrapped: HostedEntry["key"] = "none";
639 try {
640 await vacuumFile(join(child.dir, "forge.db"), copy);
641 const childDb = new Database(copy, { readonly: true });
642 let childRepos: ReturnType<typeof listEveryRepo>;
643 let childApps: ReturnType<typeof listApps>;
644 try {
645 childRepos = listEveryRepo(childDb);
646 childApps = listApps(childDb);
647 } finally {
648 childDb.close();
649 }
650 await putFile(`${into}forge.db`, copy, "fixed");
651 await putBytes(`${into}ledger.json`, new TextEncoder().encode(child.ledger));
652 const childKey = Bun.file(join(child.dir, "secret.key"));
653 if (forge.passphrase && (await childKey.exists())) {
654 const sealed = wrapKey(new Uint8Array(await childKey.arrayBuffer()), forge.passphrase);
655 if (unwrapKey(sealed, forge.passphrase) === undefined) throw new Error("its wrapped key does not open");
656 await putBytes(`${into}${WRAPPED_KEY}`, new TextEncoder().encode(sealed));
657 wrapped = "wrapped";
658 }
659 for (const f of filesUnder(join(child.dir, "ssh"))) await putFile(`${into}ssh/${f}`, join(child.dir, "ssh", f));
660 for (const repo of childRepos) {
661 const dir = repoDir(join(child.dir, "repos"), repo.ownerName, repo.name);
662 const bundle = join(scratch, `hosted-${child.name}-${repo.ownerName}--${repo.name}.bundle`);
663 await keepLentObjects(dir);
664 const made = await bundleRepo(dir, bundle, { deadlineMs: forge.bundleDeadlineMs, sizes });
665 if (!made.ok) {
666 rmSync(bundle, { force: true });
667 if (!made.empty) log.error("hosted repository left out of the snapshot", { forge: child.name, repo: `${repo.ownerName}/${repo.name}`, why: made.why });
668 continue;
624669 }
625 } else await putFile(`apps/${app.id}/production/data/${f}`, source);
670 await putFile(`${into}repos/${repo.ownerName}/${repo.name}.bundle`, bundle);
671 rmSync(bundle, { force: true });
672 repoCount++;
673 }
674 await putAppData(childApps, child.dir, into, `${child.name}/`);
675 hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: files.slice(from).reduce((n, f) => n + f.bytes, 0), key: wrapped });
676 } catch (error) {
677 // What went up before the failure stays listed in `files` and is harmless; the entry says
678 // why.
679 log.error("hosted forge left out of the snapshot", { forge: child.name, error: String(error) });
680 hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: 0, key: wrapped, why: String(error).slice(0, 200) });
681 } finally {
682 rmSync(copy, { force: true });
626683 }
627684 }
628685
@@ -691,6 +748,7 @@ export async function takeSnapshot(
691748 repositories,
692749 ...(absent.length === 0 ? {} : { absent }),
693750 ...(brokenRefs.length === 0 ? {} : { brokenRefs }),
751 hosted,
694752 sent: { chunks: landed, bytes: landedBytes, held: found },
695753 };
696754 // Announced only when whole: every chunk it names, seen in the store now.
src/tui/screens/resilience.ts +8
@@ -39,6 +39,7 @@ import { forgetMirror, saveMirror } from "../../actions/mirror.ts";
3939 import { JOBS } from "../../nightly/jobs.ts";
4040 import { outputTail } from "../../nightly/output.ts";
4141 import { nightlyLines } from "../../nightly/says.ts";
42 import { hostedWords, keptWords, readHosted } from "../../snapshot/hosted.ts";
4243 import { backupsLine } from "../../snapshot/backups_line.ts";
4344 import { backupVerdict } from "../../snapshot/bucket.ts";
4445 import { lastDrill } from "../../snapshot/drills.ts";
@@ -303,6 +304,13 @@ export function resilienceScreen(f: Session, ctx: Ctx): Screen {
303304 // middle.
304305 if (note !== "") out.push(...wrap(note, INNER), "");
305306 out.push(...wrap(backupsLine(f.app).says, INNER), ...verdictLines(f), "");
307 // Hosted forges in the nightly snapshot, as of the last one taken here (#13819).
308 const kept = readHosted(f.app.config.dataDir);
309 if (kept !== undefined) {
310 out.push(...wrap(hostedWords(kept) ?? "", INNER));
311 for (const h of kept) out.push(...wrap(` ${h.name}: ${keptWords(kept, h.name, Date.now())}`, INNER));
312 out.push("");
313 }
306314 for (const line of nightlyLines(f.app.config.dataDir)) out.push(...wrap(line, INNER));
307315 out.push("");
308316 out.push(...wrap(destinationLine(f), INNER), ...wrap(mirrorLine(f), INNER));
src/web/pages/backups.ts +3
@@ -30,6 +30,7 @@ import { costFacts } from "../../snapshot/drill_cost.ts";
3030 import { lastDrill } from "../../snapshot/drills.ts";
3131 import { backupVerdict } from "../../snapshot/bucket.ts";
3232 import { bodiesOf, type Store } from "../../snapshot/store.ts";
33 import { hostedWords } from "../../snapshot/hosted.ts";
3334 import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts";
3435 import { drillStatus, mirrorNow, snapshotNow, startDrillNow } from "../../actions/backups_now.ts";
3536 import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
@@ -346,6 +347,7 @@ async function movingPart(
346347 ...(d ? { drill: { at: d.at, verdict: d.verdict } } : {}),
347348 now,
348349 });
350 const hostedLine = hostedWords(snapshots[0]?.hosted);
349351 const restore = snapshots[0]
350352 ? html`<pre class="restore-command" tabindex="0" aria-label="Restore command">bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
351353 : null;
@@ -358,6 +360,7 @@ async function movingPart(
358360 fourteen.
359361 </p>
360362 ${keyNote(svc.wraps, snapshots[0])}
363 ${hostedLine === undefined ? null : html`<p data-hosted-kept>${hostedLine}</p>`}
361364 ${when(
362365 svc.progress(),
363366 (p) =>
src/web/pages/forges.ts +5 −2
@@ -32,6 +32,7 @@ import { claimLinkBody } from "./new-forge.ts";
3232 import { appLinkSummary, appLinkWords } from "../../spawn/app_links.ts";
3333 import { REACH_SETTINGS, reachByForge, type ForgeReach } from "../../actions/child_reach.ts";
3434 import { forgeReachRows } from "./hosted-forges.ts";
35 import { keptWords, readHosted } from "../../snapshot/hosted.ts";
3536 import { archiveForge, archivedForges, restoreForge, type ArchiveRecord } from "../../spawn/archive.ts";
3637 import { claimedWords, isForgeSort, keepWords, SORT_WORDS, sortForges, type ForgeSort } from "../../spawn/keep_words.ts";
3738 import { CHILD_NAME } from "../../spawn/placement.ts";
@@ -75,7 +76,7 @@ function seenWords(seen: Seen): Html {
7576
7677 const XLARGE_SAYS = `${sizeWords("xlarge")}. ${sizesView().sizes.find((x) => x.size === "xlarge")?.says ?? ""}`;
7778
78 export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = false): Html {
79 export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = false, kept?: string): Html {
7980 const state = claimState(r);
8081 const seen = seenOf(r);
8182 const meta = [
@@ -103,6 +104,7 @@ export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = f
103104 r.made && seen === null ? (r.lastActivityAt === null ? html`<span data-activity>no one has done anything here yet</span>` : html`<span data-activity>a person last did something ${time(r.lastActivityAt)}</span>`) : null,
104105 claimedWords(r) === null ? null : html`<span data-claimed-by>${claimedWords(r)}</span>`,
105106 keepWords(r) === null ? null : html`<span data-keep>${keepWords(r)}</span>`,
107 r.made && kept !== undefined ? html`<span data-backed-up>${kept}</span>` : null,
106108 r.noticedAt === null || r.noticedAt === undefined ? null : html`<span data-noticed>owner told ${time(r.noticedAt)}</span>`,
107109 badge(r.nextStep.key === "going" ? html`going: last tell ${time(r.nextStep.lastTellAt!)}` : r.nextStep.words, {
108110 variant: "outline",
@@ -239,6 +241,7 @@ export async function forgesPage(ctx: Ctx): Promise<Response> {
239241 // Two small tables, by name: the rows themselves come from memory, and this reads no ledger.
240242 const reachOf = timePhase(ctx.request, "reach", () => reachByForge(ctx.app, user));
241243 // Sizes that together pass the machine's memory are said in one line; nothing is refused for it.
244 const hostedKept = readHosted(ctx.app.config.dataDir);
242245 const over = memoryOverLine(forgesMemory(ctx.app, memoryOf(rows)));
243246 const rendering = performance.now();
244247 const action =
@@ -272,7 +275,7 @@ export async function forgesPage(ctx: Ctx): Promise<Response> {
272275 ${btn("Archive selected", { size: "sm", title: "Pick forges with the boxes; the next page says what happens before anything does" })}
273276 </form>
274277 <ul class="list setting-list">
275 ${rows.map((r) => forgeRow(r, Date.now(), reachOf.get(r.name), claimsBill(user, r.billTo)))}
278 ${rows.map((r) => forgeRow(r, Date.now(), reachOf.get(r.name), claimsBill(user, r.billTo), keptWords(hostedKept, r.name, Date.now())))}
276279 </ul>
277280 </section>`}
278281 ${sealedLine(ctx)}
test/client-list.test.ts +18
@@ -8,7 +8,9 @@
88 */
99
1010 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
11 import { rmSync } from "node:fs";
1112 import { mkdtemp, rm } from "node:fs/promises";
13 import { noteHosted } from "../src/snapshot/hosted.ts";
1214 import { tmpdir } from "node:os";
1315 import { join } from "node:path";
1416 import { createRepo } from "../src/actions/repos.ts";
@@ -230,6 +232,22 @@ describe("the client list", () => {
230232 expect(html).not.toContain(claimUrl.split("#")[1]!);
231233 });
232234
235 test("each row says when the newest backup kept its forge, or why not (#13819)", async () => {
236 const at = new Date(Date.now() - 5 * 3_600_000).toISOString();
237 noteHosted(dir, [
238 { name: "maria", at, repos: 1, bytes: 9, key: "wrapped" },
239 { name: "lou", at, repos: 0, bytes: 0, key: "none", why: "its database would not open" },
240 ]);
241 const html = (await (await list(admin)).text()).replace(/\s+/g, " ");
242 const maria = html.slice(html.indexOf('data-forge="maria"'));
243 expect(maria.slice(0, maria.indexOf("</li>"))).toContain("<span data-backed-up>backed up 5 h ago</span>");
244 const lou = html.slice(html.indexOf('data-forge="lou"'));
245 expect(lou.slice(0, lou.indexOf("</li>"))).toContain("left out of the newest backup: its database would not open");
246 rmSync(join(dir, "snapshot", "hosted.json"));
247 const none = (await (await list(admin)).text()).replace(/\s+/g, " ");
248 expect(none).toContain("not in a backup yet");
249 });
250
233251 test("the admin's page carries the forges side by side: columns, shared use, no heartbeat yet", async () => {
234252 const res = await list(admin);
235253 expect(res.status).toBe(200);
test/hosted-surfaces.test.ts added +86
@@ -0,0 +1,86 @@
1 /**
2 * Where a hosted forge's backup shows (#13819): the Backups page says how many the newest snapshot
3 * kept, and the Resilience screen says it again with one line per forge.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createUser, getUserByName } from "../src/core/users.ts";
11 import { serve, type Servers } from "../src/http/server.ts";
12 import { noteHosted } from "../src/snapshot/hosted.ts";
13 import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
14 import { takeSnapshot } from "../src/snapshot/snapshot.ts";
15 import { dirStore } from "../src/snapshot/store.ts";
16 import { createForgeApp } from "../src/tui/forge.ts";
17 import { headlessTerminal } from "../src/tui/term/terminal.ts";
18 import { quietLogs } from "./helpers/globals.ts";
19 import { settleApp, settlePalette, settleUntil } from "./helpers/tui.ts";
20
21 quietLogs();
22
23 let root = "";
24 let app: App;
25 let child: App;
26 let svc: SnapshotsHandle;
27 let servers: Servers;
28 let cookie = "";
29
30 beforeAll(async () => {
31 for (const k of Object.keys(process.env)) if (k.startsWith("S3_") || k === "FORGE_SNAPSHOT_DIR") delete process.env[k];
32 root = mkdtempSync(join(tmpdir(), "forge-hosted-surfaces-"));
33 app = createApp({ dataDir: root, port: 0 });
34 child = createApp({ dataDir: join(root, "children", "b1"), port: 0 });
35 mkdirSync(join(root, "spawn"), { recursive: true });
36 writeFileSync(join(root, "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "b1", done: true }));
37 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
38 if (!u.ok) throw new Error(u.error);
39 const store = dirStore(join(root, "store"));
40 await takeSnapshot({ db: app.db, dataDir: root, reposRoot: app.reposRoot, version: "t" }, store, new Date());
41 svc = startSnapshots(app, store, { version: "t", nightly: false });
42 servers = await serve(app);
43 const r = await fetch(`${servers.url}/signin`, {
44 method: "POST",
45 headers: { origin: servers.url },
46 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
47 redirect: "manual",
48 });
49 cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!;
50 });
51 afterAll(async () => {
52 servers.stop();
53 await svc.stop();
54 child.close();
55 app.close();
56 rmSync(root, { recursive: true, force: true });
57 });
58
59 test("the Backups page says how many hosted forges the newest snapshot kept", async () => {
60 const html = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
61 expect(html).toMatch(/<p data-hosted-kept>1 hosted forge in the newest backup\.<\/p>/);
62 });
63
64 test("the Resilience screen says it, and which forge is kept or left out", async () => {
65 const at = new Date(Date.now() - 2 * 3_600_000).toISOString();
66 noteHosted(root, [
67 { name: "kid", at, repos: 1, bytes: 9, key: "none" },
68 { name: "sad", at, repos: 0, bytes: 0, key: "none", why: "its database would not open" },
69 ]);
70 const terminal = headlessTerminal(140, 50);
71 const user = getUserByName(app.db, "alice")!;
72 const { app: tui } = createForgeApp(app, { sshUsername: "t", remoteAddress: "127.0.0.1", user }, { terminal, exitHooks: false });
73 tui.start();
74 const key = (k: string, ctrl = false) => tui.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
75 key("k", true);
76 await settleApp(tui);
77 for (const ch of "Resilience") key(ch);
78 await settlePalette(tui);
79 key("enter");
80 await settleUntil(tui, "hosted forge");
81 const text = tui.screen.toText();
82 tui.stop();
83 expect(text).toContain("1 hosted forge in the newest backup; 1 left out");
84 expect(text).toContain("kid: backed up 2 h ago");
85 expect(text).toContain("sad: left out of the newest backup: its database would not open");
86 });
test/snapshot-hosted.test.ts added +192
@@ -0,0 +1,192 @@
1 /**
2 * Hosted forges are in the nightly snapshot (#13819): a forge the parent hosts goes up under
3 * `children/<name>/`, restores alone, and the Backups and /forges words say when it was kept.
4 */
5 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
6 import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
7 import { rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
11 import { createApp, type App } from "../src/app.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { commitLog } from "../src/git/read.ts";
14 import { repoDir } from "../src/git/repos.ts";
15 import { bodyIdOf, hostedSources, hostedWords, keptWords, noteHosted, readHosted } from "../src/snapshot/hosted.ts";
16 import { restoreHosted, restoreHostedInto, restoreSnapshot } from "../src/snapshot/restore.ts";
17 import { takeSnapshot } from "../src/snapshot/snapshot.ts";
18 import { dirStore } from "../src/snapshot/store.ts";
19
20 const PASS = "correct horse battery staple";
21
22 describe("hosted forges in the snapshot", () => {
23 let root: string;
24 let parent: App;
25 let child: App;
26 const BODY = "body1";
27
28 beforeAll(async () => {
29 root = mkdtempSync(join(tmpdir(), "forge-hosted-snap-"));
30 parent = createApp({ dataDir: join(root, "parent"), port: 0 });
31 child = createApp({ dataDir: join(root, "parent", "children", BODY), port: 0 });
32 const u = await createUser(child.db, { origin: "seed", trusted: true, username: "glint", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
33 if (!u.ok) throw new Error(u.error);
34 const repo = await createRepo(child, u.value, { name: "work", visibility: "private" });
35 if (!repo.ok) throw new Error(String(repo.error));
36 const c = await commitToBranch(child, u.value, repo.value, { message: "hosted work", files: [{ path: "a.txt", content: "kept\n" }] });
37 if (!c.ok) throw new Error(JSON.stringify(c.error));
38 writeFileSync(join(child.config.dataDir, "secret.key"), "k".repeat(32), { mode: 0o600 });
39 mkdirSync(join(root, "parent", "spawn"), { recursive: true });
40 writeFileSync(join(root, "parent", "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: BODY, done: true }));
41 // Not finished, and not a forge name: neither is kept.
42 writeFileSync(join(root, "parent", "spawn", "half.json"), JSON.stringify({ name: "half", bodyId: BODY, done: false }));
43 writeFileSync(join(root, "parent", "spawn", "Bad_Name.json"), JSON.stringify({ name: "Bad_Name", bodyId: BODY, done: true }));
44 });
45
46 afterAll(async () => {
47 child.close();
48 parent.close();
49 await rm(root, { recursive: true, force: true });
50 });
51
52 const take = (passphrase?: string) => {
53 const store = dirStore(join(root, `store-${Math.random().toString(36).slice(2)}`));
54 return takeSnapshot({ db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase }, store, new Date("2026-11-16T03:00:00Z")).then((manifest) => ({ store, manifest }));
55 };
56
57 test("only finished forges with a database are named", () => {
58 expect(hostedSources(parent.config.dataDir).map((s) => s.name)).toEqual(["kid"]);
59 });
60
61 test("the snapshot keeps the forge's database, repository, key and ledger under its name", async () => {
62 const { manifest } = await take(PASS);
63 const paths = manifest.files.map((f) => f.path);
64 for (const p of ["children/kid/forge.db", "children/kid/ledger.json", "children/kid/secret.key.wrapped", "children/kid/repos/glint/work.bundle"])
65 expect(paths).toContain(p);
66 expect(manifest.hosted).toHaveLength(1);
67 expect(manifest.hosted?.[0]).toMatchObject({ name: "kid", repos: 1, key: "wrapped" });
68 expect(Number.isNaN(Date.parse(manifest.hosted![0]!.at))).toBe(false);
69 });
70
71 test("without a passphrase the key stays home and the entry says so", async () => {
72 const { manifest } = await take();
73 expect(manifest.hosted?.[0]?.key).toBe("none");
74 expect(manifest.files.some((f) => f.path.endsWith("secret.key.wrapped"))).toBe(false);
75 });
76
77 test("one forge restores alone, with its history and key, and opens", async () => {
78 const { store, manifest } = await take(PASS);
79 const dir = join(root, "alone");
80 const got = await restoreHosted(store, manifest.name, "kid", dir, { passphrase: PASS });
81 expect(got.repos).toBe(1);
82 expect(got.key).toBe("restored");
83 expect(JSON.parse(got.ledger).bodyId).toBe(BODY);
84 expect(readFileSync(join(dir, "secret.key"), "utf8")).toBe("k".repeat(32));
85 const log = await commitLog(repoDir(join(dir, "repos"), "glint", "work"), "refs/heads/main", { limit: 5 });
86 expect(log.ok && log.value.map((c) => c.subject)).toContain("hosted work");
87 const opened = createApp({ dataDir: dir, port: 0 });
88 try {
89 expect(opened.db.query("SELECT username FROM user").all()).toEqual([{ username: "glint" }]);
90 } finally {
91 opened.close();
92 }
93 });
94
95 test("restore refuses a forge the snapshot does not hold, and a wrong passphrase", async () => {
96 const { store, manifest } = await take(PASS);
97 await expect(restoreHosted(store, manifest.name, "nobody", join(root, "x1"), { passphrase: PASS })).rejects.toThrow("does not hold nobody");
98 await expect(restoreHosted(store, manifest.name, "kid", join(root, "x2"), { passphrase: "wrong" })).rejects.toThrow("passphrase");
99 });
100
101 test("restoring the parent whole leaves the hosted forges for their own restore", async () => {
102 const { store, manifest } = await take(PASS);
103 const dir = join(root, "whole");
104 await restoreSnapshot(store, manifest.name, dir, { passphrase: PASS });
105 expect(existsSync(join(dir, "children"))).toBe(false);
106 expect(existsSync(join(dir, "forge.db"))).toBe(true);
107 });
108
109 test("a second snapshot of an unchanged forge sends no new chunks for it", async () => {
110 const store = dirStore(join(root, "twice"));
111 const f = { db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase: PASS };
112 await takeSnapshot(f, store, new Date("2026-11-16T03:00:00Z"));
113 const second = await takeSnapshot(f, store, new Date("2026-11-17T03:00:00Z"));
114 expect(second.sent?.chunks).toBeLessThan(3);
115 });
116
117 test("the words say how many forges, and each forge's age and reason", () => {
118 const at = "2026-11-16T03:00:00Z";
119 const now = Date.parse("2026-11-16T08:30:00Z");
120 const hosted = [{ name: "kid", at, repos: 1, bytes: 9, key: "wrapped" as const }, { name: "sad", at, repos: 0, bytes: 0, key: "none" as const, why: "x" }];
121 expect(hostedWords(hosted)).toBe("1 hosted forge in the newest backup; 1 left out (see the log).");
122 expect(hostedWords([])).toBe("No hosted forges to keep.");
123 expect(hostedWords(undefined)).toBeUndefined();
124 expect(keptWords(hosted, "kid", now)).toBe("backed up 5 h ago");
125 expect(keptWords(hosted, "sad", now)).toBe("left out of the newest backup: x");
126 expect(keptWords(hosted, "new", now)).toBe("not in the newest backup");
127 expect(keptWords(undefined, "kid", now)).toBe("not in a backup yet");
128 });
129
130 test("noteHosted and readHosted round-trip, and a missing, corrupt or ill-shaped file reads as little as it can", () => {
131 const d = join(root, "noted");
132 expect(readHosted(d)).toBeUndefined();
133 const one = { name: "kid", at: "2026-11-16T03:00:00Z", repos: 1, bytes: 9, key: "wrapped" as const };
134 noteHosted(d, [one, { ...one, name: "sad", key: "none", why: "x" }]);
135 expect(readHosted(d)).toEqual([one, { ...one, name: "sad", key: "none", why: "x" }]);
136 writeFileSync(join(d, "snapshot", "hosted.json"), "{not json");
137 expect(readHosted(d)).toBeUndefined();
138 writeFileSync(join(d, "snapshot", "hosted.json"), JSON.stringify({ name: "kid" }));
139 expect(readHosted(d)).toBeUndefined();
140 writeFileSync(join(d, "snapshot", "hosted.json"), JSON.stringify([one, { name: "a", at: "t" }, { ...one, key: "other" }, { ...one, why: 3 }, null, { ...one, repos: "1" }]));
141 expect(readHosted(d)).toEqual([one]);
142 });
143
144 test("bodyIdOf names the data directory or nothing", () => {
145 expect(bodyIdOf('{"bodyId":"abc_1"}')).toBe("abc_1");
146 expect(bodyIdOf('{"bodyId":"../x"}')).toBeUndefined();
147 expect(bodyIdOf('{"bodyId":3}')).toBeUndefined();
148 expect(bodyIdOf("")).toBeUndefined();
149 });
150
151 test("restoreHostedInto writes the forge under children/<bodyId> and its ledger under spawn/", async () => {
152 const { store, manifest } = await take(PASS);
153 const dir = join(root, "into");
154 const got = await restoreHostedInto(store, manifest.name, "kid", dir, { passphrase: PASS });
155 expect(got.dir).toBe(join(dir, "children", BODY));
156 expect(existsSync(join(dir, "children", BODY, "forge.db"))).toBe(true);
157 expect(JSON.parse(readFileSync(join(dir, "spawn", "kid.json"), "utf8")).bodyId).toBe(BODY);
158 });
159
160 test("restore says plainly when there is no snapshot", async () => {
161 const store = dirStore(join(root, "empty-store"));
162 await expect(restoreHostedInto(store, "2026-11-16T03-00-00Z", "kid", join(root, "none"))).rejects.toThrow("no snapshot 2026-11-16T03-00-00Z");
163 });
164
165 test("the --forge command restores one forge, and says in a sentence what it cannot", async () => {
166 const storeDir = join(root, "cli-store");
167 const store = dirStore(storeDir);
168 await takeSnapshot({ db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase: PASS }, store, new Date("2026-11-16T03:00:00Z"));
169 const run = async (...args: string[]) => {
170 const p = Bun.spawn(["bun", "run", "src/snapshot/restore.ts", ...args], {
171 env: { ...process.env, FORGE_SNAPSHOT_DIR: storeDir, FORGE_SNAPSHOT_PASSPHRASE: PASS },
172 stdout: "pipe",
173 stderr: "pipe",
174 });
175 const [out, err] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited]);
176 return { code: p.exitCode, out, err };
177 };
178 const dir = join(root, "cli-out");
179 const ok = await run("latest", dir, "--forge", "kid");
180 expect(ok.code).toBe(0);
181 expect(ok.out).toContain("restored kid from 2026-11-16T03-00-00Z: ");
182 expect(existsSync(join(dir, "children", BODY, "forge.db"))).toBe(true);
183 const nobody = await run("latest", join(root, "cli-out2"), "--forge", "nobody");
184 expect(nobody.code).toBe(1);
185 expect(nobody.err).toContain("does not hold nobody");
186 const missing = await run("2020-01-01T00-00-00Z", join(root, "cli-out3"), "--forge", "kid");
187 expect(missing.code).toBe(1);
188 expect(missing.err).toContain("no snapshot 2020-01-01T00-00-00Z");
189 expect(missing.err).not.toContain(" at ");
190 expect((await run("latest", join(root, "cli-out4"), "--forge")).code).toBe(2);
191 });
192 });
test/worker-database-opens.test.ts +1 −1
@@ -65,7 +65,7 @@ const REACHED: Record<string, { opens: number; why: string }> = {
6565 "src/release/rollback.ts": { opens: 1, why: "liveSchema reads the schema count once for a rollback or a swap, on the serving thread or in the release tool" },
6666 "src/snapshot/drill.ts": { opens: 1, why: "a restore drill opens the restored copy once, as a duty of the serving thread" },
6767 "src/snapshot/restore.ts": { opens: 1, why: "a restore reads the sealed names of the restored copy once, in the restore tool or a drill on the serving thread" },
68 "src/snapshot/snapshot.ts": { opens: 1, why: "a snapshot counts the rows of its own copy once, as a duty of the serving thread; the mirror Worker imports this file for the manifest only" },
68 "src/snapshot/snapshot.ts": { opens: 2, why: "a snapshot counts the rows of its own copy once, and reads each hosted forge's copy once, as a duty of the serving thread; the mirror Worker imports this file for the manifest only" },
6969 "src/spawn/child_runs.ts": { opens: 2, why: "called by the reader process (spawn/child_db_reader.ts), never by a thread of the forge; cell.ts re-exports the two for tests that read a file directly" },
7070 };
7171

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs