Public: anyone can read this; the crew can change it.
Hosted forges are in no backup #13839
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13819-hosted-forges-are-in-no-backup → main 7 commits fork
14 files
changed +674
−41
docs/RECOVERY.md +3 −3
| @@ -35,10 +35,10 @@ A machine with `git`, `bun` and a clone of the forge (`bun install`). | ||
| 35 | 35 | |
| 36 | 36 | ## Route 1 — the restore command |
| 37 | 37 | |
| 38 | bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] | |
| 38 | bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME] | |
| 39 | 39 | |
| 40 | `<data-dir>` must be empty or new. `--clones N` (1–32): repositories cloned at once | |
| 41 | (default: cores less one, 2–16). `--offsite`: from the sealed lane, [](features/offsite-lane.md). | |
| 40 | `<data-dir>` must be empty or new. `--clones N` (1–32): clones at once (default: cores less one, 2–16). `--offsite`: [](features/offsite-lane.md). | |
| 41 | `--forge NAME`: one hosted forge, [](features/hosted-backup.md). | |
| 42 | 42 | |
| 43 | 43 | Then boot the forge on that directory, on a side port, off the public one: |
| 44 | 44 |
docs/features/README.md +1
| @@ -468,6 +468,7 @@ Read with the guide page [Backups and moving](../guide-backups-moving.md). | ||
| 468 | 468 | - [](follow-an-origin.md) |
| 469 | 469 | - [](resilience-backups.md) |
| 470 | 470 | - [](slow-line-backups.md) |
| 471 | - [](hosted-backup.md) | |
| 471 | 472 | - [](offsite-lane.md) |
| 472 | 473 | - [](origin-guard.md) |
| 473 | 474 | - [](forge-archive.md) |
docs/features/hosted-backup.md added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | # Hosted forges in the nightly backup | |
| 2 | ||
| 3 | Each forge this one hosts goes into the nightly snapshot (#13819), under `children/<name>/`. | |
| 4 | ||
| 5 | | What | Where in the snapshot | | |
| 6 | | --- | --- | | |
| 7 | | Database (a consistent copy, taken while it runs) | `forge.db` | | |
| 8 | | Repositories | `repos/<owner>/<name>.bundle` | | |
| 9 | | SSH host keys | `ssh/` | | |
| 10 | | Its key, wrapped under `FORGE_SNAPSHOT_PASSPHRASE` | `secret.key.wrapped` (without a passphrase the key stays home) | | |
| 11 | | Every app's production data | alongside | | |
| 12 | | This forge's own record of it | `ledger.json` | | |
| 13 | ||
| 14 | Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends nothing. A forge that cannot be read is named in the manifest's `hosted` list and the log; the snapshot goes on without it. | |
| 15 | ||
| 16 | ## Where it shows | |
| 17 | ||
| 18 | - Backups page: how many forges the newest backup kept, and how many were left out. | |
| 19 | - Resilience screen (terminal): the same line, then one line per forge: backed up H h ago, or left out and why. | |
| 20 | - `/forges`: each row, "backed up H h ago". | |
| 21 | ||
| 22 | ## Restore one forge | |
| 23 | ||
| 24 | Into an empty data directory; `children/<bodyId>/` and `spawn/<name>.json` are written there: | |
| 25 | ||
| 26 | bun run src/snapshot/restore.ts latest <data-dir> --forge <name> | |
| 27 | ||
| 28 | A whole restore of the parent leaves `children/` out; restore each forge this way. |
src/snapshot/hosted.ts added +129
| @@ -0,0 +1,129 @@ | ||
| 1 | /** | |
| 2 | * Hosted forges in the nightly snapshot (#13819). Each forge the parent hosts goes in under | |
| 3 | * `children/<name>/`, laid out as its own data directory so one can be restored alone: | |
| 4 | * `forge.db` (a consistent copy), `repos/<owner>/<name>.bundle`, `ssh/`, the key wrapped | |
| 5 | * under the passphrase (`secret.key.wrapped`), every app's production data, and `ledger.json`, | |
| 6 | * the parent's own record of it. The manifest's `hosted` lists each forge with when it was kept. | |
| 7 | * Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends | |
| 8 | * nothing. | |
| 9 | */ | |
| 10 | ||
| 11 | import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { CHILD_NAME } from "../spawn/placement.ts"; | |
| 14 | ||
| 15 | /** One hosted forge as a snapshot kept it. */ | |
| 16 | export interface HostedEntry { | |
| 17 | readonly name: string; | |
| 18 | /** When its copy was taken. */ | |
| 19 | readonly at: string; | |
| 20 | readonly repos: number; | |
| 21 | readonly bytes: number; | |
| 22 | /** `none` when no passphrase wrapped its key: its sealed values are lost on restore. */ | |
| 23 | readonly key: "wrapped" | "none"; | |
| 24 | /** Why it is not in this snapshot, when it is not. */ | |
| 25 | readonly why?: string; | |
| 26 | } | |
| 27 | ||
| 28 | export interface HostedSource { | |
| 29 | readonly name: string; | |
| 30 | readonly dir: string; | |
| 31 | readonly ledger: string; | |
| 32 | } | |
| 33 | ||
| 34 | const BODY_ID = /^[A-Za-z0-9_-]{1,64}$/; | |
| 35 | ||
| 36 | /** The data directory name a forge's ledger text names, or undefined when it names none. */ | |
| 37 | export function bodyIdOf(ledger: string): string | undefined { | |
| 38 | try { | |
| 39 | const v = JSON.parse(ledger) as { bodyId?: unknown }; | |
| 40 | return typeof v.bodyId === "string" && BODY_ID.test(v.bodyId) ? v.bodyId : undefined; | |
| 41 | } catch { | |
| 42 | return undefined; | |
| 43 | } | |
| 44 | } | |
| 45 | export const HOSTED_PREFIX = "children/"; | |
| 46 | ||
| 47 | /** The forges the parent hosts that are finished and have a data directory, by name. */ | |
| 48 | export function hostedSources(dataDir: string): HostedSource[] { | |
| 49 | let files: string[]; | |
| 50 | try { | |
| 51 | files = readdirSync(join(dataDir, "spawn")); | |
| 52 | } catch { | |
| 53 | return []; | |
| 54 | } | |
| 55 | const out: HostedSource[] = []; | |
| 56 | for (const f of files.sort()) { | |
| 57 | if (!f.endsWith(".json")) continue; | |
| 58 | const name = f.slice(0, -5); | |
| 59 | if (!CHILD_NAME.test(name)) continue; | |
| 60 | try { | |
| 61 | const text = readFileSync(join(dataDir, "spawn", f), "utf8"); | |
| 62 | const v = JSON.parse(text) as { name?: unknown; done?: unknown }; | |
| 63 | const bodyId = bodyIdOf(text); | |
| 64 | if (v.name !== name || v.done !== true || bodyId === undefined) continue; | |
| 65 | const dir = join(dataDir, "children", bodyId); | |
| 66 | if (existsSync(join(dir, "forge.db"))) out.push({ name, dir, ledger: text }); | |
| 67 | } catch { | |
| 68 | // A ledger that does not read is not a forge to keep. | |
| 69 | } | |
| 70 | } | |
| 71 | return out; | |
| 72 | } | |
| 73 | ||
| 74 | const hours = (ms: number): string => (ms < 3_600_000 ? "under an hour" : `${Math.floor(ms / 3_600_000)} h`); | |
| 75 | ||
| 76 | /** | |
| 77 | * The Backups line and the Resilience screen's head: how many hosted forges the newest snapshot | |
| 78 | * kept. Each forge's age is `keptWords`. | |
| 79 | */ | |
| 80 | export function hostedWords(hosted: readonly HostedEntry[] | undefined): string | undefined { | |
| 81 | if (hosted === undefined) return undefined; | |
| 82 | if (hosted.length === 0) return "No hosted forges to keep."; | |
| 83 | const left = hosted.filter((h) => h.why !== undefined).length; | |
| 84 | const kept = hosted.length - left; | |
| 85 | return `${kept} hosted forge${kept === 1 ? "" : "s"} in the newest backup${left > 0 ? `; ${left} left out (see the log)` : ""}.`; | |
| 86 | } | |
| 87 | ||
| 88 | /** What /forges says beside one forge. */ | |
| 89 | export function keptWords(hosted: readonly HostedEntry[] | undefined, name: string, now: number): string { | |
| 90 | if (hosted === undefined) return "not in a backup yet"; | |
| 91 | const h = hosted.find((e) => e.name === name); | |
| 92 | if (h === undefined) return "not in the newest backup"; | |
| 93 | if (h.why !== undefined) return `left out of the newest backup: ${h.why}`; | |
| 94 | return `backed up ${hours(now - Date.parse(h.at))} ago`; | |
| 95 | } | |
| 96 | ||
| 97 | const keptFile = (dataDir: string): string => join(dataDir, "snapshot", "hosted.json"); | |
| 98 | ||
| 99 | /** | |
| 100 | * What the last whole snapshot kept of each hosted forge, held on disk so /forges reads no store. | |
| 101 | */ | |
| 102 | export function noteHosted(dataDir: string, hosted: readonly HostedEntry[]): void { | |
| 103 | mkdirSync(join(dataDir, "snapshot"), { recursive: true }); | |
| 104 | const tmp = `${keptFile(dataDir)}.tmp`; | |
| 105 | writeFileSync(tmp, JSON.stringify(hosted)); | |
| 106 | renameSync(tmp, keptFile(dataDir)); | |
| 107 | } | |
| 108 | ||
| 109 | /** | |
| 110 | * The entries `noteHosted` kept; entries that do not read whole are dropped, a missing or corrupt | |
| 111 | * file is undefined. | |
| 112 | */ | |
| 113 | export function readHosted(dataDir: string): HostedEntry[] | undefined { | |
| 114 | try { | |
| 115 | const v: unknown = JSON.parse(readFileSync(keptFile(dataDir), "utf8")); | |
| 116 | if (!Array.isArray(v)) return undefined; | |
| 117 | const out: HostedEntry[] = []; | |
| 118 | for (const h of v as Record<string, unknown>[]) { | |
| 119 | if (typeof h?.["name"] !== "string" || typeof h["at"] !== "string") continue; | |
| 120 | if (typeof h["repos"] !== "number" || typeof h["bytes"] !== "number") continue; | |
| 121 | if (h["key"] !== "wrapped" && h["key"] !== "none") continue; | |
| 122 | if (h["why"] !== undefined && typeof h["why"] !== "string") continue; | |
| 123 | out.push({ name: h["name"], at: h["at"], repos: h["repos"], bytes: h["bytes"], key: h["key"], ...(h["why"] === undefined ? {} : { why: h["why"] }) }); | |
| 124 | } | |
| 125 | return out; | |
| 126 | } catch { | |
| 127 | return undefined; | |
| 128 | } | |
| 129 | } |
src/snapshot/restore.ts +111 −7
| @@ -38,7 +38,7 @@ | ||
| 38 | 38 | * open is reported by name. |
| 39 | 39 | */ |
| 40 | 40 | |
| 41 | import { chmodSync, existsSync, mkdirSync, readdirSync } from "node:fs"; | |
| 41 | import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs"; | |
| 42 | 42 | import { open as openFile, type FileHandle } from "node:fs/promises"; |
| 43 | 43 | import { availableParallelism } from "node:os"; |
| 44 | 44 | import { dirname, join } from "node:path"; |
| @@ -55,6 +55,7 @@ import { | ||
| 55 | 55 | type Manifest, |
| 56 | 56 | type SnapshotFile, |
| 57 | 57 | } from "./snapshot.ts"; |
| 58 | import { bodyIdOf, HOSTED_PREFIX } from "./hosted.ts"; | |
| 58 | 59 | import { chunkKey, isChunkId, openChunk } from "./chunks.ts"; |
| 59 | 60 | import { unwrapKey, WRAPPED_KEY } from "./wrap.ts"; |
| 60 | 61 | import { isPackedFile, partPaths, safePath, unpacker, type PackedFile, type Unpacker } from "./parts.ts"; |
| @@ -233,7 +234,7 @@ export async function readWhole(store: Store, name: string, entry: SnapshotFile) | ||
| 233 | 234 | * more than the reads in flight. Each chunk is proved against its name as |
| 234 | 235 | * it is opened, each file against its size and hash as it closes. |
| 235 | 236 | */ |
| 236 | async function fetchChunked( | |
| 237 | export async function fetchChunked( | |
| 237 | 238 | store: Store, |
| 238 | 239 | name: string, |
| 239 | 240 | entries: readonly SnapshotFile[], |
| @@ -392,10 +393,13 @@ export async function restoreSnapshot( | ||
| 392 | 393 | const raw = await store.get(`${prefix}manifest.json`); |
| 393 | 394 | if (raw === undefined) |
| 394 | 395 | throw new Error(`no snapshot ${name} in ${store.where}`); |
| 395 | const manifest = JSON.parse(new TextDecoder().decode(raw)) as Manifest; | |
| 396 | const parsed = JSON.parse(new TextDecoder().decode(raw)) as Manifest; | |
| 396 | 397 | say("manifest", 1, 1, raw.byteLength); |
| 397 | if (typeof manifest !== "object" || manifest === null || !Array.isArray(manifest.files)) | |
| 398 | if (typeof parsed !== "object" || parsed === null || !Array.isArray(parsed.files)) | |
| 398 | 399 | throw new Error(`snapshot ${name}: its manifest does not read`); |
| 400 | // The hosted forges' files (`hosted.ts`) come back one forge at a time (`restoreHosted`), not | |
| 401 | // into the parent's own data directory. | |
| 402 | const manifest: Manifest = { ...parsed, files: parsed.files.filter((f) => typeof f.path !== "string" || !f.path.startsWith(HOSTED_PREFIX)) }; | |
| 399 | 403 | const format: unknown = manifest.format; |
| 400 | 404 | if (format !== undefined && format !== 2) |
| 401 | 405 | throw new Error(`snapshot ${name} is of format ${String(format)}, which this forge does not read`); |
| @@ -573,7 +577,6 @@ export async function restoreSnapshot( | ||
| 573 | 577 | ); |
| 574 | 578 | if (failed.length > 0) |
| 575 | 579 | throw new Error(failed.sort((a, b) => a.at - b.at)[0]!.error); |
| 576 | const { rmSync } = await import("node:fs"); | |
| 577 | 580 | rmSync(join(dataDir, "restore-bundles"), { recursive: true, force: true }); |
| 578 | 581 | say("key", 0, key === undefined ? 0 : 1, 0); |
| 579 | 582 | if (key !== undefined) { |
| @@ -591,6 +594,90 @@ export async function restoreSnapshot( | ||
| 591 | 594 | return { manifest, files, repos, key: have, sealedMissing }; |
| 592 | 595 | } |
| 593 | 596 | |
| 597 | /** What one hosted forge's restore brought back. */ | |
| 598 | export interface RestoredHosted { | |
| 599 | readonly name: string; | |
| 600 | /** Where its files were written. */ | |
| 601 | readonly dir: string; | |
| 602 | /** The parent's ledger line for it, to be written to `<data>/spawn/<name>.json`. */ | |
| 603 | readonly ledger: string; | |
| 604 | readonly files: number; | |
| 605 | readonly repos: number; | |
| 606 | readonly key: "restored" | "missing"; | |
| 607 | } | |
| 608 | ||
| 609 | /** | |
| 610 | * One hosted forge out of a snapshot, alone (#13819): its files written as its own data directory | |
| 611 | * into `dir` (empty; a function of the ledger's bodyId when the name is the ledger's to give), | |
| 612 | * its repositories cloned back from their bundles. The caller writes the ledger and starts it. | |
| 613 | * Refuses, in a plain sentence, a snapshot that is missing or does not hold the forge. | |
| 614 | */ | |
| 615 | export async function restoreHosted( | |
| 616 | store: Store, | |
| 617 | snapshot: string, | |
| 618 | forge: string, | |
| 619 | where: string | ((bodyId: string) => string), | |
| 620 | options: { readonly passphrase?: string | undefined; readonly git?: typeof runGit | undefined } = {}, | |
| 621 | ): Promise<RestoredHosted> { | |
| 622 | if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$/.test(snapshot)) throw new Error(`not a snapshot name: ${snapshot}`); | |
| 623 | if (!/^[a-z0-9-]{1,40}$/.test(forge)) throw new Error(`not a forge name: ${forge}`); | |
| 624 | const raw = await store.get(`${SNAPSHOT_PREFIX}${snapshot}/manifest.json`); | |
| 625 | if (raw === undefined) throw new Error(`no snapshot ${snapshot} in ${store.where}`); | |
| 626 | const manifest = JSON.parse(new TextDecoder().decode(raw)) as Manifest; | |
| 627 | if (manifest.format !== 2 || !Array.isArray(manifest.files)) throw new Error(`snapshot ${snapshot} holds no hosted forges`); | |
| 628 | const entry = manifest.hosted?.find((h) => h.name === forge); | |
| 629 | if (entry === undefined || entry.why !== undefined) throw new Error(`snapshot ${snapshot} does not hold ${forge}${entry?.why === undefined ? "" : ` (${entry.why})`}`); | |
| 630 | const prefix = `${HOSTED_PREFIX}${forge}/`; | |
| 631 | const mine = manifest.files.filter((f) => f.path.startsWith(prefix)).map((f) => ({ ...f, path: f.path.slice(prefix.length) })); | |
| 632 | const ledger = mine.find((f) => f.path === "ledger.json"); | |
| 633 | if (ledger === undefined || !mine.some((f) => f.path === "forge.db")) throw new Error(`snapshot ${snapshot}: ${forge} is not whole in it`); | |
| 634 | const ledgerText = new TextDecoder().decode(await readWhole(store, snapshot, ledger)); | |
| 635 | const bodyId = bodyIdOf(ledgerText); | |
| 636 | if (bodyId === undefined) throw new Error(`the ledger of ${forge} in ${snapshot} names no data directory`); | |
| 637 | const dir = typeof where === "string" ? where : where(bodyId); | |
| 638 | if (existsSync(dir) && readdirSync(dir).length > 0) throw new Error(`${dir} is not empty; a restore needs an empty data directory`); | |
| 639 | let key: Buffer | undefined; | |
| 640 | const keyEntry = mine.find((f) => f.path === WRAPPED_KEY); | |
| 641 | if (keyEntry !== undefined && options.passphrase) { | |
| 642 | key = unwrapKey(await readWhole(store, snapshot, keyEntry), options.passphrase); | |
| 643 | if (key === undefined) throw new Error("the passphrase does not open this forge's wrapped key"); | |
| 644 | } | |
| 645 | const files = mine.filter((f) => f.path !== "ledger.json" && f.path !== WRAPPED_KEY); | |
| 646 | const got = await fetchChunked(store, snapshot, files, dir, () => {}); | |
| 647 | const git = options.git ?? runGit; | |
| 648 | for (const b of got.bundles) { | |
| 649 | const to = repoDir(join(dir, "repos"), b.owner, b.name); | |
| 650 | mkdirSync(dirname(to), { recursive: true }); | |
| 651 | const cloned = await git(["clone", "--quiet", "--mirror", b.file, to]); | |
| 652 | if (!cloned.ok) throw new Error(`could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`); | |
| 653 | await git(["remote", "remove", "origin"], { cwd: to }); | |
| 654 | await git(["config", "core.logAllRefUpdates", "true"], { cwd: to }); | |
| 655 | await installHooks(to); | |
| 656 | } | |
| 657 | rmSync(join(dir, "restore-bundles"), { recursive: true, force: true }); | |
| 658 | if (key !== undefined) { | |
| 659 | await Bun.write(join(dir, "secret.key"), key); | |
| 660 | chmodSync(join(dir, "secret.key"), 0o600); | |
| 661 | } | |
| 662 | return { name: forge, dir, ledger: ledgerText, files: got.files, repos: got.bundles.length, key: key === undefined ? "missing" : "restored" }; | |
| 663 | } | |
| 664 | ||
| 665 | /** | |
| 666 | * The `--forge` restore: the forge into `<data>/children/<bodyId>`, its ledger into `<data>/spawn`. | |
| 667 | */ | |
| 668 | export async function restoreHostedInto( | |
| 669 | store: Store, | |
| 670 | snapshot: string, | |
| 671 | forge: string, | |
| 672 | dataDir: string, | |
| 673 | options: { readonly passphrase?: string | undefined; readonly git?: typeof runGit | undefined } = {}, | |
| 674 | ): Promise<RestoredHosted> { | |
| 675 | const one = await restoreHosted(store, snapshot, forge, (id) => join(dataDir, "children", id), options); | |
| 676 | mkdirSync(join(dataDir, "spawn"), { recursive: true }); | |
| 677 | await Bun.write(join(dataDir, "spawn", `${forge}.json`), one.ledger); | |
| 678 | return one; | |
| 679 | } | |
| 680 | ||
| 594 | 681 | if (import.meta.main) { |
| 595 | 682 | const args = process.argv.slice(2); |
| 596 | 683 | let clones: number | undefined; |
| @@ -607,11 +694,16 @@ if (import.meta.main) { | ||
| 607 | 694 | // `--offsite`: read the sealed off-site lane (FORGE_MIRROR_S3_*) in place of the destination (#11805). |
| 608 | 695 | const offsite = args.indexOf("--offsite"); |
| 609 | 696 | if (offsite >= 0) args.splice(offsite, 1); |
| 697 | // `--forge NAME`: one hosted forge alone, into <data-dir>/children/<bodyId> with its ledger under | |
| 698 | // spawn/ (#13819). | |
| 699 | const only = args.indexOf("--forge"); | |
| 700 | const forgeName = only >= 0 ? args[only + 1] : undefined; | |
| 701 | if (only >= 0) args.splice(only, 2); | |
| 610 | 702 | const [name, dataDir] = args; |
| 611 | if (!name || !dataDir || args.length > 2) { | |
| 703 | if (!name || !dataDir || args.length > 2 || (only >= 0 && !forgeName)) { | |
| 612 | 704 | // CLI usage on stderr for the person at the terminal, not a forge event. |
| 613 | 705 | console.error( |
| 614 | "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite]", | |
| 706 | "usage: bun run src/snapshot/restore.ts <snapshot-name|latest> <data-dir> [--clones N] [--offsite] [--forge NAME]", | |
| 615 | 707 | ); |
| 616 | 708 | process.exit(2); |
| 617 | 709 | } |
| @@ -637,6 +729,18 @@ if (import.meta.main) { | ||
| 637 | 729 | console.error(`no snapshot in ${store.where}`); |
| 638 | 730 | process.exit(1); |
| 639 | 731 | } |
| 732 | if (forgeName !== undefined) { | |
| 733 | const snap = newest?.name ?? name; | |
| 734 | try { | |
| 735 | const one = await restoreHostedInto(store, snap, forgeName, dataDir, { passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined }); | |
| 736 | console.log(`restored ${forgeName} from ${snap}: ${one.files} files, ${one.repos} repositories into ${one.dir}${one.key === "missing" ? "; its key was not restored" : ""}`); | |
| 737 | process.exit(0); | |
| 738 | } catch (e) { | |
| 739 | // CLI message on stderr for the person at the terminal. | |
| 740 | console.error(e instanceof Error ? e.message : String(e)); | |
| 741 | process.exit(1); | |
| 742 | } | |
| 743 | } | |
| 640 | 744 | // One line per phase when it ends: its count, bytes and seconds. |
| 641 | 745 | let began = performance.now(); |
| 642 | 746 | const done = await restoreSnapshot(store, newest?.name ?? name, dataDir, { |
src/snapshot/service.ts +3
| @@ -15,6 +15,7 @@ import { activeCrewRuns, crewHoldsCore, forgeCpus, holdClock } from "../core/one | ||
| 15 | 15 | import { appendSnapshot } from "../core/ledger.ts"; |
| 16 | 16 | import { createLog } from "../kernel/log.ts"; |
| 17 | 17 | import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts"; |
| 18 | import { noteHosted } from "./hosted.ts"; | |
| 18 | 19 | import { |
| 19 | 20 | KEEP, |
| 20 | 21 | listSnapshots, |
| @@ -285,6 +286,8 @@ export function startSnapshots( | ||
| 285 | 286 | running = undefined; |
| 286 | 287 | startedAt = undefined; |
| 287 | 288 | }); |
| 289 | // /forges says when each hosted forge was kept (#13819). | |
| 290 | if (manifest.hosted !== undefined) noteHosted(app.config.dataDir, manifest.hosted); | |
| 288 | 291 | // No snapshot is being taken now (runs are one after another), so the sweep may run every |
| 289 | 292 | // time. |
| 290 | 293 | await pruneSnapshots(store, KEEP, Date.now(), true); |
src/snapshot/snapshot.ts +86 −28
| @@ -46,6 +46,7 @@ import { listEveryRepo } from "../core/repos.ts"; | ||
| 46 | 46 | import { keepLentObjects, repoDir } from "../git/repos.ts"; |
| 47 | 47 | import { createLog } from "../kernel/log.ts"; |
| 48 | 48 | import { bundleRepo } from "./bundle.ts"; |
| 49 | import { hostedSources, HOSTED_PREFIX, type HostedEntry } from "./hosted.ts"; | |
| 49 | 50 | import { countRows, type DrillCounts } from "./counts.ts"; |
| 50 | 51 | import { CHUNK_PREFIX, chunkId, chunkKey, chunkSeed, contentChunker, fixedChunker, isChunkId, packChunk, type Chunker } from "./chunks.ts"; |
| 51 | 52 | import type { PackedFile } from "./parts.ts"; |
| @@ -122,6 +123,8 @@ export interface Manifest { | ||
| 122 | 123 | readonly absent?: readonly AbsentRepo[]; |
| 123 | 124 | /** Refs a bundle could not hold because their commits are missing, by repository. */ |
| 124 | 125 | readonly brokenRefs?: readonly BrokenRefs[]; |
| 126 | /** The hosted forges this snapshot kept under `children/<name>/` (#13819). */ | |
| 127 | readonly hosted?: readonly HostedEntry[]; | |
| 125 | 128 | /** |
| 126 | 129 | * What this snapshot itself put in the store: chunks sent and their bytes as stored, and chunks |
| 127 | 130 | * it found there. |
| @@ -594,35 +597,89 @@ export async function takeSnapshot( | ||
| 594 | 597 | |
| 595 | 598 | // Every app's production data. SQLite inside is copied the consistent |
| 596 | 599 | // way; its side-files are the live process's and not part of a copy. |
| 597 | const apps = listApps(forge.db); | |
| 598 | let kept = 0; | |
| 599 | for (const app of apps) { | |
| 600 | const dir = envDataDir(forge.dataDir, app.id, "production"); | |
| 601 | const inside = filesUnder(dir); | |
| 602 | if (inside.length === 0) continue; | |
| 603 | kept++; | |
| 604 | for (const f of inside) { | |
| 605 | if (SIDE_FILE.test(f)) continue; | |
| 606 | const source = join(dir, f); | |
| 607 | if (SQLITE.test(f)) { | |
| 608 | const copy = join( | |
| 609 | scratch, | |
| 610 | `app-${app.id}-${f.replace(/[\/\\]/g, "__")}`, | |
| 611 | ); | |
| 612 | try { | |
| 613 | await vacuumFile(source, copy); | |
| 614 | await putFile(`apps/${app.id}/production/data/${f}`, copy, "fixed"); | |
| 615 | } catch (error) { | |
| 616 | log.warn("app database not copied consistently; copied as a file", { | |
| 617 | app: app.slug, | |
| 618 | file: f, | |
| 619 | error: String(error), | |
| 620 | }); | |
| 621 | await putFile(`apps/${app.id}/production/data/${f}`, source, "fixed"); | |
| 622 | } finally { | |
| 623 | rmSync(copy, { force: true }); | |
| 600 | // SQLite inside is copied the consistent way; side-files are the live process's. | |
| 601 | const putAppData = async (apps: ReturnType<typeof listApps>, root: string, into: string, label: string): Promise<number> => { | |
| 602 | let withData = 0; | |
| 603 | for (const app of apps) { | |
| 604 | const dir = envDataDir(root, app.id, "production"); | |
| 605 | const inside = filesUnder(dir); | |
| 606 | if (inside.length === 0) continue; | |
| 607 | withData++; | |
| 608 | for (const f of inside) { | |
| 609 | if (SIDE_FILE.test(f)) continue; | |
| 610 | const source = join(dir, f); | |
| 611 | const path = `${into}apps/${app.id}/production/data/${f}`; | |
| 612 | if (SQLITE.test(f)) { | |
| 613 | const copy = join(scratch, `app-${label}${app.id}-${f.replace(/[\/\\]/g, "__")}`); | |
| 614 | try { | |
| 615 | await vacuumFile(source, copy); | |
| 616 | await putFile(path, copy, "fixed"); | |
| 617 | } catch (error) { | |
| 618 | log.warn("app database not copied consistently; copied as a file", { app: `${label}${app.slug}`, file: f, error: String(error) }); | |
| 619 | await putFile(path, source, "fixed"); | |
| 620 | } finally { | |
| 621 | rmSync(copy, { force: true }); | |
| 622 | } | |
| 623 | } else await putFile(path, source); | |
| 624 | } | |
| 625 | } | |
| 626 | return withData; | |
| 627 | }; | |
| 628 | const kept = await putAppData(listApps(forge.db), forge.dataDir, "", ""); | |
| 629 | ||
| 630 | // Every hosted forge (#13819), each under its own prefix so one restores alone. A forge that | |
| 631 | // cannot be read costs that forge, never the snapshot; the manifest says which. | |
| 632 | const hosted: HostedEntry[] = []; | |
| 633 | for (const child of hostedSources(forge.dataDir)) { | |
| 634 | const into = `${HOSTED_PREFIX}${child.name}/`; | |
| 635 | const from = files.length; | |
| 636 | const copy = join(scratch, `hosted-${child.name}.db`); | |
| 637 | let repoCount = 0; | |
| 638 | let wrapped: HostedEntry["key"] = "none"; | |
| 639 | try { | |
| 640 | await vacuumFile(join(child.dir, "forge.db"), copy); | |
| 641 | const childDb = new Database(copy, { readonly: true }); | |
| 642 | let childRepos: ReturnType<typeof listEveryRepo>; | |
| 643 | let childApps: ReturnType<typeof listApps>; | |
| 644 | try { | |
| 645 | childRepos = listEveryRepo(childDb); | |
| 646 | childApps = listApps(childDb); | |
| 647 | } finally { | |
| 648 | childDb.close(); | |
| 649 | } | |
| 650 | await putFile(`${into}forge.db`, copy, "fixed"); | |
| 651 | await putBytes(`${into}ledger.json`, new TextEncoder().encode(child.ledger)); | |
| 652 | const childKey = Bun.file(join(child.dir, "secret.key")); | |
| 653 | if (forge.passphrase && (await childKey.exists())) { | |
| 654 | const sealed = wrapKey(new Uint8Array(await childKey.arrayBuffer()), forge.passphrase); | |
| 655 | if (unwrapKey(sealed, forge.passphrase) === undefined) throw new Error("its wrapped key does not open"); | |
| 656 | await putBytes(`${into}${WRAPPED_KEY}`, new TextEncoder().encode(sealed)); | |
| 657 | wrapped = "wrapped"; | |
| 658 | } | |
| 659 | for (const f of filesUnder(join(child.dir, "ssh"))) await putFile(`${into}ssh/${f}`, join(child.dir, "ssh", f)); | |
| 660 | for (const repo of childRepos) { | |
| 661 | const dir = repoDir(join(child.dir, "repos"), repo.ownerName, repo.name); | |
| 662 | const bundle = join(scratch, `hosted-${child.name}-${repo.ownerName}--${repo.name}.bundle`); | |
| 663 | await keepLentObjects(dir); | |
| 664 | const made = await bundleRepo(dir, bundle, { deadlineMs: forge.bundleDeadlineMs, sizes }); | |
| 665 | if (!made.ok) { | |
| 666 | rmSync(bundle, { force: true }); | |
| 667 | if (!made.empty) log.error("hosted repository left out of the snapshot", { forge: child.name, repo: `${repo.ownerName}/${repo.name}`, why: made.why }); | |
| 668 | continue; | |
| 624 | 669 | } |
| 625 | } else await putFile(`apps/${app.id}/production/data/${f}`, source); | |
| 670 | await putFile(`${into}repos/${repo.ownerName}/${repo.name}.bundle`, bundle); | |
| 671 | rmSync(bundle, { force: true }); | |
| 672 | repoCount++; | |
| 673 | } | |
| 674 | await putAppData(childApps, child.dir, into, `${child.name}/`); | |
| 675 | hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: files.slice(from).reduce((n, f) => n + f.bytes, 0), key: wrapped }); | |
| 676 | } catch (error) { | |
| 677 | // What went up before the failure stays listed in `files` and is harmless; the entry says | |
| 678 | // why. | |
| 679 | log.error("hosted forge left out of the snapshot", { forge: child.name, error: String(error) }); | |
| 680 | hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: 0, key: wrapped, why: String(error).slice(0, 200) }); | |
| 681 | } finally { | |
| 682 | rmSync(copy, { force: true }); | |
| 626 | 683 | } |
| 627 | 684 | } |
| 628 | 685 | |
| @@ -691,6 +748,7 @@ export async function takeSnapshot( | ||
| 691 | 748 | repositories, |
| 692 | 749 | ...(absent.length === 0 ? {} : { absent }), |
| 693 | 750 | ...(brokenRefs.length === 0 ? {} : { brokenRefs }), |
| 751 | hosted, | |
| 694 | 752 | sent: { chunks: landed, bytes: landedBytes, held: found }, |
| 695 | 753 | }; |
| 696 | 754 | // Announced only when whole: every chunk it names, seen in the store now. |
src/tui/screens/resilience.ts +8
| @@ -39,6 +39,7 @@ import { forgetMirror, saveMirror } from "../../actions/mirror.ts"; | ||
| 39 | 39 | import { JOBS } from "../../nightly/jobs.ts"; |
| 40 | 40 | import { outputTail } from "../../nightly/output.ts"; |
| 41 | 41 | import { nightlyLines } from "../../nightly/says.ts"; |
| 42 | import { hostedWords, keptWords, readHosted } from "../../snapshot/hosted.ts"; | |
| 42 | 43 | import { backupsLine } from "../../snapshot/backups_line.ts"; |
| 43 | 44 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 44 | 45 | import { lastDrill } from "../../snapshot/drills.ts"; |
| @@ -303,6 +304,13 @@ export function resilienceScreen(f: Session, ctx: Ctx): Screen { | ||
| 303 | 304 | // middle. |
| 304 | 305 | if (note !== "") out.push(...wrap(note, INNER), ""); |
| 305 | 306 | out.push(...wrap(backupsLine(f.app).says, INNER), ...verdictLines(f), ""); |
| 307 | // Hosted forges in the nightly snapshot, as of the last one taken here (#13819). | |
| 308 | const kept = readHosted(f.app.config.dataDir); | |
| 309 | if (kept !== undefined) { | |
| 310 | out.push(...wrap(hostedWords(kept) ?? "", INNER)); | |
| 311 | for (const h of kept) out.push(...wrap(` ${h.name}: ${keptWords(kept, h.name, Date.now())}`, INNER)); | |
| 312 | out.push(""); | |
| 313 | } | |
| 306 | 314 | for (const line of nightlyLines(f.app.config.dataDir)) out.push(...wrap(line, INNER)); |
| 307 | 315 | out.push(""); |
| 308 | 316 | out.push(...wrap(destinationLine(f), INNER), ...wrap(mirrorLine(f), INNER)); |
src/web/pages/backups.ts +3
| @@ -30,6 +30,7 @@ import { costFacts } from "../../snapshot/drill_cost.ts"; | ||
| 30 | 30 | import { lastDrill } from "../../snapshot/drills.ts"; |
| 31 | 31 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 32 | 32 | import { bodiesOf, type Store } from "../../snapshot/store.ts"; |
| 33 | import { hostedWords } from "../../snapshot/hosted.ts"; | |
| 33 | 34 | import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts"; |
| 34 | 35 | import { drillStatus, mirrorNow, snapshotNow, startDrillNow } from "../../actions/backups_now.ts"; |
| 35 | 36 | import { lastSnapshotOutcome } from "../../snapshot/outcome.ts"; |
| @@ -346,6 +347,7 @@ async function movingPart( | ||
| 346 | 347 | ...(d ? { drill: { at: d.at, verdict: d.verdict } } : {}), |
| 347 | 348 | now, |
| 348 | 349 | }); |
| 350 | const hostedLine = hostedWords(snapshots[0]?.hosted); | |
| 349 | 351 | const restore = snapshots[0] |
| 350 | 352 | ? html`<pre class="restore-command" tabindex="0" aria-label="Restore command">bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>` |
| 351 | 353 | : null; |
| @@ -358,6 +360,7 @@ async function movingPart( | ||
| 358 | 360 | fourteen. |
| 359 | 361 | </p> |
| 360 | 362 | ${keyNote(svc.wraps, snapshots[0])} |
| 363 | ${hostedLine === undefined ? null : html`<p data-hosted-kept>${hostedLine}</p>`} | |
| 361 | 364 | ${when( |
| 362 | 365 | svc.progress(), |
| 363 | 366 | (p) => |
src/web/pages/forges.ts +5 −2
| @@ -32,6 +32,7 @@ import { claimLinkBody } from "./new-forge.ts"; | ||
| 32 | 32 | import { appLinkSummary, appLinkWords } from "../../spawn/app_links.ts"; |
| 33 | 33 | import { REACH_SETTINGS, reachByForge, type ForgeReach } from "../../actions/child_reach.ts"; |
| 34 | 34 | import { forgeReachRows } from "./hosted-forges.ts"; |
| 35 | import { keptWords, readHosted } from "../../snapshot/hosted.ts"; | |
| 35 | 36 | import { archiveForge, archivedForges, restoreForge, type ArchiveRecord } from "../../spawn/archive.ts"; |
| 36 | 37 | import { claimedWords, isForgeSort, keepWords, SORT_WORDS, sortForges, type ForgeSort } from "../../spawn/keep_words.ts"; |
| 37 | 38 | import { CHILD_NAME } from "../../spawn/placement.ts"; |
| @@ -75,7 +76,7 @@ function seenWords(seen: Seen): Html { | ||
| 75 | 76 | |
| 76 | 77 | const XLARGE_SAYS = `${sizeWords("xlarge")}. ${sizesView().sizes.find((x) => x.size === "xlarge")?.says ?? ""}`; |
| 77 | 78 | |
| 78 | export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = false): Html { | |
| 79 | export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = false, kept?: string): Html { | |
| 79 | 80 | const state = claimState(r); |
| 80 | 81 | const seen = seenOf(r); |
| 81 | 82 | const meta = [ |
| @@ -103,6 +104,7 @@ export function forgeRow(r: ClientRow, now: number, reach?: ForgeReach, bill = f | ||
| 103 | 104 | r.made && seen === null ? (r.lastActivityAt === null ? html`<span data-activity>no one has done anything here yet</span>` : html`<span data-activity>a person last did something ${time(r.lastActivityAt)}</span>`) : null, |
| 104 | 105 | claimedWords(r) === null ? null : html`<span data-claimed-by>${claimedWords(r)}</span>`, |
| 105 | 106 | keepWords(r) === null ? null : html`<span data-keep>${keepWords(r)}</span>`, |
| 107 | r.made && kept !== undefined ? html`<span data-backed-up>${kept}</span>` : null, | |
| 106 | 108 | r.noticedAt === null || r.noticedAt === undefined ? null : html`<span data-noticed>owner told ${time(r.noticedAt)}</span>`, |
| 107 | 109 | badge(r.nextStep.key === "going" ? html`going: last tell ${time(r.nextStep.lastTellAt!)}` : r.nextStep.words, { |
| 108 | 110 | variant: "outline", |
| @@ -239,6 +241,7 @@ export async function forgesPage(ctx: Ctx): Promise<Response> { | ||
| 239 | 241 | // Two small tables, by name: the rows themselves come from memory, and this reads no ledger. |
| 240 | 242 | const reachOf = timePhase(ctx.request, "reach", () => reachByForge(ctx.app, user)); |
| 241 | 243 | // Sizes that together pass the machine's memory are said in one line; nothing is refused for it. |
| 244 | const hostedKept = readHosted(ctx.app.config.dataDir); | |
| 242 | 245 | const over = memoryOverLine(forgesMemory(ctx.app, memoryOf(rows))); |
| 243 | 246 | const rendering = performance.now(); |
| 244 | 247 | const action = |
| @@ -272,7 +275,7 @@ export async function forgesPage(ctx: Ctx): Promise<Response> { | ||
| 272 | 275 | ${btn("Archive selected", { size: "sm", title: "Pick forges with the boxes; the next page says what happens before anything does" })} |
| 273 | 276 | </form> |
| 274 | 277 | <ul class="list setting-list"> |
| 275 | ${rows.map((r) => forgeRow(r, Date.now(), reachOf.get(r.name), claimsBill(user, r.billTo)))} | |
| 278 | ${rows.map((r) => forgeRow(r, Date.now(), reachOf.get(r.name), claimsBill(user, r.billTo), keptWords(hostedKept, r.name, Date.now())))} | |
| 276 | 279 | </ul> |
| 277 | 280 | </section>`} |
| 278 | 281 | ${sealedLine(ctx)} |
test/client-list.test.ts +18
| @@ -8,7 +8,9 @@ | ||
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | 10 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 11 | import { rmSync } from "node:fs"; | |
| 11 | 12 | import { mkdtemp, rm } from "node:fs/promises"; |
| 13 | import { noteHosted } from "../src/snapshot/hosted.ts"; | |
| 12 | 14 | import { tmpdir } from "node:os"; |
| 13 | 15 | import { join } from "node:path"; |
| 14 | 16 | import { createRepo } from "../src/actions/repos.ts"; |
| @@ -230,6 +232,22 @@ describe("the client list", () => { | ||
| 230 | 232 | expect(html).not.toContain(claimUrl.split("#")[1]!); |
| 231 | 233 | }); |
| 232 | 234 | |
| 235 | test("each row says when the newest backup kept its forge, or why not (#13819)", async () => { | |
| 236 | const at = new Date(Date.now() - 5 * 3_600_000).toISOString(); | |
| 237 | noteHosted(dir, [ | |
| 238 | { name: "maria", at, repos: 1, bytes: 9, key: "wrapped" }, | |
| 239 | { name: "lou", at, repos: 0, bytes: 0, key: "none", why: "its database would not open" }, | |
| 240 | ]); | |
| 241 | const html = (await (await list(admin)).text()).replace(/\s+/g, " "); | |
| 242 | const maria = html.slice(html.indexOf('data-forge="maria"')); | |
| 243 | expect(maria.slice(0, maria.indexOf("</li>"))).toContain("<span data-backed-up>backed up 5 h ago</span>"); | |
| 244 | const lou = html.slice(html.indexOf('data-forge="lou"')); | |
| 245 | expect(lou.slice(0, lou.indexOf("</li>"))).toContain("left out of the newest backup: its database would not open"); | |
| 246 | rmSync(join(dir, "snapshot", "hosted.json")); | |
| 247 | const none = (await (await list(admin)).text()).replace(/\s+/g, " "); | |
| 248 | expect(none).toContain("not in a backup yet"); | |
| 249 | }); | |
| 250 | ||
| 233 | 251 | test("the admin's page carries the forges side by side: columns, shared use, no heartbeat yet", async () => { |
| 234 | 252 | const res = await list(admin); |
| 235 | 253 | expect(res.status).toBe(200); |
test/hosted-surfaces.test.ts added +86
| @@ -0,0 +1,86 @@ | ||
| 1 | /** | |
| 2 | * Where a hosted forge's backup shows (#13819): the Backups page says how many the newest snapshot | |
| 3 | * kept, and the Resilience screen says it again with one line per forge. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import { noteHosted } from "../src/snapshot/hosted.ts"; | |
| 13 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; | |
| 14 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; | |
| 15 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 16 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 17 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 18 | import { quietLogs } from "./helpers/globals.ts"; | |
| 19 | import { settleApp, settlePalette, settleUntil } from "./helpers/tui.ts"; | |
| 20 | ||
| 21 | quietLogs(); | |
| 22 | ||
| 23 | let root = ""; | |
| 24 | let app: App; | |
| 25 | let child: App; | |
| 26 | let svc: SnapshotsHandle; | |
| 27 | let servers: Servers; | |
| 28 | let cookie = ""; | |
| 29 | ||
| 30 | beforeAll(async () => { | |
| 31 | for (const k of Object.keys(process.env)) if (k.startsWith("S3_") || k === "FORGE_SNAPSHOT_DIR") delete process.env[k]; | |
| 32 | root = mkdtempSync(join(tmpdir(), "forge-hosted-surfaces-")); | |
| 33 | app = createApp({ dataDir: root, port: 0 }); | |
| 34 | child = createApp({ dataDir: join(root, "children", "b1"), port: 0 }); | |
| 35 | mkdirSync(join(root, "spawn"), { recursive: true }); | |
| 36 | writeFileSync(join(root, "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "b1", done: true })); | |
| 37 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 38 | if (!u.ok) throw new Error(u.error); | |
| 39 | const store = dirStore(join(root, "store")); | |
| 40 | await takeSnapshot({ db: app.db, dataDir: root, reposRoot: app.reposRoot, version: "t" }, store, new Date()); | |
| 41 | svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 42 | servers = await serve(app); | |
| 43 | const r = await fetch(`${servers.url}/signin`, { | |
| 44 | method: "POST", | |
| 45 | headers: { origin: servers.url }, | |
| 46 | body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }), | |
| 47 | redirect: "manual", | |
| 48 | }); | |
| 49 | cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!; | |
| 50 | }); | |
| 51 | afterAll(async () => { | |
| 52 | servers.stop(); | |
| 53 | await svc.stop(); | |
| 54 | child.close(); | |
| 55 | app.close(); | |
| 56 | rmSync(root, { recursive: true, force: true }); | |
| 57 | }); | |
| 58 | ||
| 59 | test("the Backups page says how many hosted forges the newest snapshot kept", async () => { | |
| 60 | const html = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text(); | |
| 61 | expect(html).toMatch(/<p data-hosted-kept>1 hosted forge in the newest backup\.<\/p>/); | |
| 62 | }); | |
| 63 | ||
| 64 | test("the Resilience screen says it, and which forge is kept or left out", async () => { | |
| 65 | const at = new Date(Date.now() - 2 * 3_600_000).toISOString(); | |
| 66 | noteHosted(root, [ | |
| 67 | { name: "kid", at, repos: 1, bytes: 9, key: "none" }, | |
| 68 | { name: "sad", at, repos: 0, bytes: 0, key: "none", why: "its database would not open" }, | |
| 69 | ]); | |
| 70 | const terminal = headlessTerminal(140, 50); | |
| 71 | const user = getUserByName(app.db, "alice")!; | |
| 72 | const { app: tui } = createForgeApp(app, { sshUsername: "t", remoteAddress: "127.0.0.1", user }, { terminal, exitHooks: false }); | |
| 73 | tui.start(); | |
| 74 | const key = (k: string, ctrl = false) => tui.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false }); | |
| 75 | key("k", true); | |
| 76 | await settleApp(tui); | |
| 77 | for (const ch of "Resilience") key(ch); | |
| 78 | await settlePalette(tui); | |
| 79 | key("enter"); | |
| 80 | await settleUntil(tui, "hosted forge"); | |
| 81 | const text = tui.screen.toText(); | |
| 82 | tui.stop(); | |
| 83 | expect(text).toContain("1 hosted forge in the newest backup; 1 left out"); | |
| 84 | expect(text).toContain("kid: backed up 2 h ago"); | |
| 85 | expect(text).toContain("sad: left out of the newest backup: its database would not open"); | |
| 86 | }); |
test/snapshot-hosted.test.ts added +192
| @@ -0,0 +1,192 @@ | ||
| 1 | /** | |
| 2 | * Hosted forges are in the nightly snapshot (#13819): a forge the parent hosts goes up under | |
| 3 | * `children/<name>/`, restores alone, and the Backups and /forges words say when it was kept. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; | |
| 7 | import { rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { commitLog } from "../src/git/read.ts"; | |
| 14 | import { repoDir } from "../src/git/repos.ts"; | |
| 15 | import { bodyIdOf, hostedSources, hostedWords, keptWords, noteHosted, readHosted } from "../src/snapshot/hosted.ts"; | |
| 16 | import { restoreHosted, restoreHostedInto, restoreSnapshot } from "../src/snapshot/restore.ts"; | |
| 17 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; | |
| 18 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 19 | ||
| 20 | const PASS = "correct horse battery staple"; | |
| 21 | ||
| 22 | describe("hosted forges in the snapshot", () => { | |
| 23 | let root: string; | |
| 24 | let parent: App; | |
| 25 | let child: App; | |
| 26 | const BODY = "body1"; | |
| 27 | ||
| 28 | beforeAll(async () => { | |
| 29 | root = mkdtempSync(join(tmpdir(), "forge-hosted-snap-")); | |
| 30 | parent = createApp({ dataDir: join(root, "parent"), port: 0 }); | |
| 31 | child = createApp({ dataDir: join(root, "parent", "children", BODY), port: 0 }); | |
| 32 | const u = await createUser(child.db, { origin: "seed", trusted: true, username: "glint", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 33 | if (!u.ok) throw new Error(u.error); | |
| 34 | const repo = await createRepo(child, u.value, { name: "work", visibility: "private" }); | |
| 35 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 36 | const c = await commitToBranch(child, u.value, repo.value, { message: "hosted work", files: [{ path: "a.txt", content: "kept\n" }] }); | |
| 37 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 38 | writeFileSync(join(child.config.dataDir, "secret.key"), "k".repeat(32), { mode: 0o600 }); | |
| 39 | mkdirSync(join(root, "parent", "spawn"), { recursive: true }); | |
| 40 | writeFileSync(join(root, "parent", "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: BODY, done: true })); | |
| 41 | // Not finished, and not a forge name: neither is kept. | |
| 42 | writeFileSync(join(root, "parent", "spawn", "half.json"), JSON.stringify({ name: "half", bodyId: BODY, done: false })); | |
| 43 | writeFileSync(join(root, "parent", "spawn", "Bad_Name.json"), JSON.stringify({ name: "Bad_Name", bodyId: BODY, done: true })); | |
| 44 | }); | |
| 45 | ||
| 46 | afterAll(async () => { | |
| 47 | child.close(); | |
| 48 | parent.close(); | |
| 49 | await rm(root, { recursive: true, force: true }); | |
| 50 | }); | |
| 51 | ||
| 52 | const take = (passphrase?: string) => { | |
| 53 | const store = dirStore(join(root, `store-${Math.random().toString(36).slice(2)}`)); | |
| 54 | return takeSnapshot({ db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase }, store, new Date("2026-11-16T03:00:00Z")).then((manifest) => ({ store, manifest })); | |
| 55 | }; | |
| 56 | ||
| 57 | test("only finished forges with a database are named", () => { | |
| 58 | expect(hostedSources(parent.config.dataDir).map((s) => s.name)).toEqual(["kid"]); | |
| 59 | }); | |
| 60 | ||
| 61 | test("the snapshot keeps the forge's database, repository, key and ledger under its name", async () => { | |
| 62 | const { manifest } = await take(PASS); | |
| 63 | const paths = manifest.files.map((f) => f.path); | |
| 64 | for (const p of ["children/kid/forge.db", "children/kid/ledger.json", "children/kid/secret.key.wrapped", "children/kid/repos/glint/work.bundle"]) | |
| 65 | expect(paths).toContain(p); | |
| 66 | expect(manifest.hosted).toHaveLength(1); | |
| 67 | expect(manifest.hosted?.[0]).toMatchObject({ name: "kid", repos: 1, key: "wrapped" }); | |
| 68 | expect(Number.isNaN(Date.parse(manifest.hosted![0]!.at))).toBe(false); | |
| 69 | }); | |
| 70 | ||
| 71 | test("without a passphrase the key stays home and the entry says so", async () => { | |
| 72 | const { manifest } = await take(); | |
| 73 | expect(manifest.hosted?.[0]?.key).toBe("none"); | |
| 74 | expect(manifest.files.some((f) => f.path.endsWith("secret.key.wrapped"))).toBe(false); | |
| 75 | }); | |
| 76 | ||
| 77 | test("one forge restores alone, with its history and key, and opens", async () => { | |
| 78 | const { store, manifest } = await take(PASS); | |
| 79 | const dir = join(root, "alone"); | |
| 80 | const got = await restoreHosted(store, manifest.name, "kid", dir, { passphrase: PASS }); | |
| 81 | expect(got.repos).toBe(1); | |
| 82 | expect(got.key).toBe("restored"); | |
| 83 | expect(JSON.parse(got.ledger).bodyId).toBe(BODY); | |
| 84 | expect(readFileSync(join(dir, "secret.key"), "utf8")).toBe("k".repeat(32)); | |
| 85 | const log = await commitLog(repoDir(join(dir, "repos"), "glint", "work"), "refs/heads/main", { limit: 5 }); | |
| 86 | expect(log.ok && log.value.map((c) => c.subject)).toContain("hosted work"); | |
| 87 | const opened = createApp({ dataDir: dir, port: 0 }); | |
| 88 | try { | |
| 89 | expect(opened.db.query("SELECT username FROM user").all()).toEqual([{ username: "glint" }]); | |
| 90 | } finally { | |
| 91 | opened.close(); | |
| 92 | } | |
| 93 | }); | |
| 94 | ||
| 95 | test("restore refuses a forge the snapshot does not hold, and a wrong passphrase", async () => { | |
| 96 | const { store, manifest } = await take(PASS); | |
| 97 | await expect(restoreHosted(store, manifest.name, "nobody", join(root, "x1"), { passphrase: PASS })).rejects.toThrow("does not hold nobody"); | |
| 98 | await expect(restoreHosted(store, manifest.name, "kid", join(root, "x2"), { passphrase: "wrong" })).rejects.toThrow("passphrase"); | |
| 99 | }); | |
| 100 | ||
| 101 | test("restoring the parent whole leaves the hosted forges for their own restore", async () => { | |
| 102 | const { store, manifest } = await take(PASS); | |
| 103 | const dir = join(root, "whole"); | |
| 104 | await restoreSnapshot(store, manifest.name, dir, { passphrase: PASS }); | |
| 105 | expect(existsSync(join(dir, "children"))).toBe(false); | |
| 106 | expect(existsSync(join(dir, "forge.db"))).toBe(true); | |
| 107 | }); | |
| 108 | ||
| 109 | test("a second snapshot of an unchanged forge sends no new chunks for it", async () => { | |
| 110 | const store = dirStore(join(root, "twice")); | |
| 111 | const f = { db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase: PASS }; | |
| 112 | await takeSnapshot(f, store, new Date("2026-11-16T03:00:00Z")); | |
| 113 | const second = await takeSnapshot(f, store, new Date("2026-11-17T03:00:00Z")); | |
| 114 | expect(second.sent?.chunks).toBeLessThan(3); | |
| 115 | }); | |
| 116 | ||
| 117 | test("the words say how many forges, and each forge's age and reason", () => { | |
| 118 | const at = "2026-11-16T03:00:00Z"; | |
| 119 | const now = Date.parse("2026-11-16T08:30:00Z"); | |
| 120 | const hosted = [{ name: "kid", at, repos: 1, bytes: 9, key: "wrapped" as const }, { name: "sad", at, repos: 0, bytes: 0, key: "none" as const, why: "x" }]; | |
| 121 | expect(hostedWords(hosted)).toBe("1 hosted forge in the newest backup; 1 left out (see the log)."); | |
| 122 | expect(hostedWords([])).toBe("No hosted forges to keep."); | |
| 123 | expect(hostedWords(undefined)).toBeUndefined(); | |
| 124 | expect(keptWords(hosted, "kid", now)).toBe("backed up 5 h ago"); | |
| 125 | expect(keptWords(hosted, "sad", now)).toBe("left out of the newest backup: x"); | |
| 126 | expect(keptWords(hosted, "new", now)).toBe("not in the newest backup"); | |
| 127 | expect(keptWords(undefined, "kid", now)).toBe("not in a backup yet"); | |
| 128 | }); | |
| 129 | ||
| 130 | test("noteHosted and readHosted round-trip, and a missing, corrupt or ill-shaped file reads as little as it can", () => { | |
| 131 | const d = join(root, "noted"); | |
| 132 | expect(readHosted(d)).toBeUndefined(); | |
| 133 | const one = { name: "kid", at: "2026-11-16T03:00:00Z", repos: 1, bytes: 9, key: "wrapped" as const }; | |
| 134 | noteHosted(d, [one, { ...one, name: "sad", key: "none", why: "x" }]); | |
| 135 | expect(readHosted(d)).toEqual([one, { ...one, name: "sad", key: "none", why: "x" }]); | |
| 136 | writeFileSync(join(d, "snapshot", "hosted.json"), "{not json"); | |
| 137 | expect(readHosted(d)).toBeUndefined(); | |
| 138 | writeFileSync(join(d, "snapshot", "hosted.json"), JSON.stringify({ name: "kid" })); | |
| 139 | expect(readHosted(d)).toBeUndefined(); | |
| 140 | writeFileSync(join(d, "snapshot", "hosted.json"), JSON.stringify([one, { name: "a", at: "t" }, { ...one, key: "other" }, { ...one, why: 3 }, null, { ...one, repos: "1" }])); | |
| 141 | expect(readHosted(d)).toEqual([one]); | |
| 142 | }); | |
| 143 | ||
| 144 | test("bodyIdOf names the data directory or nothing", () => { | |
| 145 | expect(bodyIdOf('{"bodyId":"abc_1"}')).toBe("abc_1"); | |
| 146 | expect(bodyIdOf('{"bodyId":"../x"}')).toBeUndefined(); | |
| 147 | expect(bodyIdOf('{"bodyId":3}')).toBeUndefined(); | |
| 148 | expect(bodyIdOf("")).toBeUndefined(); | |
| 149 | }); | |
| 150 | ||
| 151 | test("restoreHostedInto writes the forge under children/<bodyId> and its ledger under spawn/", async () => { | |
| 152 | const { store, manifest } = await take(PASS); | |
| 153 | const dir = join(root, "into"); | |
| 154 | const got = await restoreHostedInto(store, manifest.name, "kid", dir, { passphrase: PASS }); | |
| 155 | expect(got.dir).toBe(join(dir, "children", BODY)); | |
| 156 | expect(existsSync(join(dir, "children", BODY, "forge.db"))).toBe(true); | |
| 157 | expect(JSON.parse(readFileSync(join(dir, "spawn", "kid.json"), "utf8")).bodyId).toBe(BODY); | |
| 158 | }); | |
| 159 | ||
| 160 | test("restore says plainly when there is no snapshot", async () => { | |
| 161 | const store = dirStore(join(root, "empty-store")); | |
| 162 | await expect(restoreHostedInto(store, "2026-11-16T03-00-00Z", "kid", join(root, "none"))).rejects.toThrow("no snapshot 2026-11-16T03-00-00Z"); | |
| 163 | }); | |
| 164 | ||
| 165 | test("the --forge command restores one forge, and says in a sentence what it cannot", async () => { | |
| 166 | const storeDir = join(root, "cli-store"); | |
| 167 | const store = dirStore(storeDir); | |
| 168 | await takeSnapshot({ db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t", passphrase: PASS }, store, new Date("2026-11-16T03:00:00Z")); | |
| 169 | const run = async (...args: string[]) => { | |
| 170 | const p = Bun.spawn(["bun", "run", "src/snapshot/restore.ts", ...args], { | |
| 171 | env: { ...process.env, FORGE_SNAPSHOT_DIR: storeDir, FORGE_SNAPSHOT_PASSPHRASE: PASS }, | |
| 172 | stdout: "pipe", | |
| 173 | stderr: "pipe", | |
| 174 | }); | |
| 175 | const [out, err] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited]); | |
| 176 | return { code: p.exitCode, out, err }; | |
| 177 | }; | |
| 178 | const dir = join(root, "cli-out"); | |
| 179 | const ok = await run("latest", dir, "--forge", "kid"); | |
| 180 | expect(ok.code).toBe(0); | |
| 181 | expect(ok.out).toContain("restored kid from 2026-11-16T03-00-00Z: "); | |
| 182 | expect(existsSync(join(dir, "children", BODY, "forge.db"))).toBe(true); | |
| 183 | const nobody = await run("latest", join(root, "cli-out2"), "--forge", "nobody"); | |
| 184 | expect(nobody.code).toBe(1); | |
| 185 | expect(nobody.err).toContain("does not hold nobody"); | |
| 186 | const missing = await run("2020-01-01T00-00-00Z", join(root, "cli-out3"), "--forge", "kid"); | |
| 187 | expect(missing.code).toBe(1); | |
| 188 | expect(missing.err).toContain("no snapshot 2020-01-01T00-00-00Z"); | |
| 189 | expect(missing.err).not.toContain(" at "); | |
| 190 | expect((await run("latest", join(root, "cli-out4"), "--forge")).code).toBe(2); | |
| 191 | }); | |
| 192 | }); |
test/worker-database-opens.test.ts +1 −1
| @@ -65,7 +65,7 @@ const REACHED: Record<string, { opens: number; why: string }> = { | ||
| 65 | 65 | "src/release/rollback.ts": { opens: 1, why: "liveSchema reads the schema count once for a rollback or a swap, on the serving thread or in the release tool" }, |
| 66 | 66 | "src/snapshot/drill.ts": { opens: 1, why: "a restore drill opens the restored copy once, as a duty of the serving thread" }, |
| 67 | 67 | "src/snapshot/restore.ts": { opens: 1, why: "a restore reads the sealed names of the restored copy once, in the restore tool or a drill on the serving thread" }, |
| 68 | "src/snapshot/snapshot.ts": { opens: 1, why: "a snapshot counts the rows of its own copy once, as a duty of the serving thread; the mirror Worker imports this file for the manifest only" }, | |
| 68 | "src/snapshot/snapshot.ts": { opens: 2, why: "a snapshot counts the rows of its own copy once, and reads each hosted forge's copy once, as a duty of the serving thread; the mirror Worker imports this file for the manifest only" }, | |
| 69 | 69 | "src/spawn/child_runs.ts": { opens: 2, why: "called by the reader process (spawn/child_db_reader.ts), never by a thread of the forge; cell.ts re-exports the two for tests that read a file directly" }, |
| 70 | 70 | }; |
| 71 | 71 |
new above
offline