Public: anyone can read this; the crew can change it.
A hosted forge's backup carries everything it needs to come back whole, and the copy cannot be raced #14209
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14204-a-hosted-forge-s-backup-carries-everythi → main 2 commits fork 3 threads · 3 unresolved
5 files
changed +358
−12
docs/features/hosted-backup.md +4
| @@ -10,6 +10,10 @@ Each forge this one hosts goes into the nightly snapshot (#13819), under `childr | ||
| 10 | 10 | | Its key, wrapped under `FORGE_SNAPSHOT_PASSPHRASE` | `secret.key.wrapped` (without a passphrase the key stays home) | |
| 11 | 11 | | Every app's production data | alongside | |
| 12 | 12 | | This forge's own record of it | `ledger.json` | |
| 13 | | Its history: run logs, drop, cost ledger, `parent.json`, `plan.json` | `runs/`, `drop/`, `ledger/`, alongside | | |
| 14 | | What the host holds of it: credit and spend, the own-code switch | `parent/credit.json`, `parent/own-code.json`; `--forge` puts them back under `<data>/credit` and `<data>/own-code` | | |
| 15 | ||
| 16 | `parent/credit.json` holds the forge's `tokenHash`; it is backed up like everything else and is covered by the same encryption: a snapshot kept on a sealed lane is sealed chunk by chunk, this file with the rest. Files are opened without following a link and only if regular; a link, a pipe, a file that changed while it was copied, or history past the 512 MiB cap on `runs/`, `drop/` and `ledger/` together is left out and named in the manifest's `omitted` list (first 100, `omittedTotal` counts all). A restore refuses a `parent/` entry it has no place for. | |
| 13 | 17 | |
| 14 | 18 | Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends nothing. A forge that cannot be read is named in the manifest's `hosted` list and the log; the snapshot goes on without it. |
| 15 | 19 |
src/snapshot/hosted.ts +149 −5
| @@ -20,7 +20,7 @@ | ||
| 20 | 20 | |
| 21 | 21 | import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, readSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; |
| 22 | 22 | import { lstat, readdir } from "node:fs/promises"; |
| 23 | import { join, resolve, sep } from "node:path"; | |
| 23 | import { dirname, join, resolve, sep } from "node:path"; | |
| 24 | 24 | import { runGit } from "../git/env.ts"; |
| 25 | 25 | import { findingWords, scanRepos } from "../spawn/move_scan.ts"; |
| 26 | 26 | import { CHILD_NAME } from "../spawn/placement.ts"; |
| @@ -41,6 +41,18 @@ export interface HostedEntry { | ||
| 41 | 41 | readonly why?: string; |
| 42 | 42 | /** Repositories of it this snapshot holds no bundle of, each with why; absent when none. */ |
| 43 | 43 | readonly absent?: readonly HostedAbsent[]; |
| 44 | /** | |
| 45 | * History and host files this snapshot left out, each by name with why (at most | |
| 46 | * `OMITTED_LISTED`). | |
| 47 | */ | |
| 48 | readonly omitted?: readonly HostedOmitted[]; | |
| 49 | /** How many were left out in all, when more than are listed. */ | |
| 50 | readonly omittedTotal?: number; | |
| 51 | } | |
| 52 | ||
| 53 | export interface HostedOmitted { | |
| 54 | readonly path: string; | |
| 55 | readonly why: string; | |
| 44 | 56 | } |
| 45 | 57 | |
| 46 | 58 | /** A repository a hosted forge's copy holds no bundle of, and why. `empty`: it has no refs. */ |
| @@ -54,8 +66,17 @@ export interface HostedSource { | ||
| 54 | 66 | readonly name: string; |
| 55 | 67 | readonly dir: string; |
| 56 | 68 | readonly ledger: string; |
| 69 | /** The parent's own files about the forge: its credit and its own-code switch. */ | |
| 70 | readonly parentFiles?: readonly { readonly path: string; readonly file: string }[]; | |
| 57 | 71 | } |
| 58 | 72 | |
| 73 | /** Where the parent's files about a hosted forge sit in its snapshot, by what they are. */ | |
| 74 | export const PARENT_SIDE = "parent/"; | |
| 75 | export const PARENT_FILES: readonly string[] = [`${PARENT_SIDE}credit.json`, `${PARENT_SIDE}own-code.json`]; | |
| 76 | /** What a forge keeps in its own directory beside its database: history and records, whole. */ | |
| 77 | export const HISTORY_DIRS = ["runs", "drop", "ledger"] as const; | |
| 78 | export const HISTORY_FILES = ["parent.json", "plan.json"] as const; | |
| 79 | ||
| 59 | 80 | const BODY_ID = /^[A-Za-z0-9_-]{1,64}$/; |
| 60 | 81 | |
| 61 | 82 | /** The data directory name a forge's ledger text names, or undefined when it names none. */ |
| @@ -90,7 +111,16 @@ export function hostedSources(dataDir: string): HostedSource[] { | ||
| 90 | 111 | const dir = join(dataDir, "children", bodyId); |
| 91 | 112 | // Whatever is at the name, unfollowed: one that is not the forge's own file is refused by |
| 92 | 113 | // the copy, by name, rather than passed over in silence. |
| 93 | if (lstatSync(join(dir, "forge.db"), { throwIfNoEntry: false }) !== undefined) out.push({ name, dir, ledger: text }); | |
| 114 | if (lstatSync(join(dir, "forge.db"), { throwIfNoEntry: false }) !== undefined) | |
| 115 | out.push({ | |
| 116 | name, | |
| 117 | dir, | |
| 118 | ledger: text, | |
| 119 | parentFiles: [ | |
| 120 | { path: `${PARENT_SIDE}credit.json`, file: join(dataDir, "credit", `${name}.json`) }, | |
| 121 | { path: `${PARENT_SIDE}own-code.json`, file: join(dataDir, "own-code", `${name}.json`) }, | |
| 122 | ], | |
| 123 | }); | |
| 94 | 124 | } catch { |
| 95 | 125 | // A ledger that does not read is not a forge to keep. |
| 96 | 126 | } |
| @@ -176,7 +206,12 @@ export const REFETCHED_REFS = ["refs/upstream/"] as const; | ||
| 176 | 206 | export class HostedRefusal extends Error {} |
| 177 | 207 | |
| 178 | 208 | /** What `copyHosted` returns: the part of a `HostedEntry` only the copy knows. */ |
| 179 | export type HostedCopy = Pick<HostedEntry, "repos" | "key" | "absent">; | |
| 209 | export type HostedCopy = Pick<HostedEntry, "repos" | "key" | "absent" | "omitted" | "omittedTotal">; | |
| 210 | ||
| 211 | /** Names of left-out files a manifest lists; the rest are counted. */ | |
| 212 | export const OMITTED_LISTED = 100; | |
| 213 | /** Bytes of runs/, drop/ and ledger/ together a snapshot of one forge carries. */ | |
| 214 | export const HISTORY_BYTES_MAX = 512 * 1_048_576; | |
| 180 | 215 | |
| 181 | 216 | type Cut = "content" | "fixed" | "bundle"; |
| 182 | 217 | |
| @@ -188,7 +223,7 @@ export interface HostedTools { | ||
| 188 | 223 | /** |
| 189 | 224 | * A file as chunks; with `own`, opened without following a link and only if it is inside `own`. |
| 190 | 225 | */ |
| 191 | readonly putFile: (path: string, file: string, cut?: Cut, own?: string) => Promise<void>; | |
| 226 | readonly putFile: (path: string, file: string, cut?: Cut, own?: string) => Promise<boolean | void>; | |
| 192 | 227 | readonly putBytes: (path: string, body: Uint8Array, cut?: Cut) => Promise<void>; |
| 193 | 228 | readonly bundleDeadlineMs?: ((bytes: number) => number) | undefined; |
| 194 | 229 | /** Test seam: the database copy. */ |
| @@ -197,6 +232,10 @@ export interface HostedTools { | ||
| 197 | 232 | readonly git?: typeof runGit | undefined; |
| 198 | 233 | /** The most files read from the forge's directory; `HOSTED_FILES_MAX` unless a test lowers it. */ |
| 199 | 234 | readonly filesMax?: number | undefined; |
| 235 | /** | |
| 236 | * The most bytes of history (runs, drop, ledger); `HISTORY_BYTES_MAX` unless a test lowers it. | |
| 237 | */ | |
| 238 | readonly historyMax?: number | undefined; | |
| 200 | 239 | } |
| 201 | 240 | |
| 202 | 241 | const SEGMENT = /^[A-Za-z0-9._-]{1,100}$/; |
| @@ -294,6 +333,55 @@ async function filesInside(dir: string, left: { n: number; readonly max: number | ||
| 294 | 333 | return out; |
| 295 | 334 | } |
| 296 | 335 | |
| 336 | /** | |
| 337 | * Whether a file read as `bytes` is the one that was there when it was `before`: the same file, the | |
| 338 | * same size and time, and as long as it was said to be. A log written to while it was copied is | |
| 339 | * not. | |
| 340 | */ | |
| 341 | export function unchanged( | |
| 342 | before: { ino: number; size: number; mtimeMs: number }, | |
| 343 | after: { ino: number; size: number; mtimeMs: number } | undefined, | |
| 344 | bytes: number, | |
| 345 | ): boolean { | |
| 346 | return after !== undefined && after.ino === before.ino && after.size === before.size && after.mtimeMs === before.mtimeMs && bytes === before.size; | |
| 347 | } | |
| 348 | ||
| 349 | /** | |
| 350 | * Every regular file under `dir` for the history copy, and by name what is not one: a link, a pipe, | |
| 351 | * a directory that cannot be listed. Nothing is followed. | |
| 352 | */ | |
| 353 | async function historyIn( | |
| 354 | dir: string, | |
| 355 | name: string, | |
| 356 | left: { n: number; readonly max: number }, | |
| 357 | ): Promise<{ files: { path: string; size: number }[]; omitted: HostedOmitted[] }> { | |
| 358 | const files: { path: string; size: number }[] = []; | |
| 359 | const omitted: HostedOmitted[] = []; | |
| 360 | const top = await lstat(dir).catch((e: NodeJS.ErrnoException) => { | |
| 361 | if (e.code === "ENOENT") return undefined; | |
| 362 | throw e; | |
| 363 | }); | |
| 364 | if (top === undefined) return { files, omitted }; | |
| 365 | if (!top.isDirectory()) return { files, omitted: [{ path: name, why: "it is not a directory of its own" }] }; | |
| 366 | const walk = async (at: string, rel: string): Promise<void> => { | |
| 367 | const entries = await readdir(at, { withFileTypes: true }); | |
| 368 | for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) { | |
| 369 | const path = join(at, entry.name); | |
| 370 | const shown = `${name}/${rel === "" ? "" : rel + "/"}${entry.name}`; | |
| 371 | const here = rel === "" ? entry.name : `${rel}/${entry.name}`; | |
| 372 | if (entry.isDirectory()) await walk(path, here); | |
| 373 | else if (entry.isFile()) { | |
| 374 | if (--left.n < 0) throw new HostedRefusal(`it holds more than ${left.max} files to copy`); | |
| 375 | const s = await lstat(path); | |
| 376 | if (s.isFile()) files.push({ path: here, size: s.size }); | |
| 377 | else omitted.push({ path: shown, why: "it changed into something else while it was listed" }); | |
| 378 | } else omitted.push({ path: shown, why: entry.isSymbolicLink() ? "it is a link" : "it is not a regular file" }); | |
| 379 | } | |
| 380 | }; | |
| 381 | await walk(dir, ""); | |
| 382 | return { files, omitted }; | |
| 383 | } | |
| 384 | ||
| 297 | 385 | /** The directories directly under `dir` that are directories themselves, by name. */ |
| 298 | 386 | async function dirsIn(dir: string): Promise<string[]> { |
| 299 | 387 | if ((await lstat(dir).catch(() => undefined))?.isDirectory() !== true) return []; |
| @@ -447,6 +535,56 @@ export async function copyHosted(child: HostedSource, tools: HostedTools): Promi | ||
| 447 | 535 | } |
| 448 | 536 | for (const f of await filesInside(join(root, "ssh"), left)) await tools.putFile(`ssh/${f.path}`, join(root, "ssh", f.path), "content", root); |
| 449 | 537 | |
| 538 | // Its history and records: run logs, its drop, its cost ledger, what its parent wrote of it. A | |
| 539 | // file is read once and again if it changed under the read; what still changes, what is not a | |
| 540 | // file of its own and what goes past the byte cap is left out by name. | |
| 541 | const omitted: HostedOmitted[] = []; | |
| 542 | const put = async (at: string, file: string): Promise<boolean> => { | |
| 543 | for (let tries = 0; tries < 2; tries++) if ((await tools.putFile(at, file, "content", root)) !== false) return true; | |
| 544 | return false; | |
| 545 | }; | |
| 546 | let room = tools.historyMax ?? HISTORY_BYTES_MAX; | |
| 547 | for (const d of HISTORY_DIRS) { | |
| 548 | const found = await historyIn(join(root, d), d, left); | |
| 549 | omitted.push(...found.omitted); | |
| 550 | for (const f of found.files) { | |
| 551 | if (f.size > room) { | |
| 552 | omitted.push({ path: `${d}/${f.path}`, why: `past the ${tools.historyMax ?? HISTORY_BYTES_MAX} byte cap on history` }); | |
| 553 | continue; | |
| 554 | } | |
| 555 | if (await put(`${d}/${f.path}`, join(root, d, f.path))) room -= f.size; | |
| 556 | else omitted.push({ path: `${d}/${f.path}`, why: "it changed while it was copied" }); | |
| 557 | } | |
| 558 | } | |
| 559 | const small = (file: string, base: string, path: string): Uint8Array | undefined => { | |
| 560 | let s: import("node:fs").Stats | undefined; | |
| 561 | try { | |
| 562 | s = lstatSync(file, { throwIfNoEntry: false }); | |
| 563 | } catch (e) { | |
| 564 | throw new HostedRefusal(`${path} could not be read: ${String((e as Error).message).slice(0, 100)}`); | |
| 565 | } | |
| 566 | if (s === undefined) return undefined; | |
| 567 | const body = s.isFile() && s.size <= 1_048_576 ? readInside(file, base) : undefined; | |
| 568 | if (body === undefined) omitted.push({ path, why: "it is not a regular file of its own, or is over 1 MiB" }); | |
| 569 | return body; | |
| 570 | }; | |
| 571 | for (const name of HISTORY_FILES) { | |
| 572 | const body = small(join(root, name), root, name); | |
| 573 | if (body !== undefined) await tools.putBytes(name, body, "content"); | |
| 574 | } | |
| 575 | // What this host holds of it: its credit and spend, the own-code switch. The host's own files. | |
| 576 | for (const p of child.parentFiles ?? []) { | |
| 577 | let base: string; | |
| 578 | try { | |
| 579 | base = realpathSync(dirname(p.file)); | |
| 580 | } catch (e) { | |
| 581 | if ((e as NodeJS.ErrnoException).code === "ENOENT") continue; | |
| 582 | throw e; | |
| 583 | } | |
| 584 | const body = small(p.file, base, p.path); | |
| 585 | if (body !== undefined) await tools.putBytes(p.path, body, "content"); | |
| 586 | } | |
| 587 | ||
| 450 | 588 | // Every repository on its disk, and every one its database lists: a bundle, or a reason. |
| 451 | 589 | const reposRoot = join(root, "repos"); |
| 452 | 590 | const onDisk = new Set<string>(); |
| @@ -526,5 +664,11 @@ export async function copyHosted(child: HostedSource, tools: HostedTools): Promi | ||
| 526 | 664 | } |
| 527 | 665 | } |
| 528 | 666 | |
| 529 | return { repos: bundled, key, ...(absent.length === 0 ? {} : { absent }) }; | |
| 667 | return { | |
| 668 | repos: bundled, | |
| 669 | key, | |
| 670 | ...(absent.length === 0 ? {} : { absent }), | |
| 671 | ...(omitted.length === 0 ? {} : { omitted: omitted.slice(0, OMITTED_LISTED) }), | |
| 672 | ...(omitted.length > OMITTED_LISTED ? { omittedTotal: omitted.length } : {}), | |
| 673 | }; | |
| 530 | 674 | } |
src/snapshot/restore.ts +21 −3
| @@ -55,7 +55,7 @@ import { | ||
| 55 | 55 | type Manifest, |
| 56 | 56 | type SnapshotFile, |
| 57 | 57 | } from "./snapshot.ts"; |
| 58 | import { bodyIdOf, HOSTED_PREFIX } from "./hosted.ts"; | |
| 58 | import { bodyIdOf, HOSTED_PREFIX, PARENT_FILES, PARENT_SIDE } from "./hosted.ts"; | |
| 59 | 59 | import { chunkKey, isChunkId, openChunk } from "./chunks.ts"; |
| 60 | 60 | import { unwrapKey, WRAPPED_KEY } from "./wrap.ts"; |
| 61 | 61 | import { isPackedFile, partPaths, safePath, unpacker, type PackedFile, type Unpacker } from "./parts.ts"; |
| @@ -601,6 +601,10 @@ export interface RestoredHosted { | ||
| 601 | 601 | readonly dir: string; |
| 602 | 602 | /** The parent's ledger line for it, to be written to `<data>/spawn/<name>.json`. */ |
| 603 | 603 | readonly ledger: string; |
| 604 | /** | |
| 605 | * The host's own files about it (`parent/credit.json`, `parent/own-code.json`), by snapshot path. | |
| 606 | */ | |
| 607 | readonly parentFiles: ReadonlyMap<string, Uint8Array>; | |
| 604 | 608 | readonly files: number; |
| 605 | 609 | readonly repos: number; |
| 606 | 610 | readonly key: "restored" | "missing"; |
| @@ -642,7 +646,11 @@ export async function restoreHosted( | ||
| 642 | 646 | key = unwrapKey(await readWhole(store, snapshot, keyEntry), options.passphrase); |
| 643 | 647 | if (key === undefined) throw new Error("the passphrase does not open this forge's wrapped key"); |
| 644 | 648 | } |
| 645 | const files = mine.filter((f) => f.path !== "ledger.json" && f.path !== WRAPPED_KEY); | |
| 649 | const parentFiles = new Map<string, Uint8Array>(); | |
| 650 | for (const f of mine) if (f.path.startsWith(PARENT_SIDE) && !PARENT_FILES.includes(f.path)) | |
| 651 | throw new Error(`the snapshot holds ${f.path} of this forge, which this restore does not know where to put`); | |
| 652 | for (const f of mine) if (f.path.startsWith(PARENT_SIDE)) parentFiles.set(f.path, await readWhole(store, snapshot, f)); | |
| 653 | const files = mine.filter((f) => f.path !== "ledger.json" && f.path !== WRAPPED_KEY && !f.path.startsWith(PARENT_SIDE)); | |
| 646 | 654 | const got = await fetchChunked(store, snapshot, files, dir, () => {}); |
| 647 | 655 | const git = options.git ?? runGit; |
| 648 | 656 | for (const b of got.bundles) { |
| @@ -659,7 +667,9 @@ export async function restoreHosted( | ||
| 659 | 667 | await Bun.write(join(dir, "secret.key"), key); |
| 660 | 668 | chmodSync(join(dir, "secret.key"), 0o600); |
| 661 | 669 | } |
| 662 | return { name: forge, dir, ledger: ledgerText, files: got.files, repos: got.bundles.length, key: key === undefined ? "missing" : "restored" }; | |
| 670 | // What its parent wrote of it was private to it, and is again. | |
| 671 | if (existsSync(join(dir, "parent.json"))) chmodSync(join(dir, "parent.json"), 0o600); | |
| 672 | return { name: forge, dir, ledger: ledgerText, parentFiles, files: got.files, repos: got.bundles.length, key: key === undefined ? "missing" : "restored" }; | |
| 663 | 673 | } |
| 664 | 674 | |
| 665 | 675 | /** |
| @@ -675,6 +685,14 @@ export async function restoreHostedInto( | ||
| 675 | 685 | const one = await restoreHosted(store, snapshot, forge, (id) => join(dataDir, "children", id), options); |
| 676 | 686 | mkdirSync(join(dataDir, "spawn"), { recursive: true }); |
| 677 | 687 | await Bun.write(join(dataDir, "spawn", `${forge}.json`), one.ledger); |
| 688 | // Its credit and spend, and the own-code switch, where this host keeps them. | |
| 689 | for (const [where, file] of [["credit", "credit.json"], ["own-code", "own-code.json"]] as const) { | |
| 690 | const body = one.parentFiles.get(`${PARENT_SIDE}${file}`); | |
| 691 | if (body === undefined) continue; | |
| 692 | mkdirSync(join(dataDir, where), { recursive: true }); | |
| 693 | await Bun.write(join(dataDir, where, `${forge}.json`), body); | |
| 694 | chmodSync(join(dataDir, where, `${forge}.json`), 0o600); | |
| 695 | } | |
| 678 | 696 | return one; |
| 679 | 697 | } |
| 680 | 698 |
src/snapshot/snapshot.ts +11 −4
| @@ -37,7 +37,7 @@ | ||
| 37 | 37 | |
| 38 | 38 | import { Database } from "bun:sqlite"; |
| 39 | 39 | import { vacuumFile, vacuumInto } from "./vacuum.ts"; |
| 40 | import { createReadStream, mkdtempSync, readdirSync, rmSync, statSync } from "node:fs"; | |
| 40 | import { createReadStream, fstatSync, lstatSync, mkdtempSync, readdirSync, rmSync, statSync } from "node:fs"; | |
| 41 | 41 | import { tmpdir } from "node:os"; |
| 42 | 42 | import { join, relative } from "node:path"; |
| 43 | 43 | import { envDataDir } from "../apps/data.ts"; |
| @@ -46,7 +46,7 @@ import { listEveryRepo } from "../core/repos.ts"; | ||
| 46 | 46 | import { keepLentObjects, repoDir } from "../git/repos.ts"; |
| 47 | 47 | import { createLog } from "../kernel/log.ts"; |
| 48 | 48 | import { bundleRepo } from "./bundle.ts"; |
| 49 | import { copyHosted, HostedRefusal, hostedSources, HOSTED_PREFIX, openInside, type HostedEntry, type HostedTools } from "./hosted.ts"; | |
| 49 | import { copyHosted, unchanged, HostedRefusal, hostedSources, HOSTED_PREFIX, openInside, type HostedEntry, type HostedTools } from "./hosted.ts"; | |
| 50 | 50 | import { countRows, type DrillCounts } from "./counts.ts"; |
| 51 | 51 | import { CHUNK_PREFIX, chunkId, chunkKey, chunkSeed, contentChunker, fixedChunker, isChunkId, packChunk, type Chunker } from "./chunks.ts"; |
| 52 | 52 | import type { PackedFile } from "./parts.ts"; |
| @@ -330,6 +330,7 @@ export interface Snapshotter { | ||
| 330 | 330 | /** Test seams: the git runner a hosted forge's repositories are read with, and its file limit. */ |
| 331 | 331 | readonly hostedGit?: HostedTools["git"]; |
| 332 | 332 | readonly hostedFilesMax?: HostedTools["filesMax"]; |
| 333 | readonly hostedHistoryMax?: HostedTools["historyMax"]; | |
| 333 | 334 | } |
| 334 | 335 | |
| 335 | 336 | /** |
| @@ -505,9 +506,10 @@ export async function takeSnapshot( | ||
| 505 | 506 | // chunks. |
| 506 | 507 | // With `own`, a hosted forge's file (`hosted.ts`): opened without following a link, and only when |
| 507 | 508 | // what was opened is inside that directory. |
| 508 | const putFile = async (path: string, file: string, cut: Cut = "content", own?: string): Promise<void> => { | |
| 509 | const putFile = async (path: string, file: string, cut: Cut = "content", own?: string): Promise<boolean> => { | |
| 509 | 510 | const fd = own === undefined ? undefined : openInside(file, own); |
| 510 | if (own === undefined ? statSync(file, { throwIfNoEntry: false })?.isFile() !== true : fd === undefined) return; | |
| 511 | if (own === undefined ? statSync(file, { throwIfNoEntry: false })?.isFile() !== true : fd === undefined) return true; | |
| 512 | const before = fd === undefined ? undefined : fstatSync(fd); | |
| 511 | 513 | const cutter = chunker(cut); |
| 512 | 514 | const whole = new Bun.CryptoHasher("sha256"); |
| 513 | 515 | const chunks: string[] = []; |
| @@ -518,7 +520,11 @@ export async function takeSnapshot( | ||
| 518 | 520 | for (const c of cutter.push(part)) chunks.push(await send(c)); |
| 519 | 521 | } |
| 520 | 522 | for (const c of cutter.end()) chunks.push(await send(c)); |
| 523 | // A hosted forge's file that changed while it was read is not listed: false, and the caller | |
| 524 | // says so. | |
| 525 | if (before !== undefined && !unchanged(before, lstatSync(file, { throwIfNoEntry: false }), bytes)) return false; | |
| 521 | 526 | listed(path, bytes, whole.digest("hex"), chunks); |
| 527 | return true; | |
| 522 | 528 | }; |
| 523 | 529 | const putBytes = async (path: string, body: Uint8Array, cut: Cut = "content"): Promise<void> => { |
| 524 | 530 | const cutter = chunker(cut); |
| @@ -653,6 +659,7 @@ export async function takeSnapshot( | ||
| 653 | 659 | copy: forge.hostedCopy, |
| 654 | 660 | git: forge.hostedGit, |
| 655 | 661 | filesMax: forge.hostedFilesMax, |
| 662 | historyMax: forge.hostedHistoryMax, | |
| 656 | 663 | }); |
| 657 | 664 | for (const a of took.absent ?? []) |
| 658 | 665 | if (a.empty !== true) log.error("hosted repository left out of the snapshot", { forge: child.name, repo: a.repo, why: a.why }); |
test/snapshot-hosted-history.test.ts added +173
| @@ -0,0 +1,173 @@ | ||
| 1 | /** | |
| 2 | * A hosted forge restored from the backup comes back with its history (#14204): run logs, its drop, | |
| 3 | * its cost ledger, `parent.json`, `plan.json`, and what its host holds of it (credit, own-code). | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, statSync, symlinkSync, writeFileSync } from "node:fs"; | |
| 7 | import { rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { copyHosted, unchanged } from "../src/snapshot/hosted.ts"; | |
| 12 | import { restoreHostedInto } from "../src/snapshot/restore.ts"; | |
| 13 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; | |
| 14 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 15 | ||
| 16 | let root: string; | |
| 17 | let parent: App; | |
| 18 | let child: App; | |
| 19 | ||
| 20 | beforeAll(() => { | |
| 21 | root = mkdtempSync(join(tmpdir(), "forge-hosted-history-")); | |
| 22 | parent = createApp({ dataDir: join(root, "parent"), port: 0 }); | |
| 23 | const dir = join(root, "parent", "children", "b1"); | |
| 24 | child = createApp({ dataDir: dir, port: 0 }); | |
| 25 | for (const d of ["runs/r1", "drop", "ledger"]) mkdirSync(join(dir, d), { recursive: true }); | |
| 26 | writeFileSync(join(dir, "runs", "r1", "log.txt"), "ran the thing\n"); | |
| 27 | writeFileSync(join(dir, "drop", "note.txt"), "dropped\n"); | |
| 28 | writeFileSync(join(dir, "ledger", "2026-11.jsonl"), '{"usd":1.25}\n'); | |
| 29 | writeFileSync(join(dir, "parent.json"), '{"name":"kid"}', { mode: 0o600 }); | |
| 30 | writeFileSync(join(dir, "plan.json"), '{"plan":"free"}'); | |
| 31 | mkdirSync(join(root, "parent", "spawn"), { recursive: true }); | |
| 32 | writeFileSync(join(root, "parent", "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "b1", done: true })); | |
| 33 | for (const [d, body] of [["credit", '{"name":"kid","capUsd":10,"spentUsd":3.5,"tokenHash":"h"}'], ["own-code", '{"on":true}']] as const) { | |
| 34 | mkdirSync(join(root, "parent", d), { recursive: true }); | |
| 35 | writeFileSync(join(root, "parent", d, "kid.json"), body); | |
| 36 | } | |
| 37 | }); | |
| 38 | ||
| 39 | afterAll(async () => { | |
| 40 | child.close(); | |
| 41 | parent.close(); | |
| 42 | await rm(root, { recursive: true, force: true }); | |
| 43 | }); | |
| 44 | ||
| 45 | test("a restored hosted forge has its runs, drop, ledger, records, spend and switch", async () => { | |
| 46 | const store = dirStore(join(root, "store")); | |
| 47 | const m = await takeSnapshot({ db: parent.db, dataDir: parent.config.dataDir, reposRoot: parent.reposRoot, version: "t" }, store, new Date("2026-11-17T03:00:00Z")); | |
| 48 | expect(m.hosted?.[0]?.why).toBeUndefined(); | |
| 49 | const back = join(root, "back"); | |
| 50 | await restoreHostedInto(store, m.name, "kid", back); | |
| 51 | const at = join(back, "children", "b1"); | |
| 52 | expect(readFileSync(join(at, "runs", "r1", "log.txt"), "utf8")).toBe("ran the thing\n"); | |
| 53 | expect(readFileSync(join(at, "drop", "note.txt"), "utf8")).toBe("dropped\n"); | |
| 54 | expect(readFileSync(join(at, "ledger", "2026-11.jsonl"), "utf8")).toContain("1.25"); | |
| 55 | expect(readFileSync(join(at, "plan.json"), "utf8")).toContain("free"); | |
| 56 | expect(statSync(join(at, "parent.json")).mode & 0o077).toBe(0); | |
| 57 | expect(JSON.parse(readFileSync(join(back, "credit", "kid.json"), "utf8")).spentUsd).toBe(3.5); | |
| 58 | expect(readFileSync(join(back, "own-code", "kid.json"), "utf8")).toContain("true"); | |
| 59 | }); | |
| 60 | ||
| 61 | /** A host with one hosted forge `kid` laid out by `lay`, snapshotted into a directory store. */ | |
| 62 | async function snapWith(lay: (dir: string, host: string) => void, more: Partial<Parameters<typeof takeSnapshot>[0]> = {}) { | |
| 63 | const base = mkdtempSync(join(tmpdir(), "forge-hosted-history-x-")); | |
| 64 | const host = createApp({ dataDir: join(base, "parent"), port: 0 }); | |
| 65 | const dir = join(base, "parent", "children", "b1"); | |
| 66 | const open = createApp({ dataDir: dir, port: 0 }); | |
| 67 | mkdirSync(join(base, "parent", "spawn"), { recursive: true }); | |
| 68 | writeFileSync(join(base, "parent", "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "b1", done: true })); | |
| 69 | lay(dir, join(base, "parent")); | |
| 70 | const store = dirStore(join(base, "store")); | |
| 71 | const m = await takeSnapshot({ db: host.db, dataDir: host.config.dataDir, reposRoot: host.reposRoot, version: "t", ...more }, store, new Date("2026-11-17T03:00:00Z")); | |
| 72 | open.close(); | |
| 73 | host.close(); | |
| 74 | return { base, store, m, paths: m.files.map((f) => f.path) }; | |
| 75 | } | |
| 76 | ||
| 77 | test("a link in runs/ or drop/ is refused by name and what it points at is not kept", async () => { | |
| 78 | const { base, m, paths } = await snapWith((dir) => { | |
| 79 | mkdirSync(join(dir, "runs"), { recursive: true }); | |
| 80 | mkdirSync(join(dir, "drop"), { recursive: true }); | |
| 81 | writeFileSync(join(dir, "..", "..", "secret.txt"), "outside"); | |
| 82 | symlinkSync(join(dir, "..", "..", "secret.txt"), join(dir, "runs", "evil.log")); | |
| 83 | symlinkSync(join(dir, "..", ".."), join(dir, "drop", "up")); | |
| 84 | writeFileSync(join(dir, "drop", "fine.txt"), "ok"); | |
| 85 | }); | |
| 86 | const names = (m.hosted?.[0]?.omitted ?? []).map((o) => o.path).sort(); | |
| 87 | expect(names).toEqual(["drop/up", "runs/evil.log"]); | |
| 88 | expect(paths).toContain("children/kid/drop/fine.txt"); | |
| 89 | expect(paths.some((p) => p.includes("evil") || p.includes("secret"))).toBe(false); | |
| 90 | await rm(base, { recursive: true, force: true }); | |
| 91 | }); | |
| 92 | ||
| 93 | test("a forge with no credit, own-code, plan or history still snapshots and restores", async () => { | |
| 94 | const { base, store, m } = await snapWith(() => {}); | |
| 95 | expect(m.hosted?.[0]?.why).toBeUndefined(); | |
| 96 | expect(m.hosted?.[0]?.omitted).toBeUndefined(); | |
| 97 | await restoreHostedInto(store, m.name, "kid", join(base, "back")); | |
| 98 | expect(existsSync(join(base, "back", "credit"))).toBe(false); | |
| 99 | await rm(base, { recursive: true, force: true }); | |
| 100 | }); | |
| 101 | ||
| 102 | test("a credit file that is a link is left out by name, not read", async () => { | |
| 103 | const { base, m } = await snapWith((_dir, host) => { | |
| 104 | mkdirSync(join(host, "credit"), { recursive: true }); | |
| 105 | writeFileSync(join(host, "elsewhere.json"), "{}"); | |
| 106 | symlinkSync(join(host, "elsewhere.json"), join(host, "credit", "kid.json")); | |
| 107 | }); | |
| 108 | expect(m.files.some((f) => f.path.endsWith("parent/credit.json"))).toBe(false); | |
| 109 | expect(m.hosted?.[0]?.omitted?.map((o) => o.path)).toEqual(["parent/credit.json"]); | |
| 110 | await rm(base, { recursive: true, force: true }); | |
| 111 | }); | |
| 112 | ||
| 113 | test("history past the byte cap is left out by name", async () => { | |
| 114 | const { base, m, paths } = await snapWith( | |
| 115 | (dir) => { | |
| 116 | mkdirSync(join(dir, "runs"), { recursive: true }); | |
| 117 | writeFileSync(join(dir, "runs", "a.log"), "x".repeat(10)); | |
| 118 | writeFileSync(join(dir, "runs", "b.log"), "y".repeat(100)); | |
| 119 | }, | |
| 120 | { hostedHistoryMax: 50 }, | |
| 121 | ); | |
| 122 | expect(paths).toContain("children/kid/runs/a.log"); | |
| 123 | expect(paths).not.toContain("children/kid/runs/b.log"); | |
| 124 | expect(m.hosted?.[0]?.omitted?.map((o) => o.path)).toEqual(["runs/b.log"]); | |
| 125 | await rm(base, { recursive: true, force: true }); | |
| 126 | }); | |
| 127 | ||
| 128 | test("a file that changed while it was read is not the one that was listed", async () => { | |
| 129 | const was = { ino: 1, size: 10, mtimeMs: 5 }; | |
| 130 | expect(unchanged(was, { ...was }, 10)).toBe(true); | |
| 131 | expect(unchanged(was, { ...was, size: 12 }, 10)).toBe(false); | |
| 132 | expect(unchanged(was, { ...was, mtimeMs: 6 }, 10)).toBe(false); | |
| 133 | expect(unchanged(was, { ...was, ino: 2 }, 10)).toBe(false); | |
| 134 | expect(unchanged(was, was, 12)).toBe(false); | |
| 135 | expect(unchanged(was, undefined, 10)).toBe(false); | |
| 136 | }); | |
| 137 | ||
| 138 | test("copyHosted names a history file whose copy keeps changing, and keeps the rest", async () => { | |
| 139 | const base = mkdtempSync(join(tmpdir(), "forge-hosted-history-c-")); | |
| 140 | const dir = join(base, "b1"); | |
| 141 | const open = createApp({ dataDir: dir, port: 0 }); | |
| 142 | mkdirSync(join(dir, "runs"), { recursive: true }); | |
| 143 | writeFileSync(join(dir, "runs", "live.log"), "a"); | |
| 144 | writeFileSync(join(dir, "runs", "done.log"), "b"); | |
| 145 | open.close(); | |
| 146 | const put: string[] = []; | |
| 147 | const got = await copyHosted( | |
| 148 | { name: "kid", dir, ledger: "{}" }, | |
| 149 | { | |
| 150 | scratch: base, | |
| 151 | putFile: async (path) => { | |
| 152 | if (path === "runs/live.log") return false; | |
| 153 | put.push(path); | |
| 154 | }, | |
| 155 | putBytes: async () => {}, | |
| 156 | }, | |
| 157 | ); | |
| 158 | expect(got.omitted).toEqual([{ path: "runs/live.log", why: "it changed while it was copied" }]); | |
| 159 | expect(put).toContain("runs/done.log"); | |
| 160 | await rm(base, { recursive: true, force: true }); | |
| 161 | }); | |
| 162 | ||
| 163 | test("a restore refuses a parent/ entry it has no place for", async () => { | |
| 164 | const { base, store, m } = await snapWith((_dir, host) => { | |
| 165 | mkdirSync(join(host, "credit"), { recursive: true }); | |
| 166 | writeFileSync(join(host, "credit", "kid.json"), "{}"); | |
| 167 | }); | |
| 168 | const file = join(base, "store", "snapshots", m.name, "manifest.json"); | |
| 169 | const text = readFileSync(file, "utf8").replace("children/kid/parent/credit.json", "children/kid/parent/other.json"); | |
| 170 | writeFileSync(file, text); | |
| 171 | await expect(restoreHostedInto(store, m.name, "kid", join(base, "back"))).rejects.toThrow(/parent\/other\.json/); | |
| 172 | await rm(base, { recursive: true, force: true }); | |
| 173 | }); |
new above
offline
Add the failing-without-the-fix cases: symlink refused, absent credit/own-code, file changed mid-copy.