Public: anyone can read this; the crew can change it.
The snapshot stays inside a hosted forge's directory #14009
Check · Review · Land · Live done — Landed and live in production.
trevato/snapshot-stays-inside-a-forge → main 2 commits 2 threads · 2 unresolved
7 files
changed +932
−55
docs/features/hosted-backup.md +8
| @@ -13,6 +13,14 @@ Each forge this one hosts goes into the nightly snapshot (#13819), under `childr | ||
| 13 | 13 | |
| 14 | 14 | Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends nothing. A forge that cannot be read is named in the manifest's `hosted` list and the log; the snapshot goes on without it. |
| 15 | 15 | |
| 16 | ## Read from inside the forge's directory only | |
| 17 | ||
| 18 | A hosted forge's directory is the forge's own to write, and its host reads it for the snapshot from outside the forge's cell (`copyHosted`, `src/snapshot/hosted.ts`). So the copy stays inside it: | |
| 19 | ||
| 20 | - No link is followed. A file is kept only if the file that was opened is inside the forge's own directory; anything else at the name is passed over. | |
| 21 | - The database is copied in a process of its own (`src/snapshot/hosted_copy.ts`), which proves the same of the files SQLite opened and refuses the copy otherwise: the forge is then left out, by name, with "its database is not a file of its own". That process, not the serving thread, reads what the copy lists. | |
| 22 | - Git runs on a repository only after it was read for what git would run on the host or follow out of the forge: a hook or configuration key that names a command, a link, objects borrowed from outside the forge. One that is refused is left out, and the forge's entry in `hosted` names it with why (`absent`). The rest of the forge is kept. | |
| 23 | ||
| 16 | 24 | ## Where it shows |
| 17 | 25 | |
| 18 | 26 | - Backups page: how many forges the newest backup kept, and how many were left out. |
src/snapshot/hosted.ts +389 −3
| @@ -6,11 +6,27 @@ | ||
| 6 | 6 | * the parent's own record of it. The manifest's `hosted` lists each forge with when it was kept. |
| 7 | 7 | * Chunks are shared with the rest of the snapshot, so a forge unchanged since last night sends |
| 8 | 8 | * nothing. |
| 9 | * | |
| 10 | * The directory is the hosted forge's to write, and its host reads it here, outside the forge's | |
| 11 | * cell. So the copy (`copyHosted`) stays inside it. Nothing follows a link: a file is opened | |
| 12 | * without following one, and is kept only if the file that was opened is, by the path the system | |
| 13 | * gives the open descriptor, inside the forge's own directory. Its database is copied in a | |
| 14 | * process of its own (`hosted_copy.ts`) that proves the same of what SQLite opened, and that | |
| 15 | * process, not the serving thread, reads what the copy lists. Git runs on one of its repositories | |
| 16 | * only after the repository was read for what git would run or follow out of the forge (a hook or | |
| 17 | * configuration key, `spawn/move_scan.ts`; a link; objects borrowed from outside), and then with | |
| 18 | * no transport and no fetch; one that is refused is left out by name, with why, in the manifest. | |
| 9 | 19 | */ |
| 10 | 20 | |
| 11 | import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 12 | import { join } from "node:path"; | |
| 21 | import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, readSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; | |
| 22 | import { lstat, readdir } from "node:fs/promises"; | |
| 23 | import { join, resolve, sep } from "node:path"; | |
| 24 | import { runGit } from "../git/env.ts"; | |
| 25 | import { findingWords, scanRepos } from "../spawn/move_scan.ts"; | |
| 13 | 26 | import { CHILD_NAME } from "../spawn/placement.ts"; |
| 27 | import { bundleRepo } from "./bundle.ts"; | |
| 28 | import type { CopyJob, CopyReply } from "./hosted_copy.ts"; | |
| 29 | import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts"; | |
| 14 | 30 | |
| 15 | 31 | /** One hosted forge as a snapshot kept it. */ |
| 16 | 32 | export interface HostedEntry { |
| @@ -23,6 +39,15 @@ export interface HostedEntry { | ||
| 23 | 39 | readonly key: "wrapped" | "none"; |
| 24 | 40 | /** Why it is not in this snapshot, when it is not. */ |
| 25 | 41 | readonly why?: string; |
| 42 | /** Repositories of it this snapshot holds no bundle of, each with why; absent when none. */ | |
| 43 | readonly absent?: readonly HostedAbsent[]; | |
| 44 | } | |
| 45 | ||
| 46 | /** A repository a hosted forge's copy holds no bundle of, and why. `empty`: it has no refs. */ | |
| 47 | export interface HostedAbsent { | |
| 48 | readonly repo: string; | |
| 49 | readonly why: string; | |
| 50 | readonly empty?: true; | |
| 26 | 51 | } |
| 27 | 52 | |
| 28 | 53 | export interface HostedSource { |
| @@ -63,7 +88,9 @@ export function hostedSources(dataDir: string): HostedSource[] { | ||
| 63 | 88 | const bodyId = bodyIdOf(text); |
| 64 | 89 | if (v.name !== name || v.done !== true || bodyId === undefined) continue; |
| 65 | 90 | const dir = join(dataDir, "children", bodyId); |
| 66 | if (existsSync(join(dir, "forge.db"))) out.push({ name, dir, ledger: text }); | |
| 91 | // Whatever is at the name, unfollowed: one that is not the forge's own file is refused by | |
| 92 | // the copy, by name, rather than passed over in silence. | |
| 93 | if (lstatSync(join(dir, "forge.db"), { throwIfNoEntry: false }) !== undefined) out.push({ name, dir, ledger: text }); | |
| 67 | 94 | } catch { |
| 68 | 95 | // A ledger that does not read is not a forge to keep. |
| 69 | 96 | } |
| @@ -127,3 +154,362 @@ export function readHosted(dataDir: string): HostedEntry[] | undefined { | ||
| 127 | 154 | return undefined; |
| 128 | 155 | } |
| 129 | 156 | } |
| 157 | ||
| 158 | // ---- the copy of one hosted forge, from outside its cell ---- | |
| 159 | ||
| 160 | /** A forge left out, with a reason a person may read: the host's words, never the forge's. */ | |
| 161 | export class HostedRefusal extends Error {} | |
| 162 | ||
| 163 | /** What `copyHosted` returns: the part of a `HostedEntry` only the copy knows. */ | |
| 164 | export type HostedCopy = Pick<HostedEntry, "repos" | "key" | "absent">; | |
| 165 | ||
| 166 | type Cut = "content" | "fixed" | "bundle"; | |
| 167 | ||
| 168 | /** The snapshot's own hands, lent for one forge: every path is under that forge's prefix. */ | |
| 169 | export interface HostedTools { | |
| 170 | /** The snapshot's scratch directory. */ | |
| 171 | readonly scratch: string; | |
| 172 | readonly passphrase?: string | undefined; | |
| 173 | /** | |
| 174 | * A file as chunks; with `own`, opened without following a link and only if it is inside `own`. | |
| 175 | */ | |
| 176 | readonly putFile: (path: string, file: string, cut?: Cut, own?: string) => Promise<void>; | |
| 177 | readonly putBytes: (path: string, body: Uint8Array, cut?: Cut) => Promise<void>; | |
| 178 | readonly bundleDeadlineMs?: ((bytes: number) => number) | undefined; | |
| 179 | /** Test seam: the database copy. */ | |
| 180 | readonly copy?: typeof copyInProcess | undefined; | |
| 181 | /** Test seam: the git runner its repositories are read with. */ | |
| 182 | readonly git?: typeof runGit | undefined; | |
| 183 | /** The most files read from the forge's directory; `HOSTED_FILES_MAX` unless a test lowers it. */ | |
| 184 | readonly filesMax?: number | undefined; | |
| 185 | } | |
| 186 | ||
| 187 | const SEGMENT = /^[A-Za-z0-9._-]{1,100}$/; | |
| 188 | const SQLITE = /\.(db|sqlite|sqlite3)$/; | |
| 189 | const SIDE_FILE = /\.(db|sqlite|sqlite3)-(wal|shm|journal)$/; | |
| 190 | ||
| 191 | const within = (path: string, root: string): boolean => path === root || path.startsWith(root + sep); | |
| 192 | ||
| 193 | /** | |
| 194 | * Open `file` for reading without following a link at its name, and only if | |
| 195 | * the file opened is a regular file inside `root` (a real path) by the path | |
| 196 | * the system gives the open descriptor. Undefined otherwise; the caller closes | |
| 197 | * what it gets. | |
| 198 | */ | |
| 199 | export function openInside(file: string, root: string): number | undefined { | |
| 200 | let fd: number; | |
| 201 | try { | |
| 202 | // Never waits: a pipe someone left at the name opens at once and is refused below. | |
| 203 | fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); | |
| 204 | } catch { | |
| 205 | return undefined; | |
| 206 | } | |
| 207 | try { | |
| 208 | if (!fstatSync(fd).isFile()) throw new Error("not a file"); | |
| 209 | let real: string; | |
| 210 | try { | |
| 211 | real = readlinkSync(`/proc/self/fd/${fd}`); | |
| 212 | } catch { | |
| 213 | // No `/proc` here: the name's real path is all there is to go by. | |
| 214 | real = realpathSync(file); | |
| 215 | } | |
| 216 | if (!within(real, root)) throw new Error("outside"); | |
| 217 | return fd; | |
| 218 | } catch { | |
| 219 | closeSync(fd); | |
| 220 | return undefined; | |
| 221 | } | |
| 222 | } | |
| 223 | ||
| 224 | /** | |
| 225 | * A small file inside `root`, whole; undefined when it is not there, not the forge's own, or over | |
| 226 | * `max`. | |
| 227 | */ | |
| 228 | export function readInside(file: string, root: string, max = 4 * 1_048_576): Uint8Array | undefined { | |
| 229 | const fd = openInside(file, root); | |
| 230 | if (fd === undefined) return undefined; | |
| 231 | try { | |
| 232 | const size = fstatSync(fd).size; | |
| 233 | if (size > max) return undefined; | |
| 234 | const out = new Uint8Array(size); | |
| 235 | let at = 0; | |
| 236 | while (at < size) { | |
| 237 | const n = readSync(fd, out, at, size - at, at); | |
| 238 | if (n <= 0) break; | |
| 239 | at += n; | |
| 240 | } | |
| 241 | return at === size ? out : out.subarray(0, at); | |
| 242 | } finally { | |
| 243 | closeSync(fd); | |
| 244 | } | |
| 245 | } | |
| 246 | ||
| 247 | /** | |
| 248 | * Files read from one hosted forge's directory at most; past it the forge is left out, and says so. | |
| 249 | */ | |
| 250 | export const HOSTED_FILES_MAX = 200_000; | |
| 251 | ||
| 252 | /** | |
| 253 | * Every regular file under `dir`, relative to it, with its size: links are not followed and not | |
| 254 | * listed. `left` is what the whole forge may still list. | |
| 255 | */ | |
| 256 | async function filesInside(dir: string, left: { n: number; readonly max: number }): Promise<{ path: string; size: number }[]> { | |
| 257 | const out: { path: string; size: number }[] = []; | |
| 258 | // A directory that is there itself: a link at its name lists somebody else's files. | |
| 259 | if ((await lstat(dir).catch(() => undefined))?.isDirectory() !== true) return out; | |
| 260 | const walk = async (at: string, rel: string): Promise<void> => { | |
| 261 | let entries: import("node:fs").Dirent[]; | |
| 262 | try { | |
| 263 | entries = await readdir(at, { withFileTypes: true }); | |
| 264 | } catch { | |
| 265 | return; | |
| 266 | } | |
| 267 | for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) { | |
| 268 | const path = join(at, entry.name); | |
| 269 | const name = rel === "" ? entry.name : `${rel}/${entry.name}`; | |
| 270 | if (entry.isDirectory()) await walk(path, name); | |
| 271 | else if (entry.isFile()) { | |
| 272 | if (--left.n < 0) throw new HostedRefusal(`it holds more than ${left.max} files to copy`); | |
| 273 | const s = await lstat(path).catch(() => undefined); | |
| 274 | if (s?.isFile() === true) out.push({ path: name, size: s.size }); | |
| 275 | } | |
| 276 | } | |
| 277 | }; | |
| 278 | await walk(dir, ""); | |
| 279 | return out; | |
| 280 | } | |
| 281 | ||
| 282 | /** The directories directly under `dir` that are directories themselves, by name. */ | |
| 283 | async function dirsIn(dir: string): Promise<string[]> { | |
| 284 | if ((await lstat(dir).catch(() => undefined))?.isDirectory() !== true) return []; | |
| 285 | const entries = await readdir(dir, { withFileTypes: true }).catch(() => []); | |
| 286 | return entries.filter((e) => e.isDirectory() && SEGMENT.test(e.name)).map((e) => e.name).sort(); | |
| 287 | } | |
| 288 | ||
| 289 | /** Whether anything under `dir` is a link. */ | |
| 290 | async function holdsLink(dir: string): Promise<boolean> { | |
| 291 | const entries = await readdir(dir, { withFileTypes: true }).catch(() => []); | |
| 292 | for (const entry of entries) { | |
| 293 | if (entry.isSymbolicLink()) return true; | |
| 294 | if (entry.isDirectory() && (await holdsLink(join(dir, entry.name)))) return true; | |
| 295 | } | |
| 296 | return false; | |
| 297 | } | |
| 298 | ||
| 299 | /** | |
| 300 | * Whether a repository borrows objects from outside its forge's repositories, itself or through a | |
| 301 | * repository it borrows from: git follows `objects/info/alternates` from one to the next. A list | |
| 302 | * this cannot read as a small file of the forge's own, or a line that names no directory it can | |
| 303 | * resolve (git also reads quoted paths), counts as outside. | |
| 304 | */ | |
| 305 | function borrowsOutside(repo: string, reposRoot: string): boolean { | |
| 306 | return alternatesLeave(join(repo, "objects"), reposRoot, 0); | |
| 307 | } | |
| 308 | ||
| 309 | function alternatesLeave(objects: string, reposRoot: string, depth: number): boolean { | |
| 310 | const file = join(objects, "info", "alternates"); | |
| 311 | if (lstatSync(file, { throwIfNoEntry: false }) === undefined) return false; | |
| 312 | const listed = readInside(file, reposRoot, 65_536); | |
| 313 | if (listed === undefined || depth > 5) return true; | |
| 314 | for (const line of new TextDecoder().decode(listed).split("\n")) { | |
| 315 | const path = line.trim(); | |
| 316 | if (path === "" || path.startsWith("#")) continue; | |
| 317 | let real: string; | |
| 318 | try { | |
| 319 | real = realpathSync(resolve(objects, path)); | |
| 320 | } catch { | |
| 321 | return true; | |
| 322 | } | |
| 323 | if (!within(real, reposRoot) || alternatesLeave(real, reposRoot, depth + 1)) return true; | |
| 324 | } | |
| 325 | return false; | |
| 326 | } | |
| 327 | ||
| 328 | /** | |
| 329 | * Git as it is run on a hosted forge's repository from outside its cell. Above what the repository | |
| 330 | * was read for before, whatever its configuration says: no transport is allowed and no missing | |
| 331 | * object is fetched (a remote's address can be a command), and nothing watches or hooks. | |
| 332 | */ | |
| 333 | const gitOutside = | |
| 334 | (git: typeof runGit): typeof runGit => | |
| 335 | (args, options = {}) => | |
| 336 | git(["-c", "protocol.allow=never", "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", ...args], { | |
| 337 | ...options, | |
| 338 | env: { ...options.env, GIT_NO_LAZY_FETCH: "1" }, | |
| 339 | }); | |
| 340 | ||
| 341 | /** | |
| 342 | * Every copy has at least this long, and this much more per MB of database; none longer than | |
| 343 | * `COPY_MAX_MS`. | |
| 344 | */ | |
| 345 | export const COPY_FLOOR_MS = 60_000; | |
| 346 | export const COPY_MS_PER_MB = 1_000; | |
| 347 | export const COPY_MAX_MS = 30 * 60_000; | |
| 348 | ||
| 349 | /** How long the copy of a database of `bytes` may take. */ | |
| 350 | export const copyDeadlineMs = (bytes: number): number => | |
| 351 | Math.min(COPY_MAX_MS, COPY_FLOOR_MS + Math.ceil(Math.max(0, bytes) / 1_000_000) * COPY_MS_PER_MB); | |
| 352 | ||
| 353 | /** | |
| 354 | * Copy a hosted forge's database in a process of its own (`hosted_copy.ts`), ended at `ms`. The | |
| 355 | * process is given nothing of this forge's: no environment, and the job (three paths) as its | |
| 356 | * argument, so it starts on its own whatever this thread is doing. Its answer is a file beside | |
| 357 | * the copy, in the snapshot's own scratch: there is no pipe to read. | |
| 358 | */ | |
| 359 | export async function copyInProcess(job: CopyJob, ms: number): Promise<CopyReply> { | |
| 360 | const answer = `${job.to}.answer`; | |
| 361 | const began = performance.now(); | |
| 362 | try { | |
| 363 | const proc = Bun.spawn([process.execPath, new URL("./hosted_copy.ts", import.meta.url).pathname, JSON.stringify(job), answer], { | |
| 364 | stdin: "ignore", | |
| 365 | stdout: "ignore", | |
| 366 | stderr: "ignore", | |
| 367 | env: {}, | |
| 368 | // The deadline is the runtime's to keep: the whole process, whatever it is in the middle of. | |
| 369 | timeout: ms, | |
| 370 | killSignal: "SIGKILL", | |
| 371 | }); | |
| 372 | await proc.exited; | |
| 373 | if (proc.signalCode !== null && performance.now() - began >= ms) | |
| 374 | return { ok: false, why: `its database was not copied within ${Math.max(1, Math.round(ms / 1000))} s` }; | |
| 375 | try { | |
| 376 | const reply = JSON.parse(readFileSync(answer, "utf8")) as CopyReply; | |
| 377 | if (typeof reply === "object" && reply !== null && typeof reply.ok === "boolean") return reply; | |
| 378 | } catch { | |
| 379 | // No answer, or not one. | |
| 380 | } | |
| 381 | return { ok: false, why: "the process copying its database ended without an answer" }; | |
| 382 | } catch (error) { | |
| 383 | return { ok: false, why: `the process copying its database could not be run: ${String(error).slice(0, 120)}` }; | |
| 384 | } finally { | |
| 385 | rmSync(answer, { force: true }); | |
| 386 | } | |
| 387 | } | |
| 388 | ||
| 389 | /** | |
| 390 | * Copy one hosted forge into the snapshot through `tools`, staying inside its directory. Throws a | |
| 391 | * `HostedRefusal` when the forge cannot be copied (no directory, a database that is not its own | |
| 392 | * file or does not copy): then nothing of it is kept. A repository that cannot be bundled is left | |
| 393 | * out by name and the rest goes in. | |
| 394 | */ | |
| 395 | export async function copyHosted(child: HostedSource, tools: HostedTools): Promise<HostedCopy> { | |
| 396 | let root: string; | |
| 397 | try { | |
| 398 | root = realpathSync(child.dir); | |
| 399 | } catch { | |
| 400 | throw new HostedRefusal("its data directory is not there"); | |
| 401 | } | |
| 402 | const copy = tools.copy ?? copyInProcess; | |
| 403 | const max = tools.filesMax ?? HOSTED_FILES_MAX; | |
| 404 | const left = { n: max, max }; | |
outdated
The link, missing-directory and bad-name branches here, and the post-bundle recheck below, are untested; each needs a case in the hosted-inside test. |
||
| 405 | const git = gitOutside(tools.git ?? runGit); | |
| 406 | const sized = async (file: string): Promise<number> => (await lstat(file).catch(() => undefined))?.size ?? 0; | |
| 407 | ||
| 408 | // The database, consistently, and the repositories it lists: read in the copy's process. | |
| 409 | const dbFile = join(root, "forge.db"); | |
| 410 | const dbCopy = join(tools.scratch, `hosted-${child.name}.db`); | |
| 411 | let listed: readonly string[] = []; | |
| 412 | try { | |
| 413 | const made = await copy({ from: dbFile, to: dbCopy, root, facts: true }, copyDeadlineMs(await sized(dbFile))); | |
| 414 | if (!made.ok) throw new HostedRefusal(made.why); | |
| 415 | listed = made.repositories ?? []; | |
| 416 | await tools.putFile("forge.db", dbCopy, "fixed"); | |
| 417 | } finally { | |
| 418 | rmSync(dbCopy, { force: true }); | |
| 419 | } | |
| 420 | await tools.putBytes("ledger.json", new TextEncoder().encode(child.ledger)); | |
| 421 | ||
| 422 | // Its key travels only wrapped, like its host's; without a passphrase it stays home. | |
| 423 | let key: HostedCopy["key"] = "none"; | |
| 424 | if (tools.passphrase) { | |
| 425 | const raw = readInside(join(root, "secret.key"), root, 4096); | |
| 426 | if (raw !== undefined && raw.byteLength > 0) { | |
| 427 | const wrapped = wrapKey(raw, tools.passphrase); | |
| 428 | if (unwrapKey(wrapped, tools.passphrase) === undefined) throw new HostedRefusal("its wrapped key does not open"); | |
| 429 | await tools.putBytes(WRAPPED_KEY, new TextEncoder().encode(wrapped)); | |
| 430 | key = "wrapped"; | |
| 431 | } | |
| 432 | } | |
| 433 | for (const f of await filesInside(join(root, "ssh"), left)) await tools.putFile(`ssh/${f.path}`, join(root, "ssh", f.path), "content", root); | |
| 434 | ||
| 435 | // Every repository on its disk, and every one its database lists: a bundle, or a reason. | |
| 436 | const reposRoot = join(root, "repos"); | |
| 437 | const onDisk = new Set<string>(); | |
| 438 | for (const owner of await dirsIn(reposRoot)) | |
| 439 | for (const entry of await dirsIn(join(reposRoot, owner))) if (entry.endsWith(".git")) onDisk.add(`${owner}/${entry.slice(0, -".git".length)}`); | |
| 440 | const findings = new Map((await scanRepos(root)).findings.map((f) => [f.repo, f])); | |
| 441 | const absent: HostedAbsent[] = []; | |
| 442 | const sizes = new Map<string, number>(); | |
| 443 | let bundled = 0; | |
| 444 | for (const repo of [...new Set([...onDisk, ...listed])].sort()) { | |
| 445 | const [owner, name] = repo.split("/") as [string, string]; | |
| 446 | if (repo.split("/").length !== 2 || !SEGMENT.test(owner) || !SEGMENT.test(name)) { | |
| 447 | absent.push({ repo: repo.replace(/[^A-Za-z0-9._\/-]/g, "?").slice(0, 200), why: "its name cannot stand in a snapshot" }); | |
| 448 | continue; | |
| 449 | } | |
| 450 | const dir = join(reposRoot, owner, `${name}.git`); | |
| 451 | if (!onDisk.has(repo)) { | |
| 452 | absent.push({ repo, why: "its database lists it and its directory is not there" }); | |
| 453 | continue; | |
| 454 | } | |
| 455 | // Git runs here, outside the forge's cell, on files the forge wrote: only on a repository that | |
| 456 | // holds nothing git would run or follow out of the forge. | |
| 457 | const found = findings.get(repo); | |
| 458 | const why = | |
| 459 | found !== undefined | |
| 460 | ? `it holds what git would run on this host (${findingWords(found).slice(repo.length + 2)})` | |
| 461 | : (await holdsLink(dir)) | |
| 462 | ? "it holds a link" | |
| 463 | : borrowsOutside(dir, reposRoot) | |
| 464 | ? "it borrows objects from outside its forge" | |
| 465 | : undefined; | |
| 466 | if (why !== undefined) { | |
| 467 | absent.push({ repo, why }); | |
| 468 | continue; | |
| 469 | } | |
| 470 | const bundle = join(tools.scratch, `hosted-${child.name}-${owner}--${name}.bundle`); | |
| 471 | try { | |
| 472 | const made = await bundleRepo(dir, bundle, { deadlineMs: tools.bundleDeadlineMs, sizes, git }); | |
| 473 | if (!made.ok) { | |
| 474 | absent.push({ repo, why: made.why.slice(0, 200), ...(made.empty ? { empty: true as const } : {}) }); | |
| 475 | continue; | |
| 476 | } | |
| 477 | // Still the directory that was looked at: one changed for a link while git read it is not | |
| 478 | // kept. | |
| 479 | if ((await lstat(dir).catch(() => undefined))?.isDirectory() !== true || (await holdsLink(dir)) || borrowsOutside(dir, reposRoot)) { | |
| 480 | absent.push({ repo, why: "it changed into something else while it was read" }); | |
| 481 | continue; | |
| 482 | } | |
| 483 | await tools.putFile(`repos/${owner}/${name}.bundle`, bundle); | |
| 484 | bundled++; | |
| 485 | } finally { | |
| 486 | rmSync(bundle, { force: true }); | |
| 487 | } | |
| 488 | } | |
| 489 | ||
| 490 | // Each app's production data; SQLite inside is copied the consistent way, its side-files left. | |
| 491 | for (const app of await dirsIn(join(root, "apps"))) { | |
| 492 | const data = join(root, "apps", app, "production", "data"); | |
| 493 | for (const f of await filesInside(data, left)) { | |
| 494 | if (SIDE_FILE.test(f.path)) continue; | |
| 495 | const at = `apps/${app}/production/data/${f.path}`; | |
| 496 | const file = join(data, f.path); | |
| 497 | if (!SQLITE.test(f.path)) { | |
| 498 | await tools.putFile(at, file, "content", root); | |
| 499 | continue; | |
| 500 | } | |
| 501 | const to = join(tools.scratch, `hosted-${child.name}-app.db`); | |
| 502 | try { | |
| 503 | const made = await copy({ from: file, to, root }, copyDeadlineMs(f.size)); | |
| 504 | if (made.ok) await tools.putFile(at, to, "fixed"); | |
| 505 | // Not a database SQLite reads, whatever its name: its bytes as they are, if they are its | |
| 506 | // own. | |
| 507 | else await tools.putFile(at, file, "fixed", root); | |
| 508 | } finally { | |
| 509 | rmSync(to, { force: true }); | |
| 510 | } | |
| 511 | } | |
| 512 | } | |
| 513 | ||
| 514 | return { repos: bundled, key, ...(absent.length === 0 ? {} : { absent }) }; | |
| 515 | } |
src/snapshot/hosted_copy.ts added +129
| @@ -0,0 +1,129 @@ | ||
| 1 | /** | |
| 2 | * The consistent copy of a hosted forge's database, in a process of its own | |
| 3 | * (`hosted.ts` starts it; #13819). | |
| 4 | * | |
| 5 | * The file is the hosted forge's to write, so the forge that hosts it does not | |
| 6 | * open it on a thread of its own process (`spawn/child_db_reader.ts` says what | |
| 7 | * that cost), and trusts neither its size nor what its name stands for. This | |
| 8 | * process opens it read-only, proves the files it opened are inside the hosted | |
| 9 | * forge's directory, and copies it with `VACUUM INTO`: one read transaction, | |
| 10 | * so a forge that is awake goes on writing while its copy is made, and what a | |
| 11 | * sleeping forge left in its WAL is in the copy (a raw copy of `forge.db` | |
| 12 | * would leave it out). | |
| 13 | * | |
| 14 | * The proof is by the open files, not by the name: a link at the name could | |
| 15 | * stand for a file outside the forge's directory, and a name checked before | |
| 16 | * the open can be swapped after the check. Once the database is open, what it | |
| 17 | * opened is what `/proc/self/fd` says, and anything outside the directory | |
| 18 | * refuses the copy. | |
| 19 | * | |
| 20 | * What the copy lists is read here too, never on the serving thread: the | |
| 21 | * schema is the hosted forge's, and a view under a table's name runs as long | |
| 22 | * as its author chose. | |
| 23 | * | |
| 24 | * One JSON job as its first argument (paths, nothing else); its answer is one | |
| 25 | * JSON file at the path in its second. The forge that waits holds the deadline | |
| 26 | * and ends this process when it passes. | |
| 27 | */ | |
| 28 | ||
| 29 | import { Database } from "bun:sqlite"; | |
| 30 | import { lstatSync, readdirSync, readlinkSync, realpathSync, writeFileSync } from "node:fs"; | |
| 31 | import { basename, sep } from "node:path"; | |
| 32 | import { notPlainTables } from "../spawn/child_db_reader.ts"; | |
| 33 | ||
| 34 | export interface CopyJob { | |
| 35 | /** The database to copy. */ | |
| 36 | readonly from: string; | |
| 37 | /** The copy to make; it must not exist. */ | |
| 38 | readonly to: string; | |
| 39 | /** The hosted forge's directory: everything opened must be inside it. */ | |
| 40 | readonly root: string; | |
| 41 | /** Also list the copy's repositories (a forge's own database). */ | |
| 42 | readonly facts?: boolean; | |
| 43 | } | |
| 44 | ||
| 45 | export type CopyReply = | |
| 46 | | { readonly ok: true; readonly repositories?: readonly string[] } | |
| 47 | | { readonly ok: false; readonly why: string }; | |
| 48 | ||
| 49 | const inside = (path: string, root: string): boolean => path === root || path.startsWith(root + sep); | |
| 50 | ||
| 51 | /** | |
| 52 | * The files this process has open that are the database `from` or its side-files, by their real | |
| 53 | * paths; undefined where the system does not say (no `/proc`). | |
| 54 | */ | |
| 55 | function openedAs(from: string): string[] | undefined { | |
| 56 | let fds: string[]; | |
| 57 | try { | |
| 58 | fds = readdirSync("/proc/self/fd"); | |
| 59 | } catch { | |
| 60 | return undefined; | |
| 61 | } | |
| 62 | const name = basename(from); | |
| 63 | const out: string[] = []; | |
| 64 | for (const fd of fds) { | |
| 65 | try { | |
| 66 | const target = readlinkSync(`/proc/self/fd/${fd}`); | |
| 67 | const b = basename(target); | |
| 68 | if (b === name || b === `${name}-wal` || b === `${name}-shm`) out.push(target); | |
| 69 | } catch { | |
| 70 | // Closed since the listing. | |
| 71 | } | |
| 72 | } | |
| 73 | return out; | |
| 74 | } | |
| 75 | ||
| 76 | /** Copy `job.from` into `job.to`; a refusal says why in the host's own words, never the file's. */ | |
| 77 | export function copyDatabase(job: CopyJob): CopyReply { | |
| 78 | let root: string; | |
| 79 | try { | |
| 80 | root = realpathSync(job.root); | |
| 81 | if (!lstatSync(job.from).isFile()) return { ok: false, why: "its database is not a file of its own" }; | |
| 82 | } catch { | |
| 83 | return { ok: false, why: "its database is not there" }; | |
| 84 | } | |
| 85 | let db: Database | undefined; | |
| 86 | try { | |
| 87 | db = new Database(job.from, { readonly: true }); | |
| 88 | db.run("PRAGMA busy_timeout = 30000"); | |
| 89 | // The constructor is lazy: reading the header opens the file, and the WAL beside it. | |
| 90 | db.query("PRAGMA schema_version").get(); | |
| 91 | const opened = openedAs(job.from); | |
| 92 | if (opened === undefined) { | |
| 93 | // No `/proc`: the name's real path is all there is to go by. | |
| 94 | if (!inside(realpathSync(job.from), root)) return { ok: false, why: "its database is not a file of its own" }; | |
| 95 | } else if (opened.length === 0 || !opened.every((path) => inside(path, root))) | |
| 96 | return { ok: false, why: "its database is not a file of its own" }; | |
| 97 | db.run("VACUUM INTO ?1", [job.to]); | |
| 98 | } catch (error) { | |
| 99 | return { ok: false, why: `its database could not be copied: ${String(error instanceof Error ? error.message : error).slice(0, 160)}` }; | |
| 100 | } finally { | |
| 101 | db?.close(); | |
| 102 | } | |
| 103 | if (job.facts !== true) return { ok: true }; | |
| 104 | // The copy is a file this process made; what it holds is still the hosted forge's. | |
| 105 | let copy: Database | undefined; | |
| 106 | try { | |
| 107 | copy = new Database(job.to, { readonly: true }); | |
| 108 | if (notPlainTables(copy, ["user", "repo"]).length > 0) return { ok: true }; | |
| 109 | const repositories = copy | |
| 110 | .query<{ owner: unknown; name: unknown }, []>("SELECT user.username AS owner, repo.name AS name FROM repo JOIN user ON user.id = repo.owner_id ORDER BY 1, 2 LIMIT 100000") | |
| 111 | .all() | |
| 112 | .flatMap((r) => (typeof r.owner === "string" && typeof r.name === "string" ? [`${r.owner}/${r.name}`] : [])); | |
| 113 | return { ok: true, repositories }; | |
| 114 | } catch { | |
| 115 | // Copied, not listed: its repositories are then the ones on its disk. | |
| 116 | return { ok: true }; | |
| 117 | } finally { | |
| 118 | copy?.close(); | |
| 119 | } | |
| 120 | } | |
| 121 | ||
| 122 | // Only as the copying process: a test imports this file for `copyDatabase`. | |
| 123 | if (import.meta.main) { | |
| 124 | const job = JSON.parse(process.argv[2] ?? "null") as CopyJob; | |
| 125 | const answer = process.argv[3]; | |
| 126 | const reply = copyDatabase(job); | |
| 127 | if (answer) writeFileSync(answer, JSON.stringify(reply), { mode: 0o600 }); | |
| 128 | process.exit(0); | |
| 129 | } |
src/snapshot/snapshot.ts +36 −51
| @@ -46,7 +46,7 @@ import { listEveryRepo } from "../core/repos.ts"; | ||
| 46 | 46 | import { keepLentObjects, repoDir } from "../git/repos.ts"; |
| 47 | 47 | import { createLog } from "../kernel/log.ts"; |
| 48 | 48 | import { bundleRepo } from "./bundle.ts"; |
| 49 | import { hostedSources, HOSTED_PREFIX, type HostedEntry } from "./hosted.ts"; | |
| 49 | import { copyHosted, HostedRefusal, hostedSources, HOSTED_PREFIX, openInside, type HostedEntry, type HostedTools } from "./hosted.ts"; | |
| 50 | 50 | import { countRows, type DrillCounts } from "./counts.ts"; |
| 51 | 51 | import { CHUNK_PREFIX, chunkId, chunkKey, chunkSeed, contentChunker, fixedChunker, isChunkId, packChunk, type Chunker } from "./chunks.ts"; |
| 52 | 52 | import type { PackedFile } from "./parts.ts"; |
| @@ -325,6 +325,11 @@ export interface Snapshotter { | ||
| 325 | 325 | readonly bundleDeadlineMs?: ((bytes: number) => number) | undefined; |
| 326 | 326 | /** Plain bytes of the last snapshot, any lane: what "to go" is judged against. */ |
| 327 | 327 | readonly expectedBytes?: number | undefined; |
| 328 | /** Test seam: how a hosted forge's database is copied. */ | |
| 329 | readonly hostedCopy?: HostedTools["copy"]; | |
| 330 | /** Test seams: the git runner a hosted forge's repositories are read with, and its file limit. */ | |
| 331 | readonly hostedGit?: HostedTools["git"]; | |
| 332 | readonly hostedFilesMax?: HostedTools["filesMax"]; | |
| 328 | 333 | } |
| 329 | 334 | |
| 330 | 335 | /** |
| @@ -498,13 +503,16 @@ export async function takeSnapshot( | ||
| 498 | 503 | }; |
| 499 | 504 | // A file on disk, read as a stream and never held whole: the hash of what was read, and its |
| 500 | 505 | // chunks. |
| 501 | const putFile = async (path: string, file: string, cut: Cut = "content"): Promise<void> => { | |
| 502 | if (statSync(file, { throwIfNoEntry: false })?.isFile() !== true) return; | |
| 506 | // With `own`, a hosted forge's file (`hosted.ts`): opened without following a link, and only when | |
| 507 | // what was opened is inside that directory. | |
| 508 | const putFile = async (path: string, file: string, cut: Cut = "content", own?: string): Promise<void> => { | |
| 509 | const fd = own === undefined ? undefined : openInside(file, own); | |
| 510 | if (own === undefined ? statSync(file, { throwIfNoEntry: false })?.isFile() !== true : fd === undefined) return; | |
| 503 | 511 | const cutter = chunker(cut); |
| 504 | 512 | const whole = new Bun.CryptoHasher("sha256"); |
| 505 | 513 | const chunks: string[] = []; |
| 506 | 514 | let bytes = 0; |
| 507 | for await (const part of createReadStream(file, { highWaterMark: READ_BYTES }) as AsyncIterable<Uint8Array>) { | |
| 515 | for await (const part of createReadStream(file, { highWaterMark: READ_BYTES, ...(fd === undefined ? {} : { fd }) }) as AsyncIterable<Uint8Array>) { | |
| 508 | 516 | whole.update(part); |
| 509 | 517 | bytes += part.byteLength; |
| 510 | 518 | for (const c of cutter.push(part)) chunks.push(await send(c)); |
| @@ -627,59 +635,36 @@ export async function takeSnapshot( | ||
| 627 | 635 | }; |
| 628 | 636 | const kept = await putAppData(listApps(forge.db), forge.dataDir, "", ""); |
| 629 | 637 | |
| 630 | // Every hosted forge (#13819), each under its own prefix so one restores alone. A forge that | |
| 631 | // cannot be read costs that forge, never the snapshot; the manifest says which. | |
| 638 | // Every hosted forge (#13819), each under its own prefix so one restores alone, and read from | |
| 639 | // inside its own directory only (`copyHosted`). A forge that cannot be read costs that forge, | |
| 640 | // never the snapshot; the manifest says which, and names a repository it left out. | |
| 632 | 641 | const hosted: HostedEntry[] = []; |
| 633 | 642 | for (const child of hostedSources(forge.dataDir)) { |
| 634 | 643 | const into = `${HOSTED_PREFIX}${child.name}/`; |
| 635 | 644 | const from = files.length; |
| 636 | const copy = join(scratch, `hosted-${child.name}.db`); | |
| 637 | let repoCount = 0; | |
| 638 | let wrapped: HostedEntry["key"] = "none"; | |
| 645 | const at = new Date().toISOString(); | |
| 639 | 646 | try { |
| 640 | await vacuumFile(join(child.dir, "forge.db"), copy); | |
| 641 | const childDb = new Database(copy, { readonly: true }); | |
| 642 | let childRepos: ReturnType<typeof listEveryRepo>; | |
| 643 | let childApps: ReturnType<typeof listApps>; | |
| 644 | try { | |
| 645 | childRepos = listEveryRepo(childDb); | |
| 646 | childApps = listApps(childDb); | |
| 647 | } finally { | |
| 648 | childDb.close(); | |
| 649 | } | |
| 650 | await putFile(`${into}forge.db`, copy, "fixed"); | |
| 651 | await putBytes(`${into}ledger.json`, new TextEncoder().encode(child.ledger)); | |
| 652 | const childKey = Bun.file(join(child.dir, "secret.key")); | |
| 653 | if (forge.passphrase && (await childKey.exists())) { | |
| 654 | const sealed = wrapKey(new Uint8Array(await childKey.arrayBuffer()), forge.passphrase); | |
| 655 | if (unwrapKey(sealed, forge.passphrase) === undefined) throw new Error("its wrapped key does not open"); | |
| 656 | await putBytes(`${into}${WRAPPED_KEY}`, new TextEncoder().encode(sealed)); | |
| 657 | wrapped = "wrapped"; | |
| 658 | } | |
| 659 | for (const f of filesUnder(join(child.dir, "ssh"))) await putFile(`${into}ssh/${f}`, join(child.dir, "ssh", f)); | |
| 660 | for (const repo of childRepos) { | |
| 661 | const dir = repoDir(join(child.dir, "repos"), repo.ownerName, repo.name); | |
| 662 | const bundle = join(scratch, `hosted-${child.name}-${repo.ownerName}--${repo.name}.bundle`); | |
| 663 | await keepLentObjects(dir); | |
| 664 | const made = await bundleRepo(dir, bundle, { deadlineMs: forge.bundleDeadlineMs, sizes }); | |
| 665 | if (!made.ok) { | |
| 666 | rmSync(bundle, { force: true }); | |
| 667 | if (!made.empty) log.error("hosted repository left out of the snapshot", { forge: child.name, repo: `${repo.ownerName}/${repo.name}`, why: made.why }); | |
| 668 | continue; | |
| 669 | } | |
| 670 | await putFile(`${into}repos/${repo.ownerName}/${repo.name}.bundle`, bundle); | |
| 671 | rmSync(bundle, { force: true }); | |
| 672 | repoCount++; | |
| 673 | } | |
| 674 | await putAppData(childApps, child.dir, into, `${child.name}/`); | |
| 675 | hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: files.slice(from).reduce((n, f) => n + f.bytes, 0), key: wrapped }); | |
| 647 | const took = await copyHosted(child, { | |
| 648 | scratch, | |
| 649 | passphrase: forge.passphrase, | |
| 650 | putFile: (path, file, cut, own) => putFile(into + path, file, cut, own), | |
| 651 | putBytes: (path, body, cut) => putBytes(into + path, body, cut), | |
| 652 | bundleDeadlineMs: forge.bundleDeadlineMs, | |
| 653 | copy: forge.hostedCopy, | |
| 654 | git: forge.hostedGit, | |
| 655 | filesMax: forge.hostedFilesMax, | |
| 656 | }); | |
| 657 | for (const a of took.absent ?? []) | |
| 658 | if (a.empty !== true) log.error("hosted repository left out of the snapshot", { forge: child.name, repo: a.repo, why: a.why }); | |
| 659 | hosted.push({ name: child.name, at, bytes: files.slice(from).reduce((n, f) => n + f.bytes, 0), ...took }); | |
| 676 | 660 | } catch (error) { |
| 677 | // What went up before the failure stays listed in `files` and is harmless; the entry says | |
| 678 | // why. | |
| 679 | log.error("hosted forge left out of the snapshot", { forge: child.name, error: String(error) }); | |
| 680 | hosted.push({ name: child.name, at: new Date().toISOString(), repos: repoCount, bytes: 0, key: wrapped, why: String(error).slice(0, 200) }); | |
| 681 | } finally { | |
| 682 | rmSync(copy, { force: true }); | |
| 661 | // A put that failed is the snapshot's failure, not this forge's. | |
| 662 | if (failure !== undefined) throw failure; | |
| 663 | // Nothing of a forge left out stays listed: its entry says why. | |
| 664 | files.length = from; | |
| 665 | const why = error instanceof HostedRefusal ? error.message : `it could not be read: ${String(error instanceof Error ? error.message : error).slice(0, 160)}`; | |
| 666 | log.error("hosted forge left out of the snapshot", { forge: child.name, why }); | |
| 667 | hosted.push({ name: child.name, at, repos: 0, bytes: 0, key: "none", why }); | |
| 683 | 668 | } |
| 684 | 669 | } |
| 685 | 670 |
test/snapshot-hosted-inside.test.ts added +367
| @@ -0,0 +1,367 @@ | ||
| 1 | /** | |
| 2 | * The snapshot stays inside a hosted forge's directory (#13819). | |
| 3 | * | |
| 4 | * A hosted forge's directory is the forge's own to write, and its host reads | |
| 5 | * it for the nightly snapshot from outside the forge's cell. A hosted forge's | |
| 6 | * directory could point the snapshot at a file outside it. Proved here, on a | |
| 7 | * host with hosted forges laid out as a spawn leaves them: | |
| 8 | * | |
| 9 | * - a link in a forge's directory is not followed, wherever its files are | |
| 10 | * read (its files, its key, its app data), and nothing from outside the | |
| 11 | * directory is in the snapshot's store; | |
| 12 | * - a forge whose database is not a file of its own is left out, by name, and | |
| 13 | * the snapshot still lands; | |
| 14 | * - a repository git would run something of on the host, that holds a link, | |
| 15 | * or that borrows its objects from outside its forge, is left out by name, | |
| 16 | * in the manifest; so is one its database lists and its disk lacks, one | |
| 17 | * whose name cannot stand in a snapshot, and one that changed while git | |
| 18 | * read it; | |
| 19 | * - a forge that holds more files than one copy reads is left out, by name; | |
| 20 | * - the forge's own lists are read in the copy's process: a view under a | |
| 21 | * table's name is never run by the snapshot; | |
| 22 | * - the copy is still one consistent moment while the forge writes, holds | |
| 23 | * what a sleeping forge left in its WAL, and is ended at its deadline. | |
| 24 | */ | |
| 25 | import { Database } from "bun:sqlite"; | |
| 26 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 27 | import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; | |
| 28 | import { rm } from "node:fs/promises"; | |
| 29 | import { tmpdir } from "node:os"; | |
| 30 | import { join } from "node:path"; | |
| 31 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 32 | import { createApp, type App } from "../src/app.ts"; | |
| 33 | import { getRepo } from "../src/core/repos.ts"; | |
| 34 | import { createUser, type User } from "../src/core/users.ts"; | |
| 35 | import { runGit } from "../src/git/env.ts"; | |
| 36 | import { repoDir } from "../src/git/repos.ts"; | |
| 37 | import { chunkKey, openChunk } from "../src/snapshot/chunks.ts"; | |
| 38 | import { copyInProcess, hostedSources, type HostedSource } from "../src/snapshot/hosted.ts"; | |
| 39 | import { restoreHosted } from "../src/snapshot/restore.ts"; | |
| 40 | import { takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts"; | |
| 41 | import type { Store } from "../src/snapshot/store.ts"; | |
| 42 | import { verifySnapshot } from "../src/snapshot/verify.ts"; | |
| 43 | import { quietLogs } from "./helpers/globals.ts"; | |
| 44 | ||
| 45 | quietLogs(); | |
| 46 | ||
| 47 | const PASS = "correct horse battery staple"; | |
| 48 | /** What a file outside every hosted forge's directory holds: it must be in no snapshot of one. */ | |
| 49 | const OUTSIDE = "outside-the-forge-0f3c9a71d2e84b56-not-to-be-copied"; | |
| 50 | ||
| 51 | /** A store in memory, as a bucket behaves. */ | |
| 52 | function memStore(): Store & { objects: Map<string, Uint8Array> } { | |
| 53 | const objects = new Map<string, Uint8Array>(); | |
| 54 | return { | |
| 55 | where: "memory", | |
| 56 | objects, | |
| 57 | put: async (key, body) => void objects.set(key, typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body)), | |
| 58 | get: async (key) => objects.get(key), | |
| 59 | list: async (prefix) => [...objects.keys()].filter((k) => k.startsWith(prefix)), | |
| 60 | names: async (prefix) => [ | |
| 61 | ...new Set( | |
| 62 | [...objects.keys()] | |
| 63 | .filter((k) => k.startsWith(prefix) && k.indexOf("/", prefix.length) >= 0) | |
| 64 | .map((k) => k.slice(prefix.length, k.indexOf("/", prefix.length))), | |
| 65 | ), | |
| 66 | ], | |
| 67 | delete: async (key) => void objects.delete(key), | |
| 68 | }; | |
| 69 | } | |
| 70 | ||
| 71 | /** Whether any chunk a manifest names holds `needle` in its plain bytes. */ | |
| 72 | function anyChunkHolds(store: ReturnType<typeof memStore>, m: Manifest, needle: string): boolean { | |
| 73 | const want = Buffer.from(needle); | |
| 74 | for (const f of m.files) | |
| 75 | for (const id of f.chunks ?? []) { | |
| 76 | const body = store.objects.get(chunkKey(id)); | |
| 77 | if (body !== undefined && Buffer.from(openChunk(id, body)).includes(want)) return true; | |
| 78 | } | |
| 79 | return false; | |
| 80 | } | |
| 81 | ||
| 82 | describe("a host's snapshot of the forges it hosts", () => { | |
| 83 | let root: string; | |
| 84 | let host: App; | |
| 85 | let made = 0; | |
| 86 | const fresh = (what: string): string => join(root, `${what}-${++made}`); | |
| 87 | const snap = (only: readonly string[], more: Partial<Parameters<typeof takeSnapshot>[0]> = {}, at = "2026-11-16T08:00:00Z") => { | |
| 88 | // Only the named forges are hosted for this snapshot: each test has its own. | |
| 89 | const spawn = join(host.config.dataDir, "spawn"); | |
| 90 | for (const name of ledgers.keys()) rmSync(join(spawn, `${name}.json`), { force: true }); | |
| 91 | for (const name of only) writeFileSync(join(spawn, `${name}.json`), ledgers.get(name)!); | |
| 92 | const store = memStore(); | |
| 93 | return takeSnapshot({ db: host.db, dataDir: host.config.dataDir, reposRoot: host.reposRoot, version: "t", passphrase: PASS, ...more }, store, new Date(at)).then((manifest) => ({ store, manifest })); | |
| 94 | }; | |
| 95 | const ledgers = new Map<string, string>(); | |
| 96 | const filesOf = (m: Manifest, name: string): string[] => m.files.filter((f) => f.path.startsWith(`children/${name}/`)).map((f) => f.path.slice(`children/${name}/`.length)).sort(); | |
| 97 | ||
| 98 | /** | |
| 99 | * A hosted forge as a spawn leaves one: its data directory, a key, a host key, app data, and its | |
| 100 | * host's ledger of it. | |
| 101 | */ | |
| 102 | async function hostForge(name: string, bodyId: string): Promise<{ source: HostedSource; open: App; owner: User }> { | |
| 103 | const dir = join(host.config.dataDir, "children", bodyId); | |
| 104 | const open = createApp({ dataDir: dir, port: 0 }); | |
| 105 | const user = await createUser(open.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: true }); | |
| 106 | if (!user.ok) throw new Error(user.error); | |
| 107 | writeFileSync(join(dir, "secret.key"), "k".repeat(32), { mode: 0o600 }); | |
| 108 | mkdirSync(join(dir, "ssh"), { recursive: true }); | |
| 109 | writeFileSync(join(dir, "ssh", "host_ed25519"), `the host key of ${name}\n`, { mode: 0o600 }); | |
| 110 | mkdirSync(join(dir, "apps", "a1", "production", "data"), { recursive: true }); | |
| 111 | writeFileSync(join(dir, "apps", "a1", "production", "data", "notes.txt"), `the data of ${name}'s app\n`); | |
| 112 | const ledger = JSON.stringify({ name, bodyId, done: true }); | |
| 113 | ledgers.set(name, ledger); | |
| 114 | return { source: { name, dir, ledger }, open, owner: user.value }; | |
| 115 | } | |
| 116 | const commit = async (f: { open: App; owner: User }, repo: string): Promise<void> => { | |
| 117 | const r = await createRepo(f.open, f.owner, { name: repo, visibility: "private" }); | |
| 118 | if (!r.ok) throw new Error(String(r.error)); | |
| 119 | const c = await commitToBranch(f.open, f.owner, getRepo(f.open.db, f.owner.username, repo)!, { message: "one", files: [{ path: "a.md", content: "a\n" }] }); | |
| 120 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 121 | }; | |
| 122 | ||
| 123 | beforeAll(async () => { | |
| 124 | root = mkdtempSync(join(tmpdir(), "forge-hosted-inside-")); | |
| 125 | host = createApp({ dataDir: join(root, "host"), port: 0 }); | |
| 126 | const keeper = await createUser(host.db, { origin: "seed", trusted: true, username: "keeper", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 127 | if (!keeper.ok) throw new Error(keeper.error); | |
| 128 | mkdirSync(join(host.config.dataDir, "spawn"), { recursive: true }); | |
| 129 | // A file, a directory and a database that are no hosted forge's. | |
| 130 | writeFileSync(join(root, "outside.txt"), OUTSIDE); | |
| 131 | mkdirSync(join(root, "outside-dir")); | |
| 132 | writeFileSync(join(root, "outside-dir", "inner.txt"), OUTSIDE); | |
| 133 | const other = new Database(join(root, "outside.db")); | |
| 134 | other.run("CREATE TABLE note (body TEXT)"); | |
| 135 | other.run("INSERT INTO note VALUES (?)", [OUTSIDE]); | |
| 136 | other.close(); | |
| 137 | }); | |
| 138 | afterAll(async () => { | |
| 139 | host.close(); | |
| 140 | await rm(root, { recursive: true, force: true }); | |
| 141 | }); | |
| 142 | ||
| 143 | test("a link in a forge's directory is not followed, wherever its files are read", async () => { | |
| 144 | const f = await hostForge("links", "body-links"); | |
| 145 | f.open.close(); | |
| 146 | const dir = f.source.dir; | |
| 147 | const outside = join(root, "outside.txt"); | |
| 148 | symlinkSync(outside, join(dir, "ssh", "linked")); | |
| 149 | symlinkSync(outside, join(dir, "apps", "a1", "production", "data", "linked.txt")); | |
| 150 | symlinkSync(join(root, "outside-dir"), join(dir, "apps", "a1", "production", "data", "linked-dir")); | |
| 151 | // A whole directory of its own that is a link, and a link where another of its files is read. | |
| 152 | mkdirSync(join(dir, "apps", "a2", "production"), { recursive: true }); | |
| 153 | symlinkSync(join(root, "outside-dir"), join(dir, "apps", "a2", "production", "data")); | |
| 154 | rmSync(join(dir, "secret.key")); | |
| 155 | symlinkSync(outside, join(dir, "secret.key")); | |
| 156 | const { store, manifest } = await snap(["links"]); | |
| 157 | expect(manifest.hosted).toMatchObject([{ name: "links", key: "none" }]); | |
| 158 | expect(manifest.hosted?.[0]?.why).toBeUndefined(); | |
| 159 | // Its own files are kept; nothing a link named is. | |
| 160 | expect(filesOf(manifest, "links")).toEqual(["apps/a1/production/data/notes.txt", "forge.db", "ledger.json", "ssh/host_ed25519"]); | |
| 161 | expect(anyChunkHolds(store, manifest, OUTSIDE)).toBe(false); | |
| 162 | await verifySnapshot(store, manifest.name, { deep: true }); | |
| 163 | }); | |
| 164 | ||
| 165 | test("a forge whose database is not a file of its own is left out, by name, and the snapshot still lands", async () => { | |
| 166 | const f = await hostForge("swapped", "body-swapped"); | |
| 167 | f.open.close(); | |
| 168 | for (const name of ["forge.db", "forge.db-wal", "forge.db-shm"]) rmSync(join(f.source.dir, name), { force: true }); | |
| 169 | symlinkSync(join(root, "outside.db"), join(f.source.dir, "forge.db")); | |
| 170 | // Still one of the forges this host is to keep: it is named, not passed over. | |
| 171 | writeFileSync(join(host.config.dataDir, "spawn", "swapped.json"), f.source.ledger); | |
| 172 | expect(hostedSources(host.config.dataDir).map((s) => s.name)).toContain("swapped"); | |
| 173 | const { store, manifest } = await snap(["swapped"]); | |
| 174 | expect(manifest.hosted).toEqual([{ name: "swapped", at: manifest.hosted![0]!.at, repos: 0, bytes: 0, key: "none", why: "its database is not a file of its own" }]); | |
| 175 | expect(filesOf(manifest, "swapped")).toEqual([]); | |
| 176 | expect(anyChunkHolds(store, manifest, OUTSIDE)).toBe(false); | |
| 177 | expect(manifest.files.some((x) => x.path === "forge.db")).toBe(true); | |
| 178 | await verifySnapshot(store, manifest.name); | |
| 179 | await expect(restoreHosted(store, manifest.name, "swapped", fresh("none"))).rejects.toThrow(/does not hold swapped \(its database is not a file of its own\)/); | |
| 180 | }); | |
| 181 | ||
| 182 | test("a repository git would run something of, that holds a link, or that borrows from outside its forge is left out by name in the manifest", async () => { | |
| 183 | const f = await hostForge("stacks", "body-stacks"); | |
| 184 | for (const name of ["fine", "borrow", "trap", "linked"]) await commit(f, name); | |
| 185 | f.open.close(); | |
| 186 | const repos = join(f.source.dir, "repos"); | |
| 187 | // A link inside a repository: git would follow it wherever it names. | |
| 188 | symlinkSync(join(root, "outside-dir"), join(repoDir(repos, "stacks", "linked"), "objects", "elsewhere")); | |
| 189 | // Configuration that names a command, and objects borrowed from a directory outside the forge | |
| 190 | // (in the quoted form git also reads). | |
| 191 | const trap = repoDir(repos, "stacks", "trap"); | |
| 192 | writeFileSync(join(trap, "config"), `${readFileSync(join(trap, "config"), "utf8")}\n[core]\n\tfsmonitor = /bin/true\n`); | |
| 193 | writeFileSync(join(repoDir(repos, "stacks", "borrow"), "objects", "info", "alternates"), `"${join(root, "outside-dir")}"\n`); | |
| 194 | const { store, manifest } = await snap(["stacks"]); | |
| 195 | const entry = manifest.hosted![0]!; | |
| 196 | expect(entry.why).toBeUndefined(); | |
| 197 | expect(entry.repos).toBe(1); | |
| 198 | expect(entry.absent).toEqual([ | |
| 199 | { repo: "stacks/borrow", why: "it borrows objects from outside its forge" }, | |
| 200 | { repo: "stacks/linked", why: "it holds a link" }, | |
| 201 | { repo: "stacks/trap", why: "it holds what git would run on this host (config core.fsmonitor)" }, | |
| 202 | ]); | |
| 203 | expect(filesOf(manifest, "stacks").filter((p) => p.startsWith("repos/"))).toEqual(["repos/stacks/fine.bundle"]); | |
| 204 | // The one that is in comes back whole. | |
| 205 | const back = fresh("repos-back"); | |
| 206 | const done = await restoreHosted(store, manifest.name, "stacks", back, { passphrase: PASS }); | |
| 207 | expect(done.repos).toBe(1); | |
| 208 | expect((await runGit(["fsck", "--no-dangling"], { cwd: repoDir(join(back, "repos"), "stacks", "fine") })).ok).toBe(true); | |
| 209 | }); | |
| 210 | ||
| 211 | test("a repository its database lists and its disk lacks, and one whose name cannot stand in a snapshot, are named and not passed over", async () => { | |
| 212 | const f = await hostForge("ledgers", "body-ledgers"); | |
| 213 | for (const name of ["fine", "gone", "odd"]) await commit(f, name); | |
| 214 | // Its database names one its disk no longer holds, and one by a name no path may carry. | |
| 215 | f.open.db.run("UPDATE repo SET name = 'no such/../name' WHERE name = 'odd'"); | |
| 216 | f.open.close(); | |
| 217 | const repos = join(f.source.dir, "repos"); | |
| 218 | for (const name of ["gone", "odd"]) rmSync(repoDir(repos, "ledgers", name), { recursive: true }); | |
| 219 | const { manifest } = await snap(["ledgers"]); | |
| 220 | const entry = manifest.hosted![0]!; | |
| 221 | expect([entry.why, entry.repos]).toEqual([undefined, 1]); | |
| 222 | expect(entry.absent).toEqual([ | |
| 223 | { repo: "ledgers/gone", why: "its database lists it and its directory is not there" }, | |
| 224 | { repo: "ledgers/no?such/../name", why: "its name cannot stand in a snapshot" }, | |
| 225 | ]); | |
| 226 | expect(filesOf(manifest, "ledgers").filter((p) => p.startsWith("repos/"))).toEqual(["repos/ledgers/fine.bundle"]); | |
| 227 | // Nothing in the manifest carries the name as a path. | |
| 228 | expect(manifest.files.some((x) => x.path.includes(".."))).toBe(false); | |
| 229 | }); | |
| 230 | ||
| 231 | test("a repository that changed into something else while git read it is not kept", async () => { | |
| 232 | const f = await hostForge("shifty", "body-shifty"); | |
| 233 | for (const name of ["fine", "swap"]) await commit(f, name); | |
| 234 | f.open.close(); | |
| 235 | const swap = repoDir(join(f.source.dir, "repos"), "shifty", "swap"); | |
| 236 | const { manifest } = await snap(["shifty"], { | |
| 237 | // The forge is awake: the moment after git has made its bundle, the repository holds a link. | |
| 238 | hostedGit: async (args, options) => { | |
| 239 | const out = await runGit(args, options); | |
| 240 | if (options?.cwd === swap && args.includes("bundle")) symlinkSync(join(root, "outside-dir"), join(swap, "objects", "elsewhere")); | |
| 241 | return out; | |
| 242 | }, | |
| 243 | }); | |
| 244 | const entry = manifest.hosted![0]!; | |
| 245 | expect([entry.why, entry.repos]).toEqual([undefined, 1]); | |
| 246 | expect(entry.absent).toEqual([{ repo: "shifty/swap", why: "it changed into something else while it was read" }]); | |
| 247 | expect(filesOf(manifest, "shifty").filter((p) => p.startsWith("repos/"))).toEqual(["repos/shifty/fine.bundle"]); | |
| 248 | }); | |
| 249 | ||
| 250 | test("a forge that holds more files than one copy reads is left out, by name", async () => { | |
| 251 | const f = await hostForge("many", "body-many"); | |
| 252 | f.open.close(); | |
| 253 | for (const n of [1, 2, 3]) writeFileSync(join(f.source.dir, "ssh", `extra-${n}`), `${n}`); | |
| 254 | const { store, manifest } = await snap(["many"], { hostedFilesMax: 3 }); | |
| 255 | expect(manifest.hosted).toEqual([{ name: "many", at: manifest.hosted![0]!.at, repos: 0, bytes: 0, key: "none", why: "it holds more than 3 files to copy" }]); | |
| 256 | expect(filesOf(manifest, "many")).toEqual([]); | |
| 257 | await verifySnapshot(store, manifest.name); | |
| 258 | // At the limit it is kept: four files under ssh/ are over three, three are not. | |
| 259 | rmSync(join(f.source.dir, "ssh", "extra-3")); | |
| 260 | rmSync(join(f.source.dir, "apps"), { recursive: true }); | |
| 261 | const kept = await snap(["many"], { hostedFilesMax: 3 }, "2026-11-17T08:00:00Z"); | |
| 262 | expect(kept.manifest.hosted?.[0]?.why).toBeUndefined(); | |
| 263 | }); | |
| 264 | ||
| 265 | test("the forge's own lists are read in the copy's process: a view under a table's name is not run", async () => { | |
| 266 | const f = await hostForge("views", "body-views"); | |
| 267 | await commit(f, "work"); | |
| 268 | f.open.close(); | |
| 269 | // `repo` is now a view the forge wrote: whatever it selects, nothing of the snapshot runs it. | |
| 270 | const db = new Database(join(f.source.dir, "forge.db")); | |
| 271 | db.run("PRAGMA foreign_keys = OFF"); | |
| 272 | db.run("ALTER TABLE repo RENAME TO repo_kept"); | |
| 273 | db.run("CREATE VIEW repo AS SELECT * FROM repo_kept"); | |
| 274 | db.close(); | |
| 275 | const { manifest } = await snap(["views"]); | |
| 276 | // Its repositories are then the ones on its disk. | |
| 277 | expect(manifest.hosted).toMatchObject([{ name: "views", repos: 1, key: "wrapped" }]); | |
| 278 | expect(manifest.hosted?.[0]?.why).toBeUndefined(); | |
| 279 | expect(filesOf(manifest, "views")).toContain("repos/views/work.bundle"); | |
| 280 | }); | |
| 281 | ||
| 282 | test("the copy holds what a sleeping forge left in its WAL", async () => { | |
| 283 | // Asleep: killed, not closed. Its last commit is in its WAL, and the database file alone lacks | |
| 284 | // it. | |
| 285 | const asleep = await hostForge("asleep", "body-asleep"); | |
| 286 | asleep.open.db.run("PRAGMA wal_autocheckpoint = 0"); | |
| 287 | asleep.open.db.run("CREATE TABLE last_words (n INTEGER)"); | |
| 288 | asleep.open.db.run("INSERT INTO last_words VALUES (7)"); | |
| 289 | const killed = fresh("killed"); | |
| 290 | mkdirSync(killed); | |
| 291 | for (const name of ["forge.db", "forge.db-wal"]) copyFileSync(join(asleep.source.dir, name), join(killed, name)); | |
| 292 | asleep.open.close(); | |
| 293 | for (const name of ["forge.db-wal", "forge.db-shm"]) rmSync(join(asleep.source.dir, name), { force: true }); | |
| 294 | for (const name of ["forge.db", "forge.db-wal"]) copyFileSync(join(killed, name), join(asleep.source.dir, name)); | |
| 295 | rmSync(join(killed, "forge.db-wal")); | |
| 296 | const bare = new Database(join(killed, "forge.db"), { readonly: true }); | |
| 297 | expect(bare.query("SELECT name FROM sqlite_master WHERE name = 'last_words'").all()).toEqual([]); | |
| 298 | bare.close(); | |
| 299 | const { store, manifest } = await snap(["asleep"]); | |
| 300 | expect(manifest.hosted?.map((h) => [h.name, h.why])).toEqual([["asleep", undefined]]); | |
| 301 | const one = fresh("asleep-back"); | |
| 302 | await restoreHosted(store, manifest.name, "asleep", one, { passphrase: PASS }); | |
| 303 | const a = new Database(join(one, "forge.db"), { readonly: true }); | |
| 304 | expect(a.query("PRAGMA integrity_check").all()).toEqual([{ integrity_check: "ok" }]); | |
| 305 | expect(a.query("SELECT n FROM last_words").all()).toEqual([{ n: 7 }]); | |
| 306 | a.close(); | |
| 307 | }); | |
| 308 | ||
| 309 | test("the copy is one consistent moment while the forge's own process goes on writing", async () => { | |
| 310 | // Awake: its own process commits before, during and after its copy is made. | |
| 311 | const awake = await hostForge("awake", "body-awake"); | |
| 312 | const db = awake.open.db; | |
| 313 | db.run("CREATE TABLE pair_a (id INTEGER PRIMARY KEY)"); | |
| 314 | db.run("CREATE TABLE pair_b (id INTEGER PRIMARY KEY)"); | |
| 315 | db.run("CREATE TABLE pair_total (n INTEGER)"); | |
| 316 | db.run("INSERT INTO pair_total VALUES (0)"); | |
| 317 | const pair = db.transaction(() => { | |
| 318 | db.run("INSERT INTO pair_a DEFAULT VALUES"); | |
| 319 | db.run("INSERT INTO pair_b DEFAULT VALUES"); | |
| 320 | db.run("UPDATE pair_total SET n = n + 1"); | |
| 321 | }); | |
| 322 | for (let i = 0; i < 50; i++) pair(); | |
| 323 | const total = (): number => (db.query("SELECT n FROM pair_total").get() as { n: number }).n; | |
| 324 | const wrote = { before: 0, during: 0 }; | |
| 325 | const { store, manifest } = await snap(["awake"], { | |
| 326 | // The real copy, in its own process, while the forge's process goes on committing: until | |
| 327 | // the copy's file is there, and then two hundred times more. | |
| 328 | hostedCopy: async (job, ms) => { | |
| 329 | if (!job.from.endsWith("body-awake/forge.db")) return copyInProcess(job, ms); | |
| 330 | wrote.before = total(); | |
| 331 | const copying = copyInProcess(job, ms); | |
| 332 | for (let i = 0; i < 2_000_000 && !existsSync(job.to); i++) pair(); | |
| 333 | for (let i = 0; i < 200; i++) pair(); | |
| 334 | wrote.during = total(); | |
| 335 | return copying; | |
| 336 | }, | |
| 337 | }); | |
| 338 | awake.open.close(); | |
| 339 | expect(manifest.hosted?.map((h) => [h.name, h.why])).toEqual([["awake", undefined]]); | |
| 340 | expect(wrote.during).toBeGreaterThanOrEqual(wrote.before + 200); | |
| 341 | const two = fresh("awake-back"); | |
| 342 | await restoreHosted(store, manifest.name, "awake", two, { passphrase: PASS }); | |
| 343 | const b = new Database(join(two, "forge.db"), { readonly: true }); | |
| 344 | try { | |
| 345 | expect(b.query("PRAGMA integrity_check").all()).toEqual([{ integrity_check: "ok" }]); | |
| 346 | const n = (sql: string): number => (b.query(sql).get() as { n: number }).n; | |
| 347 | const rows = n("SELECT count(*) AS n FROM pair_a"); | |
| 348 | // Every transaction is in the copy whole or not at all, at a moment the forge passed through. | |
| 349 | expect(n("SELECT count(*) AS n FROM pair_b")).toBe(rows); | |
| 350 | expect(n("SELECT n FROM pair_total")).toBe(rows); | |
| 351 | expect(rows).toBeGreaterThanOrEqual(wrote.before); | |
| 352 | expect(rows).toBeLessThanOrEqual(wrote.during); | |
| 353 | } finally { | |
| 354 | b.close(); | |
| 355 | } | |
| 356 | }); | |
| 357 | ||
| 358 | test("a copy that does not end in its time is ended, and says so", async () => { | |
| 359 | const f = await hostForge("late", "body-late"); | |
| 360 | f.open.close(); | |
| 361 | const scratch = fresh("late"); | |
| 362 | mkdirSync(scratch); | |
| 363 | const made = await copyInProcess({ from: join(f.source.dir, "forge.db"), to: join(scratch, "copy.db"), root: f.source.dir }, 1); | |
| 364 | expect(made).toEqual({ ok: false, why: "its database was not copied within 1 s" }); | |
| 365 | expect(existsSync(join(scratch, "copy.db.answer"))).toBe(false); | |
| 366 | }); | |
| 367 | }); |
test/spawn-sites.test.ts +2
| @@ -87,6 +87,8 @@ const SITES: Record<string, string> = { | ||
| 87 | 87 | "bun (process.execPath) on src/main.ts in the snapshot tree", |
| 88 | 88 | "src/snapshot/drill_job.ts": |
| 89 | 89 | "runs bun on the forge's own drill_child.ts with the job as JSON on stdin; argv holds only the bun path and a fixed script, nothing from a request", |
| 90 | "src/snapshot/hosted.ts": | |
| 91 | "bun (process.execPath) on the forge's own hosted_copy.ts with an empty environment; its two arguments are a JSON job of three paths the snapshot built from a validated forge name and its own scratch directory, and the path its answer is written to: nothing from a request", | |
| 90 | 92 | "src/spawn/archive.ts": |
| 91 | 93 | "tar with fixed flags on the forge's own data directory and an archive path it derived from a validated name", |
| 92 | 94 | "src/spawn/child_db.ts": |
test/worker-database-opens.test.ts +1 −1
| @@ -65,7 +65,7 @@ const REACHED: Record<string, { opens: number; why: string }> = { | ||
| 65 | 65 | "src/release/rollback.ts": { opens: 1, why: "liveSchema reads the schema count once for a rollback or a swap, on the serving thread or in the release tool" }, |
| 66 | 66 | "src/snapshot/drill.ts": { opens: 1, why: "a restore drill opens the restored copy once, as a duty of the serving thread" }, |
| 67 | 67 | "src/snapshot/restore.ts": { opens: 1, why: "a restore reads the sealed names of the restored copy once, in the restore tool or a drill on the serving thread" }, |
| 68 | "src/snapshot/snapshot.ts": { opens: 2, why: "a snapshot counts the rows of its own copy once, and reads each hosted forge's copy once, as a duty of the serving thread; the mirror Worker imports this file for the manifest only" }, | |
| 68 | "src/snapshot/snapshot.ts": { opens: 1, why: "a snapshot counts the rows of its own copy once, as a duty of the serving thread (a hosted forge's copy is read in a process of its own, snapshot/hosted_copy.ts); the mirror Worker imports this file for the manifest only" }, | |
| 69 | 69 | "src/spawn/child_runs.ts": { opens: 2, why: "called by the reader process (spawn/child_db_reader.ts), never by a thread of the forge; cell.ts re-exports the two for tests that read a file directly" }, |
| 70 | 70 | }; |
| 71 | 71 |
new above
offline
HOSTED_FILES_MAX is a fixed 200000, so the refusal cannot be tested quickly. Let HostedTools override it and test that the forge is left out by name.