Public: anyone can read this; the crew can change it.
A hosted forge's bundle leaves out what it only mirrors from its parent #14014
Check · Review · Land · Live done — Landed and live in production.
trevato/hosted-bundles-leave-out-what-they-mirror → main 1 commit
4 files
changed +61
−4
docs/features/hosted-backup.md +1 −1
| @@ -5,7 +5,7 @@ Each forge this one hosts goes into the nightly snapshot (#13819), under `childr | ||
| 5 | 5 | | What | Where in the snapshot | |
| 6 | 6 | | --- | --- | |
| 7 | 7 | | Database (a consistent copy, taken while it runs) | `forge.db` | |
| 8 | | Repositories | `repos/<owner>/<name>.bundle` | | |
| 8 | | Repositories, every ref but `refs/upstream/*` (what a forge only mirrors from its parent, fetched again every hour it is awake: with those in, an awake fork went up whole every night) | `repos/<owner>/<name>.bundle` | | |
| 9 | 9 | | SSH host keys | `ssh/` | |
| 10 | 10 | | Its key, wrapped under `FORGE_SNAPSHOT_PASSPHRASE` | `secret.key.wrapped` (without a passphrase the key stays home) | |
| 11 | 11 | | Every app's production data | alongside | |
src/snapshot/bundle.ts +8 −2
| @@ -92,6 +92,11 @@ export interface BundleOptions { | ||
| 92 | 92 | readonly sizes?: Map<string, number> | undefined; |
| 93 | 93 | /** Test seam: the git runner. */ |
| 94 | 94 | readonly git?: typeof runGit | undefined; |
| 95 | /** | |
| 96 | * Refs left out, by prefix (`refs/upstream/`): ones a fetch brings back, so they are not history | |
| 97 | * to keep and their moving does not make the bundle another file. | |
| 98 | */ | |
| 99 | readonly excludeUnder?: readonly string[] | undefined; | |
| 95 | 100 | } |
| 96 | 101 | |
| 97 | 102 | const ONE_THREAD = ["-c", "pack.threads=1"] as const; |
| @@ -112,7 +117,8 @@ export async function bundleRepo(dir: string, to: string, options: BundleOptions | ||
| 112 | 117 | const reason = (r: Awaited<ReturnType<typeof runGit>>): string => |
| 113 | 118 | late ?? (r.ok ? "" : r.error.stderr.split("\n")[0] || `git ended with ${r.error.code}`); |
| 114 | 119 | |
| 115 | const whole = await run([...ONE_THREAD, "bundle", "create", to, "--all"]); | |
| 120 | const left = options.excludeUnder ?? []; | |
| 121 | const whole = await run([...ONE_THREAD, "bundle", "create", to, ...left.map((prefix) => `--exclude=${prefix}*`), "--all"]); | |
| 116 | 122 | if (whole.ok) return { ok: true, brokenRefs: [] }; |
| 117 | 123 | const why = reason(whole); |
| 118 | 124 | if (late !== undefined) return { ok: false, why, empty: false }; |
| @@ -125,7 +131,7 @@ export async function bundleRepo(dir: string, to: string, options: BundleOptions | ||
| 125 | 131 | const refs = listed.value |
| 126 | 132 | .split("\n") |
| 127 | 133 | .map((line) => line.split(" ")) |
| 128 | .filter((p): p is [string, string] => p.length === 2 && p[0] !== "" && p[1] !== ""); | |
| 134 | .filter((p): p is [string, string] => p.length === 2 && p[0] !== "" && p[1] !== "" && !left.some((prefix) => p[0]!.startsWith(prefix))); | |
| 129 | 135 | const verified: string[] = []; |
| 130 | 136 | const bad = new Set<string>(); |
| 131 | 137 | // Halving: a set that walks whole is sound; one that does not is split until the ref is found. |
src/snapshot/hosted.ts +9 −1
| @@ -164,6 +164,14 @@ export function readHosted(dataDir: string): HostedEntry[] | undefined { | ||
| 164 | 164 | |
| 165 | 165 | // ---- the copy of one hosted forge, from outside its cell ---- |
| 166 | 166 | |
| 167 | /** | |
| 168 | * Refs a hosted forge's bundles leave out: what it only mirrors from its parent. A forge that | |
| 169 | * follows fetches its parent's code into `refs/upstream/*` of its own fork every hour it is awake | |
| 170 | * (`spawn/upstream.ts`). With those in, its fork was another bundle each night it had been awake, | |
| 171 | * and a bundle that changed goes up whole. The next fetch brings them back. | |
| 172 | */ | |
| 173 | export const REFETCHED_REFS = ["refs/upstream/"] as const; | |
| 174 | ||
| 167 | 175 | /** A forge left out, with a reason a person may read: the host's words, never the forge's. */ |
| 168 | 176 | export class HostedRefusal extends Error {} |
| 169 | 177 | |
| @@ -476,7 +484,7 @@ export async function copyHosted(child: HostedSource, tools: HostedTools): Promi | ||
| 476 | 484 | } |
| 477 | 485 | const bundle = join(tools.scratch, `hosted-${child.name}-${owner}--${name}.bundle`); |
| 478 | 486 | try { |
| 479 | const made = await bundleRepo(dir, bundle, { deadlineMs: tools.bundleDeadlineMs, sizes, git }); | |
| 487 | const made = await bundleRepo(dir, bundle, { deadlineMs: tools.bundleDeadlineMs, sizes, git, excludeUnder: REFETCHED_REFS }); | |
| 480 | 488 | if (!made.ok) { |
| 481 | 489 | absent.push({ repo, why: made.why.slice(0, 200), ...(made.empty ? { empty: true as const } : {}) }); |
| 482 | 490 | continue; |
test/snapshot-hosted-inside.test.ts +43
| @@ -355,6 +355,49 @@ describe("a host's snapshot of the forges it hosts", () => { | ||
| 355 | 355 | } |
| 356 | 356 | }); |
| 357 | 357 | |
| 358 | test("a fork that only fetched its parent's code is the same bundle: the refs it mirrors are left out, and the next fetch brings them back", async () => { | |
| 359 | const f = await hostForge("follower", "body-follower"); | |
| 360 | await commit(f, "forge"); | |
| 361 | f.open.close(); | |
| 362 | const fork = repoDir(join(f.source.dir, "repos"), "follower", "forge"); | |
| 363 | const git = async (args: string[]): Promise<string> => { | |
| 364 | const r = await runGit(args, { cwd: fork }); | |
| 365 | if (!r.ok) throw new Error(r.error.stderr); | |
| 366 | return r.value; | |
| 367 | }; | |
| 368 | // Its bundle as it is with nothing fetched: every ref of its own. | |
| 369 | const own = await git(["for-each-ref", "--format=%(refname) %(objectname)"]); | |
| 370 | const before = fresh("before.bundle"); | |
| 371 | await git(["-c", "pack.threads=1", "bundle", "create", before, "--all"]); | |
| 372 | // Then its parent moved and it fetched its parent's code as a following forge does | |
| 373 | // (`spawn/upstream.ts`): into refs/upstream, no tags, the fetched pack kept as a pack. | |
| 374 | const parent = fresh("parent.git"); | |
| 375 | const cloned = await runGit(["clone", "--quiet", "--bare", fork, parent]); | |
| 376 | if (!cloned.ok) throw new Error(cloned.error.stderr); | |
| 377 | const inParent = async (args: string[]): Promise<string> => { | |
| 378 | const r = await runGit(args, { cwd: parent }); | |
| 379 | if (!r.ok) throw new Error(r.error.stderr); | |
| 380 | return r.value; | |
| 381 | }; | |
| 382 | const tree = await inParent(["rev-parse", "refs/heads/main^{tree}"]); | |
| 383 | const moved = await inParent(["-c", "user.name=p", "-c", "[email protected]", "commit-tree", tree, "-p", "refs/heads/main", "-m", "the parent moved"]); | |
| 384 | await inParent(["update-ref", "refs/heads/main", moved]); | |
| 385 | await inParent(["update-ref", "refs/heads/next", moved]); | |
| 386 | await git(["-c", "fetch.unpackLimit=1", "fetch", "--quiet", "--no-tags", "--prune", "--", parent, "+refs/heads/*:refs/upstream/*"]); | |
| 387 | expect(await git(["for-each-ref", "--format=%(refname)", "refs/upstream"])).toBe("refs/upstream/main\nrefs/upstream/next"); | |
| 388 | expect(await git(["rev-parse", "refs/upstream/main"])).toBe(moved); | |
| 389 | const { store, manifest } = await snap(["follower"]); | |
| 390 | // The snapshot's bundle is that same file, byte for byte: nothing new to send for the fork. | |
| 391 | const kept = manifest.files.find((x) => x.path === "children/follower/repos/follower/forge.bundle"); | |
| 392 | expect(kept?.sha256).toBe(new Bun.CryptoHasher("sha256").update(readFileSync(before)).digest("hex")); | |
| 393 | // Restored, it holds every ref of its own at the same commit and none it only mirrored. | |
| 394 | const back = fresh("follower-back"); | |
| 395 | await restoreHosted(store, manifest.name, "follower", back, { passphrase: PASS }); | |
| 396 | const restored = await runGit(["for-each-ref", "--format=%(refname) %(objectname)"], { cwd: repoDir(join(back, "repos"), "follower", "forge") }); | |
| 397 | expect(restored.ok && restored.value).toBe(own); | |
| 398 | expect(own).toContain("refs/heads/main"); | |
| 399 | }); | |
| 400 | ||
| 358 | 401 | test("a copy that does not end in its time is ended, and says so", async () => { |
| 359 | 402 | const f = await hostForge("late", "body-late"); |
| 360 | 403 | f.open.close(); |
new above
offline