Public: anyone can read this; the crew can change it.

A hosted forge's bundle leaves out what it only mirrors from its parent #14014

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

trevato/hosted-bundles-leave-out-what-they-mirror → main 1 commit

  • Merged. trevato/hosted-bundles-leave-out-what-they-mirror was merged into main as 9db4265.

Review

Sign in to review this change.
4 files changed +61 −4
docs/features/hosted-backup.md +1 −1
@@ -5,7 +5,7 @@ Each forge this one hosts goes into the nightly snapshot (#13819), under `childr
55 | What | Where in the snapshot |
66 | --- | --- |
77 | Database (a consistent copy, taken while it runs) | `forge.db` |
8 | Repositories | `repos/<owner>/<name>.bundle` |
8 | Repositories, every ref but `refs/upstream/*` (what a forge only mirrors from its parent, fetched again every hour it is awake: with those in, an awake fork went up whole every night) | `repos/<owner>/<name>.bundle` |
99 | SSH host keys | `ssh/` |
1010 | Its key, wrapped under `FORGE_SNAPSHOT_PASSPHRASE` | `secret.key.wrapped` (without a passphrase the key stays home) |
1111 | Every app's production data | alongside |
src/snapshot/bundle.ts +8 −2
@@ -92,6 +92,11 @@ export interface BundleOptions {
9292 readonly sizes?: Map<string, number> | undefined;
9393 /** Test seam: the git runner. */
9494 readonly git?: typeof runGit | undefined;
95 /**
96 * Refs left out, by prefix (`refs/upstream/`): ones a fetch brings back, so they are not history
97 * to keep and their moving does not make the bundle another file.
98 */
99 readonly excludeUnder?: readonly string[] | undefined;
95100 }
96101
97102 const ONE_THREAD = ["-c", "pack.threads=1"] as const;
@@ -112,7 +117,8 @@ export async function bundleRepo(dir: string, to: string, options: BundleOptions
112117 const reason = (r: Awaited<ReturnType<typeof runGit>>): string =>
113118 late ?? (r.ok ? "" : r.error.stderr.split("\n")[0] || `git ended with ${r.error.code}`);
114119
115 const whole = await run([...ONE_THREAD, "bundle", "create", to, "--all"]);
120 const left = options.excludeUnder ?? [];
121 const whole = await run([...ONE_THREAD, "bundle", "create", to, ...left.map((prefix) => `--exclude=${prefix}*`), "--all"]);
116122 if (whole.ok) return { ok: true, brokenRefs: [] };
117123 const why = reason(whole);
118124 if (late !== undefined) return { ok: false, why, empty: false };
@@ -125,7 +131,7 @@ export async function bundleRepo(dir: string, to: string, options: BundleOptions
125131 const refs = listed.value
126132 .split("\n")
127133 .map((line) => line.split(" "))
128 .filter((p): p is [string, string] => p.length === 2 && p[0] !== "" && p[1] !== "");
134 .filter((p): p is [string, string] => p.length === 2 && p[0] !== "" && p[1] !== "" && !left.some((prefix) => p[0]!.startsWith(prefix)));
129135 const verified: string[] = [];
130136 const bad = new Set<string>();
131137 // Halving: a set that walks whole is sound; one that does not is split until the ref is found.
src/snapshot/hosted.ts +9 −1
@@ -164,6 +164,14 @@ export function readHosted(dataDir: string): HostedEntry[] | undefined {
164164
165165 // ---- the copy of one hosted forge, from outside its cell ----
166166
167 /**
168 * Refs a hosted forge's bundles leave out: what it only mirrors from its parent. A forge that
169 * follows fetches its parent's code into `refs/upstream/*` of its own fork every hour it is awake
170 * (`spawn/upstream.ts`). With those in, its fork was another bundle each night it had been awake,
171 * and a bundle that changed goes up whole. The next fetch brings them back.
172 */
173 export const REFETCHED_REFS = ["refs/upstream/"] as const;
174
167175 /** A forge left out, with a reason a person may read: the host's words, never the forge's. */
168176 export class HostedRefusal extends Error {}
169177
@@ -476,7 +484,7 @@ export async function copyHosted(child: HostedSource, tools: HostedTools): Promi
476484 }
477485 const bundle = join(tools.scratch, `hosted-${child.name}-${owner}--${name}.bundle`);
478486 try {
479 const made = await bundleRepo(dir, bundle, { deadlineMs: tools.bundleDeadlineMs, sizes, git });
487 const made = await bundleRepo(dir, bundle, { deadlineMs: tools.bundleDeadlineMs, sizes, git, excludeUnder: REFETCHED_REFS });
480488 if (!made.ok) {
481489 absent.push({ repo, why: made.why.slice(0, 200), ...(made.empty ? { empty: true as const } : {}) });
482490 continue;
test/snapshot-hosted-inside.test.ts +43
@@ -355,6 +355,49 @@ describe("a host's snapshot of the forges it hosts", () => {
355355 }
356356 });
357357
358 test("a fork that only fetched its parent's code is the same bundle: the refs it mirrors are left out, and the next fetch brings them back", async () => {
359 const f = await hostForge("follower", "body-follower");
360 await commit(f, "forge");
361 f.open.close();
362 const fork = repoDir(join(f.source.dir, "repos"), "follower", "forge");
363 const git = async (args: string[]): Promise<string> => {
364 const r = await runGit(args, { cwd: fork });
365 if (!r.ok) throw new Error(r.error.stderr);
366 return r.value;
367 };
368 // Its bundle as it is with nothing fetched: every ref of its own.
369 const own = await git(["for-each-ref", "--format=%(refname) %(objectname)"]);
370 const before = fresh("before.bundle");
371 await git(["-c", "pack.threads=1", "bundle", "create", before, "--all"]);
372 // Then its parent moved and it fetched its parent's code as a following forge does
373 // (`spawn/upstream.ts`): into refs/upstream, no tags, the fetched pack kept as a pack.
374 const parent = fresh("parent.git");
375 const cloned = await runGit(["clone", "--quiet", "--bare", fork, parent]);
376 if (!cloned.ok) throw new Error(cloned.error.stderr);
377 const inParent = async (args: string[]): Promise<string> => {
378 const r = await runGit(args, { cwd: parent });
379 if (!r.ok) throw new Error(r.error.stderr);
380 return r.value;
381 };
382 const tree = await inParent(["rev-parse", "refs/heads/main^{tree}"]);
383 const moved = await inParent(["-c", "user.name=p", "-c", "[email protected]", "commit-tree", tree, "-p", "refs/heads/main", "-m", "the parent moved"]);
384 await inParent(["update-ref", "refs/heads/main", moved]);
385 await inParent(["update-ref", "refs/heads/next", moved]);
386 await git(["-c", "fetch.unpackLimit=1", "fetch", "--quiet", "--no-tags", "--prune", "--", parent, "+refs/heads/*:refs/upstream/*"]);
387 expect(await git(["for-each-ref", "--format=%(refname)", "refs/upstream"])).toBe("refs/upstream/main\nrefs/upstream/next");
388 expect(await git(["rev-parse", "refs/upstream/main"])).toBe(moved);
389 const { store, manifest } = await snap(["follower"]);
390 // The snapshot's bundle is that same file, byte for byte: nothing new to send for the fork.
391 const kept = manifest.files.find((x) => x.path === "children/follower/repos/follower/forge.bundle");
392 expect(kept?.sha256).toBe(new Bun.CryptoHasher("sha256").update(readFileSync(before)).digest("hex"));
393 // Restored, it holds every ref of its own at the same commit and none it only mirrored.
394 const back = fresh("follower-back");
395 await restoreHosted(store, manifest.name, "follower", back, { passphrase: PASS });
396 const restored = await runGit(["for-each-ref", "--format=%(refname) %(objectname)"], { cwd: repoDir(join(back, "repos"), "follower", "forge") });
397 expect(restored.ok && restored.value).toBe(own);
398 expect(own).toContain("refs/heads/main");
399 });
400
358401 test("a copy that does not end in its time is ended, and says so", async () => {
359402 const f = await hostForge("late", "body-late");
360403 f.open.close();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs