Public: anyone can read this; the crew can change it.

Each new forge gets a full copy of the forge repository: 4.7 s of the spawn and 43 MB of every forge #13826

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13818-each-new-forge-gets-a-full-copy-of-the-f → main 3 commits fork 3 threads · 3 unresolved

  • Merged. mason/13818-each-new-forge-gets-a-full-copy-of-the-f was merged into main as 1e586fb.

Review

Sign in to review this change.
4 files changed +230 −10
src/git/repos.ts +3
@@ -10,6 +10,7 @@ import { defect } from "../kernel/error.ts";
1010 import { err, ok, type Result } from "../kernel/result.ts";
1111 import { runGit, type GitFailure } from "./env.ts";
1212 import { isSafeName, isSafeRef } from "./guard.ts";
13 import { dropSeedStore } from "./seed_history.ts";
1314
1415 /** Resolve a repo's on-disk directory, refusing unsafe segments outright. */
1516 export function repoDir(root: string, owner: string, name: string): string {
@@ -187,6 +188,8 @@ export async function deleteRepoDir(
187188 if (!dir.endsWith(".git")) {
188189 defect("git.refuse_delete", `refusing to delete non-repo path: ${dir}`);
189190 }
191 // Its shared seed packs are its history too: they go whether or not the directory is still there.
192 await dropSeedStore(dir);
190193 if (!(await repoDirExists(dir)))
191194 return err({ message: `not a repository: ${dir}` });
192195 try {
src/git/seed_history.ts +123 −8
@@ -32,8 +32,9 @@
3232 * directories), plain data out (how many refs were set). Never throws.
3333 */
3434
35 import { lstat } from "node:fs/promises";
36 import { join } from "node:path";
35 import { createHash } from "node:crypto";
36 import { copyFile, link, lstat, mkdir, readdir, rename, rm } from "node:fs/promises";
37 import { basename, dirname, join } from "node:path";
3738 import { runGit } from "./env.ts";
3839
3940 /** The longest the pack may take; the fetch after it has its own. */
@@ -86,12 +87,9 @@ async function seed(from: string, to: string): Promise<number> {
8687 if (!held.ok || held.value !== "") return 0;
8788 const tips = await tipsOf(from);
8889 if (tips.length === 0) return 0;
89 const packed = await runGit(["pack-objects", "--revs", "--delta-base-offset", "-q", join(pack, "pack")], {
90 cwd: from,
91 stdin: [...new Set(tips.map((t) => t.sha))].join("\n") + "\n",
92 timeoutMs: SEED_TIMEOUT_MS,
93 });
94 if (!packed.ok) return 0;
90 const stored = await packOnce(from, tips);
91 if (stored === undefined) return 0;
92 if (!(await placeInto(stored, pack))) return 0;
9593 // `create`: each ref only where it is still absent, all or none,
9694 // never over one something else set meanwhile.
9795 const set = await runGit(["update-ref", "--stdin"], {
@@ -100,3 +98,120 @@ async function seed(from: string, to: string): Promise<number> {
10098 });
10199 return set.ok ? tips.length : 0;
102100 }
101
102 /**
103 * Where the packs made once per set of tips are kept: beside the owners'
104 * directories under the repositories root, never inside a repository, so a
105 * repository's size (a spawn plan's, a person's storage) does not count them.

Nothing removes this store when the parent repo is deleted; deleted repo history persists. Clean it on repo destroy.

106 * Same filesystem as the repositories, which a hard link needs.
107 */
108 export function seedStoreOf(from: string): string {
109 const owner = dirname(from);
110 return join(dirname(owner), ".forge-seed", basename(owner), basename(from));
111 }
112 /**
113 * Removes the packs kept for a repository that is being deleted: its history
114 * must not outlive it. `from` is the repository's directory. Never throws.
115 */
116 export async function dropSeedStore(from: string): Promise<void> {
117 try {
118 await rm(seedStoreOf(from), { recursive: true, force: true });
119 } catch {
120 // the repository is gone either way
121 }
122 }
123
124 /** Old tip sets kept beside the newest: a spawn racing a push still finds its own. */
125 const KEEP = 2;
126
127 /**
128 * The pack for exactly these tips, made once: a second child of the same
129 * parent tip finds it and packs nothing (#13818). Built in a scratch
130 * directory and renamed whole, so a reader never sees half a pack. Answers
131 * the directory holding the pack's files, or undefined when it could not be made.
132 */
133 export async function packOnce(
134 from: string,
135 tips: readonly { sha: string; ref: string }[],
136 /** Runs between the pack and its rename; a test's way to be the other spawn. */
137 afterPack: () => Promise<void> = async () => {},
138 ): Promise<string | undefined> {
139 const key = createHash("sha256")
140 .update(tips.map((t) => `${t.sha} ${t.ref}`).sort().join("\n"))
141 .digest("hex")
142 .slice(0, 32);
143 const store = seedStoreOf(from);
144 const dir = join(store, key);
145 if (await isDir(dir)) return dir;
146 const scratch = join(store, `.tmp-${key}-${process.pid}-${Date.now()}`);
147 try {
148 await mkdir(scratch, { recursive: true });
149 const packed = await runGit(["pack-objects", "--revs", "--delta-base-offset", "-q", join(scratch, "pack")], {
150 cwd: from,
151 stdin: [...new Set(tips.map((t) => t.sha))].join("\n") + "\n",
152 timeoutMs: SEED_TIMEOUT_MS,
153 });
154 if (!packed.ok) return undefined;
155 await afterPack();
156 try {
157 await rename(scratch, dir);
158 } catch {
159 if (!(await isDir(dir))) return undefined; // another spawn's rename won: its pack is the same
160 }

Dot-prefixed .tmp-* dirs left by a killed process are never pruned. Delete scratch older than the pack timeout here.

161 await prune(store, key);
162 return dir;
163 } catch {
164 return undefined;
165 } finally {
166 await rm(scratch, { recursive: true, force: true }).catch(() => {});
167 }
168 }
169
170 /** Drops the tip sets older than the newest few; never the one just used. */
171 export async function prune(store: string, keep: string, now: number = Date.now()): Promise<void> {
172 try {
173 const dirs: { name: string; at: number }[] = [];
174 for (const name of await readdir(store)) {
175 const st = await lstat(join(store, name));
176 if (name.startsWith(".")) {
177 // Scratch a killed build left: older than any live pack could take.
178 if (name.startsWith(".tmp-") && st.isDirectory() && now - st.mtimeMs > SEED_TIMEOUT_MS)
179 await rm(join(store, name), { recursive: true, force: true });
180 continue;
181 }
182 if (st.isDirectory()) dirs.push({ name, at: st.mtimeMs });
183 }
184 dirs.sort((x, y) => y.at - x.at);
185 for (const d of dirs.slice(KEEP + 1))
186 if (d.name !== keep) await rm(join(store, d.name), { recursive: true, force: true });
187 } catch {
188 // pruning is housekeeping
189 }
190 }
191
192 /**
193 * Puts the stored pack's files into the child's pack directory: a hard link
194 * (no bytes, no time), a copy where the two are on different filesystems.
195 * A pack is never written to once made, so sharing its inode is safe; the
196 * child's own repack replaces its name, never the shared bytes.
197 */
198 export async function placeInto(stored: string, pack: string, linker: typeof link = link): Promise<boolean> {
199 try {
200 const names = (await readdir(stored)).filter((n) => /^pack-[0-9a-f]+\.(?:pack|idx|rev)$/.test(n));
201 // The index last: git treats a pack as present once its .idx is.
202 names.sort((x, y) => Number(x.endsWith(".idx")) - Number(y.endsWith(".idx")));
203 if (!names.some((n) => n.endsWith(".pack")) || !names.some((n) => n.endsWith(".idx"))) return false;
204 for (const n of names) {
205 const target = join(pack, n);
206 try {
207 await linker(join(stored, n), target);
208 } catch (e) {
209 if ((e as NodeJS.ErrnoException).code === "EEXIST") continue;
210 await copyFile(join(stored, n), target);
211 }
212 }
213 return true;
214 } catch {
215 return false;
216 }
217 }
src/kernel/destroy_reach.ts +4
@@ -63,6 +63,10 @@ const unitsOn = (why: string, units: string[]): { unit: string; why: string }[]
6363 * .test.ts` fails when an entry no longer removes anything (a stale reason).
6464 */
6565 export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
66 ...unitsOn("the shared seed packs under `<repos root>/.forge-seed` (#13818): its own half-built or abandoned scratch, tip sets past the newest few, never a forge, app or repository", [
67 "src/git/seed_history.ts:packOnce",
68 "src/git/seed_history.ts:prune",
69 ]),
6670 ...unitsOn("scratch under the checks root, per run or commit (checkouts, caches, locks, the install cache): never a forge's or an app's directory", [
6771 "src/checks/service.ts:startChecks",
6872 "src/checks/service.ts:removeRunDir",
test/seed-history.test.ts +100 −2
@@ -5,11 +5,12 @@
55 */
66
77 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdir, mkdtemp, readdir, rm, symlink } from "node:fs/promises";
8 import { mkdir, mkdtemp, readdir, rm, stat, symlink, utimes } from "node:fs/promises";
99 import { tmpdir } from "node:os";
1010 import { join } from "node:path";
11 import { deleteRepoDir } from "../src/git/repos.ts";
1112 import { runGit } from "../src/git/env.ts";
12 import { seedHistory } from "../src/git/seed_history.ts";
13 import { dropSeedStore, packOnce, placeInto, prune, SEED_TIMEOUT_MS, seedHistory, seedStoreOf, tipsOf } from "../src/git/seed_history.ts";
1314
1415 const git = async (cwd: string, ...args: string[]): Promise<string> => {
1516 const r = await runGit(args, { cwd });
@@ -100,6 +101,103 @@ describe("seedHistory", () => {
100101 expect((await runGit(["for-each-ref"], { cwd: to })).ok && (await git(to, "for-each-ref"))).toBe("");
101102 });
102103
104 test("the pack is made once per set of tips and shared by hard link with each child (#13818)", async () => {
105 const a = await fresh("share-a");
106 const b = await fresh("share-b");
107 expect(await seedHistory(parent, a)).toBe(3);
108 const stored = await readdir(seedStoreOf(parent));
109 expect(stored.length).toBe(1);
110 expect(await seedHistory(parent, b)).toBe(3);
111 expect(await readdir(seedStoreOf(parent))).toEqual(stored);
112 const packs = (await readdir(join(a, "objects", "pack"))).filter((n) => n.endsWith(".pack"));
113 expect(packs.length).toBe(1);
114 expect(await readdir(join(b, "objects", "pack"))).toEqual(await readdir(join(a, "objects", "pack")));
115 const sa = await stat(join(a, "objects", "pack", packs[0]!));
116 const sb = await stat(join(b, "objects", "pack", packs[0]!));
117 expect(sa.ino).toBe(sb.ino);
118 await git(b, "fsck", "--connectivity-only", "--no-dangling");
119 });
outdated

Add tests for prune (4+ distinct tip sets), copy fallback, and key change on new tip.

120
121 test("a new tip makes a new pack beside the old one", async () => {
122 const solo = join(dir, "solo.git");
123 await git(dir, "init", "-q", "--bare", solo);
124 await git(parent, "push", "-q", solo, "main");
125 await seedHistory(solo, await fresh("key-1"));
126 const first = await readdir(seedStoreOf(solo));
127 const w = join(dir, "work");
128 await Bun.write(join(w, "c.txt"), "three\n");
129 await git(w, "add", "c.txt");
130 await git(w, "-c", "user.name=t", "-c", "[email protected]", "commit", "-q", "-m", "c");
131 await git(w, "push", "-q", solo, "HEAD:refs/heads/main");
132 await seedHistory(solo, await fresh("key-2"));
133 const second = await readdir(seedStoreOf(solo));
134 expect(first.length).toBe(1);
135 expect(second.length).toBe(2);
136 expect(second).toContain(first[0]!);
137 });
138
139 test("prune keeps the newest three and never the one in use", async () => {
140 const store = join(dir, "prune-store");
141 for (const [i, name] of ["a", "b", "c", "d", "e"].entries()) {
142 await mkdir(join(store, name), { recursive: true });
143 const t = new Date(Date.now() - (10 - i) * 1000); // e newest, a oldest
144 await utimes(join(store, name), t, t);
145 }
146 await prune(store, "a");
147 expect((await readdir(store)).sort()).toEqual(["a", "c", "d", "e"]);
148 });
149
150 test("prune removes scratch a killed build left, once older than the pack timeout, and only then", async () => {
151 const store = join(dir, "scratch-store");
152 await mkdir(join(store, ".tmp-old"), { recursive: true });
153 await mkdir(join(store, ".tmp-live"), { recursive: true });
154 const old = new Date(Date.now() - SEED_TIMEOUT_MS - 60_000);
155 await utimes(join(store, ".tmp-old"), old, old);
156 await prune(store, "none");
157 expect((await readdir(store)).sort()).toEqual([".tmp-live"]);
158 });
159
160 test("the pack another spawn renamed first is the one used", async () => {
161 const tips = await tipsOf(parent);
162 const made = await packOnce(parent, tips); // makes it
163 await rm(made!, { recursive: true });
164 const lost = await packOnce(parent, tips, async () => {
165 await mkdir(made!, { recursive: true });
166 await Bun.write(join(made!, "marker"), "theirs");
167 });
168 expect(lost).toBe(made!);
169 expect(await Bun.file(join(made!, "marker")).exists()).toBe(true);
170 expect((await readdir(seedStoreOf(parent))).filter((n) => n.startsWith(".tmp-"))).toEqual([]);
171 await rm(made!, { recursive: true }); // the stand-in is not a pack: leave none for the next test
172 });
173
174 test("where a hard link cannot be made the files are copied", async () => {
175 const tips = await tipsOf(parent);
176 const stored = (await packOnce(parent, tips))!;
177 const to = await fresh("copied");
178 const refuse = (async () => {
179 throw Object.assign(new Error("cross-device"), { code: "EXDEV" });
180 }) as unknown as typeof import("node:fs/promises").link;
181 expect(await placeInto(stored, join(to, "objects", "pack"), refuse)).toBe(true);
182 const names = (await readdir(join(to, "objects", "pack"))).filter((n) => n.startsWith("pack-"));
183 expect(names.some((n) => n.endsWith(".pack"))).toBe(true);
184 const n = names.find((x) => x.endsWith(".pack"))!;
185 expect((await stat(join(to, "objects", "pack", n))).ino).not.toBe((await stat(join(stored, n))).ino);
186 });
187
188 test("a deleted repository's store goes with it", async () => {
189 const repo = join(dir, "owners", "o", "gone.git");
190 await mkdir(repo, { recursive: true });
191 const store = seedStoreOf(repo);
192 await mkdir(join(store, "k"), { recursive: true });
193 await Bun.write(join(repo, "HEAD"), "ref: refs/heads/main\n");
194 expect((await deleteRepoDir(repo)).ok).toBe(true);
195 expect(await stat(store).catch(() => undefined)).toBeUndefined();
196 await mkdir(join(store, "k"), { recursive: true });
197 await dropSeedStore(repo);
198 expect(await stat(store).catch(() => undefined)).toBeUndefined();
199 });
200
103201 test("an address that is not a repository does nothing", async () => {
104202 expect(await seedHistory(join(dir, "nowhere"), await fresh("orphan"))).toBe(0);
105203 expect(await seedHistory(parent, join(dir, "nowhere"))).toBe(0);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs