Public: anyone can read this; the crew can change it.
Each new forge gets a full copy of the forge repository: 4.7 s of the spawn and 43 MB of every forge #13826
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13818-each-new-forge-gets-a-full-copy-of-the-f → main 3 commits fork 3 threads · 3 unresolved
4 files
changed +230
−10
src/git/repos.ts +3
| @@ -10,6 +10,7 @@ import { defect } from "../kernel/error.ts"; | ||
| 10 | 10 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 11 | 11 | import { runGit, type GitFailure } from "./env.ts"; |
| 12 | 12 | import { isSafeName, isSafeRef } from "./guard.ts"; |
| 13 | import { dropSeedStore } from "./seed_history.ts"; | |
| 13 | 14 | |
| 14 | 15 | /** Resolve a repo's on-disk directory, refusing unsafe segments outright. */ |
| 15 | 16 | export function repoDir(root: string, owner: string, name: string): string { |
| @@ -187,6 +188,8 @@ export async function deleteRepoDir( | ||
| 187 | 188 | if (!dir.endsWith(".git")) { |
| 188 | 189 | defect("git.refuse_delete", `refusing to delete non-repo path: ${dir}`); |
| 189 | 190 | } |
| 191 | // Its shared seed packs are its history too: they go whether or not the directory is still there. | |
| 192 | await dropSeedStore(dir); | |
| 190 | 193 | if (!(await repoDirExists(dir))) |
| 191 | 194 | return err({ message: `not a repository: ${dir}` }); |
| 192 | 195 | try { |
src/git/seed_history.ts +123 −8
| @@ -32,8 +32,9 @@ | ||
| 32 | 32 | * directories), plain data out (how many refs were set). Never throws. |
| 33 | 33 | */ |
| 34 | 34 | |
| 35 | import { lstat } from "node:fs/promises"; | |
| 36 | import { join } from "node:path"; | |
| 35 | import { createHash } from "node:crypto"; | |
| 36 | import { copyFile, link, lstat, mkdir, readdir, rename, rm } from "node:fs/promises"; | |
| 37 | import { basename, dirname, join } from "node:path"; | |
| 37 | 38 | import { runGit } from "./env.ts"; |
| 38 | 39 | |
| 39 | 40 | /** The longest the pack may take; the fetch after it has its own. */ |
| @@ -86,12 +87,9 @@ async function seed(from: string, to: string): Promise<number> { | ||
| 86 | 87 | if (!held.ok || held.value !== "") return 0; |
| 87 | 88 | const tips = await tipsOf(from); |
| 88 | 89 | if (tips.length === 0) return 0; |
| 89 | const packed = await runGit(["pack-objects", "--revs", "--delta-base-offset", "-q", join(pack, "pack")], { | |
| 90 | cwd: from, | |
| 91 | stdin: [...new Set(tips.map((t) => t.sha))].join("\n") + "\n", | |
| 92 | timeoutMs: SEED_TIMEOUT_MS, | |
| 93 | }); | |
| 94 | if (!packed.ok) return 0; | |
| 90 | const stored = await packOnce(from, tips); | |
| 91 | if (stored === undefined) return 0; | |
| 92 | if (!(await placeInto(stored, pack))) return 0; | |
| 95 | 93 | // `create`: each ref only where it is still absent, all or none, |
| 96 | 94 | // never over one something else set meanwhile. |
| 97 | 95 | const set = await runGit(["update-ref", "--stdin"], { |
| @@ -100,3 +98,120 @@ async function seed(from: string, to: string): Promise<number> { | ||
| 100 | 98 | }); |
| 101 | 99 | return set.ok ? tips.length : 0; |
| 102 | 100 | } |
| 101 | ||
| 102 | /** | |
| 103 | * Where the packs made once per set of tips are kept: beside the owners' | |
| 104 | * directories under the repositories root, never inside a repository, so a | |
| 105 | * repository's size (a spawn plan's, a person's storage) does not count them. | |
| 106 | * Same filesystem as the repositories, which a hard link needs. | |
| 107 | */ | |
| 108 | export function seedStoreOf(from: string): string { | |
| 109 | const owner = dirname(from); | |
| 110 | return join(dirname(owner), ".forge-seed", basename(owner), basename(from)); | |
| 111 | } | |
| 112 | /** | |
| 113 | * Removes the packs kept for a repository that is being deleted: its history | |
| 114 | * must not outlive it. `from` is the repository's directory. Never throws. | |
| 115 | */ | |
| 116 | export async function dropSeedStore(from: string): Promise<void> { | |
| 117 | try { | |
| 118 | await rm(seedStoreOf(from), { recursive: true, force: true }); | |
| 119 | } catch { | |
| 120 | // the repository is gone either way | |
| 121 | } | |
| 122 | } | |
| 123 | ||
| 124 | /** Old tip sets kept beside the newest: a spawn racing a push still finds its own. */ | |
| 125 | const KEEP = 2; | |
| 126 | ||
| 127 | /** | |
| 128 | * The pack for exactly these tips, made once: a second child of the same | |
| 129 | * parent tip finds it and packs nothing (#13818). Built in a scratch | |
| 130 | * directory and renamed whole, so a reader never sees half a pack. Answers | |
| 131 | * the directory holding the pack's files, or undefined when it could not be made. | |
| 132 | */ | |
| 133 | export async function packOnce( | |
| 134 | from: string, | |
| 135 | tips: readonly { sha: string; ref: string }[], | |
| 136 | /** Runs between the pack and its rename; a test's way to be the other spawn. */ | |
| 137 | afterPack: () => Promise<void> = async () => {}, | |
| 138 | ): Promise<string | undefined> { | |
| 139 | const key = createHash("sha256") | |
| 140 | .update(tips.map((t) => `${t.sha} ${t.ref}`).sort().join("\n")) | |
| 141 | .digest("hex") | |
| 142 | .slice(0, 32); | |
| 143 | const store = seedStoreOf(from); | |
| 144 | const dir = join(store, key); | |
| 145 | if (await isDir(dir)) return dir; | |
| 146 | const scratch = join(store, `.tmp-${key}-${process.pid}-${Date.now()}`); | |
| 147 | try { | |
| 148 | await mkdir(scratch, { recursive: true }); | |
| 149 | const packed = await runGit(["pack-objects", "--revs", "--delta-base-offset", "-q", join(scratch, "pack")], { | |
| 150 | cwd: from, | |
| 151 | stdin: [...new Set(tips.map((t) => t.sha))].join("\n") + "\n", | |
| 152 | timeoutMs: SEED_TIMEOUT_MS, | |
| 153 | }); | |
| 154 | if (!packed.ok) return undefined; | |
| 155 | await afterPack(); | |
| 156 | try { | |
| 157 | await rename(scratch, dir); | |
| 158 | } catch { | |
| 159 | if (!(await isDir(dir))) return undefined; // another spawn's rename won: its pack is the same | |
| 160 | } | |
Dot-prefixed |
||
| 161 | await prune(store, key); | |
| 162 | return dir; | |
| 163 | } catch { | |
| 164 | return undefined; | |
| 165 | } finally { | |
| 166 | await rm(scratch, { recursive: true, force: true }).catch(() => {}); | |
| 167 | } | |
| 168 | } | |
| 169 | ||
| 170 | /** Drops the tip sets older than the newest few; never the one just used. */ | |
| 171 | export async function prune(store: string, keep: string, now: number = Date.now()): Promise<void> { | |
| 172 | try { | |
| 173 | const dirs: { name: string; at: number }[] = []; | |
| 174 | for (const name of await readdir(store)) { | |
| 175 | const st = await lstat(join(store, name)); | |
| 176 | if (name.startsWith(".")) { | |
| 177 | // Scratch a killed build left: older than any live pack could take. | |
| 178 | if (name.startsWith(".tmp-") && st.isDirectory() && now - st.mtimeMs > SEED_TIMEOUT_MS) | |
| 179 | await rm(join(store, name), { recursive: true, force: true }); | |
| 180 | continue; | |
| 181 | } | |
| 182 | if (st.isDirectory()) dirs.push({ name, at: st.mtimeMs }); | |
| 183 | } | |
| 184 | dirs.sort((x, y) => y.at - x.at); | |
| 185 | for (const d of dirs.slice(KEEP + 1)) | |
| 186 | if (d.name !== keep) await rm(join(store, d.name), { recursive: true, force: true }); | |
| 187 | } catch { | |
| 188 | // pruning is housekeeping | |
| 189 | } | |
| 190 | } | |
| 191 | ||
| 192 | /** | |
| 193 | * Puts the stored pack's files into the child's pack directory: a hard link | |
| 194 | * (no bytes, no time), a copy where the two are on different filesystems. | |
| 195 | * A pack is never written to once made, so sharing its inode is safe; the | |
| 196 | * child's own repack replaces its name, never the shared bytes. | |
| 197 | */ | |
| 198 | export async function placeInto(stored: string, pack: string, linker: typeof link = link): Promise<boolean> { | |
| 199 | try { | |
| 200 | const names = (await readdir(stored)).filter((n) => /^pack-[0-9a-f]+\.(?:pack|idx|rev)$/.test(n)); | |
| 201 | // The index last: git treats a pack as present once its .idx is. | |
| 202 | names.sort((x, y) => Number(x.endsWith(".idx")) - Number(y.endsWith(".idx"))); | |
| 203 | if (!names.some((n) => n.endsWith(".pack")) || !names.some((n) => n.endsWith(".idx"))) return false; | |
| 204 | for (const n of names) { | |
| 205 | const target = join(pack, n); | |
| 206 | try { | |
| 207 | await linker(join(stored, n), target); | |
| 208 | } catch (e) { | |
| 209 | if ((e as NodeJS.ErrnoException).code === "EEXIST") continue; | |
| 210 | await copyFile(join(stored, n), target); | |
| 211 | } | |
| 212 | } | |
| 213 | return true; | |
| 214 | } catch { | |
| 215 | return false; | |
| 216 | } | |
| 217 | } |
src/kernel/destroy_reach.ts +4
| @@ -63,6 +63,10 @@ const unitsOn = (why: string, units: string[]): { unit: string; why: string }[] | ||
| 63 | 63 | * .test.ts` fails when an entry no longer removes anything (a stale reason). |
| 64 | 64 | */ |
| 65 | 65 | export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [ |
| 66 | ...unitsOn("the shared seed packs under `<repos root>/.forge-seed` (#13818): its own half-built or abandoned scratch, tip sets past the newest few, never a forge, app or repository", [ | |
| 67 | "src/git/seed_history.ts:packOnce", | |
| 68 | "src/git/seed_history.ts:prune", | |
| 69 | ]), | |
| 66 | 70 | ...unitsOn("scratch under the checks root, per run or commit (checkouts, caches, locks, the install cache): never a forge's or an app's directory", [ |
| 67 | 71 | "src/checks/service.ts:startChecks", |
| 68 | 72 | "src/checks/service.ts:removeRunDir", |
test/seed-history.test.ts +100 −2
| @@ -5,11 +5,12 @@ | ||
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 8 | import { mkdir, mkdtemp, readdir, rm, symlink } from "node:fs/promises"; | |
| 8 | import { mkdir, mkdtemp, readdir, rm, stat, symlink, utimes } from "node:fs/promises"; | |
| 9 | 9 | import { tmpdir } from "node:os"; |
| 10 | 10 | import { join } from "node:path"; |
| 11 | import { deleteRepoDir } from "../src/git/repos.ts"; | |
| 11 | 12 | import { runGit } from "../src/git/env.ts"; |
| 12 | import { seedHistory } from "../src/git/seed_history.ts"; | |
| 13 | import { dropSeedStore, packOnce, placeInto, prune, SEED_TIMEOUT_MS, seedHistory, seedStoreOf, tipsOf } from "../src/git/seed_history.ts"; | |
| 13 | 14 | |
| 14 | 15 | const git = async (cwd: string, ...args: string[]): Promise<string> => { |
| 15 | 16 | const r = await runGit(args, { cwd }); |
| @@ -100,6 +101,103 @@ describe("seedHistory", () => { | ||
| 100 | 101 | expect((await runGit(["for-each-ref"], { cwd: to })).ok && (await git(to, "for-each-ref"))).toBe(""); |
| 101 | 102 | }); |
| 102 | 103 | |
| 104 | test("the pack is made once per set of tips and shared by hard link with each child (#13818)", async () => { | |
| 105 | const a = await fresh("share-a"); | |
| 106 | const b = await fresh("share-b"); | |
| 107 | expect(await seedHistory(parent, a)).toBe(3); | |
| 108 | const stored = await readdir(seedStoreOf(parent)); | |
| 109 | expect(stored.length).toBe(1); | |
| 110 | expect(await seedHistory(parent, b)).toBe(3); | |
| 111 | expect(await readdir(seedStoreOf(parent))).toEqual(stored); | |
| 112 | const packs = (await readdir(join(a, "objects", "pack"))).filter((n) => n.endsWith(".pack")); | |
| 113 | expect(packs.length).toBe(1); | |
| 114 | expect(await readdir(join(b, "objects", "pack"))).toEqual(await readdir(join(a, "objects", "pack"))); | |
| 115 | const sa = await stat(join(a, "objects", "pack", packs[0]!)); | |
| 116 | const sb = await stat(join(b, "objects", "pack", packs[0]!)); | |
| 117 | expect(sa.ino).toBe(sb.ino); | |
| 118 | await git(b, "fsck", "--connectivity-only", "--no-dangling"); | |
| 119 | }); | |
outdated
Add tests for prune (4+ distinct tip sets), copy fallback, and key change on new tip. |
||
| 120 | ||
| 121 | test("a new tip makes a new pack beside the old one", async () => { | |
| 122 | const solo = join(dir, "solo.git"); | |
| 123 | await git(dir, "init", "-q", "--bare", solo); | |
| 124 | await git(parent, "push", "-q", solo, "main"); | |
| 125 | await seedHistory(solo, await fresh("key-1")); | |
| 126 | const first = await readdir(seedStoreOf(solo)); | |
| 127 | const w = join(dir, "work"); | |
| 128 | await Bun.write(join(w, "c.txt"), "three\n"); | |
| 129 | await git(w, "add", "c.txt"); | |
| 130 | await git(w, "-c", "user.name=t", "-c", "[email protected]", "commit", "-q", "-m", "c"); | |
| 131 | await git(w, "push", "-q", solo, "HEAD:refs/heads/main"); | |
| 132 | await seedHistory(solo, await fresh("key-2")); | |
| 133 | const second = await readdir(seedStoreOf(solo)); | |
| 134 | expect(first.length).toBe(1); | |
| 135 | expect(second.length).toBe(2); | |
| 136 | expect(second).toContain(first[0]!); | |
| 137 | }); | |
| 138 | ||
| 139 | test("prune keeps the newest three and never the one in use", async () => { | |
| 140 | const store = join(dir, "prune-store"); | |
| 141 | for (const [i, name] of ["a", "b", "c", "d", "e"].entries()) { | |
| 142 | await mkdir(join(store, name), { recursive: true }); | |
| 143 | const t = new Date(Date.now() - (10 - i) * 1000); // e newest, a oldest | |
| 144 | await utimes(join(store, name), t, t); | |
| 145 | } | |
| 146 | await prune(store, "a"); | |
| 147 | expect((await readdir(store)).sort()).toEqual(["a", "c", "d", "e"]); | |
| 148 | }); | |
| 149 | ||
| 150 | test("prune removes scratch a killed build left, once older than the pack timeout, and only then", async () => { | |
| 151 | const store = join(dir, "scratch-store"); | |
| 152 | await mkdir(join(store, ".tmp-old"), { recursive: true }); | |
| 153 | await mkdir(join(store, ".tmp-live"), { recursive: true }); | |
| 154 | const old = new Date(Date.now() - SEED_TIMEOUT_MS - 60_000); | |
| 155 | await utimes(join(store, ".tmp-old"), old, old); | |
| 156 | await prune(store, "none"); | |
| 157 | expect((await readdir(store)).sort()).toEqual([".tmp-live"]); | |
| 158 | }); | |
| 159 | ||
| 160 | test("the pack another spawn renamed first is the one used", async () => { | |
| 161 | const tips = await tipsOf(parent); | |
| 162 | const made = await packOnce(parent, tips); // makes it | |
| 163 | await rm(made!, { recursive: true }); | |
| 164 | const lost = await packOnce(parent, tips, async () => { | |
| 165 | await mkdir(made!, { recursive: true }); | |
| 166 | await Bun.write(join(made!, "marker"), "theirs"); | |
| 167 | }); | |
| 168 | expect(lost).toBe(made!); | |
| 169 | expect(await Bun.file(join(made!, "marker")).exists()).toBe(true); | |
| 170 | expect((await readdir(seedStoreOf(parent))).filter((n) => n.startsWith(".tmp-"))).toEqual([]); | |
| 171 | await rm(made!, { recursive: true }); // the stand-in is not a pack: leave none for the next test | |
| 172 | }); | |
| 173 | ||
| 174 | test("where a hard link cannot be made the files are copied", async () => { | |
| 175 | const tips = await tipsOf(parent); | |
| 176 | const stored = (await packOnce(parent, tips))!; | |
| 177 | const to = await fresh("copied"); | |
| 178 | const refuse = (async () => { | |
| 179 | throw Object.assign(new Error("cross-device"), { code: "EXDEV" }); | |
| 180 | }) as unknown as typeof import("node:fs/promises").link; | |
| 181 | expect(await placeInto(stored, join(to, "objects", "pack"), refuse)).toBe(true); | |
| 182 | const names = (await readdir(join(to, "objects", "pack"))).filter((n) => n.startsWith("pack-")); | |
| 183 | expect(names.some((n) => n.endsWith(".pack"))).toBe(true); | |
| 184 | const n = names.find((x) => x.endsWith(".pack"))!; | |
| 185 | expect((await stat(join(to, "objects", "pack", n))).ino).not.toBe((await stat(join(stored, n))).ino); | |
| 186 | }); | |
| 187 | ||
| 188 | test("a deleted repository's store goes with it", async () => { | |
| 189 | const repo = join(dir, "owners", "o", "gone.git"); | |
| 190 | await mkdir(repo, { recursive: true }); | |
| 191 | const store = seedStoreOf(repo); | |
| 192 | await mkdir(join(store, "k"), { recursive: true }); | |
| 193 | await Bun.write(join(repo, "HEAD"), "ref: refs/heads/main\n"); | |
| 194 | expect((await deleteRepoDir(repo)).ok).toBe(true); | |
| 195 | expect(await stat(store).catch(() => undefined)).toBeUndefined(); | |
| 196 | await mkdir(join(store, "k"), { recursive: true }); | |
| 197 | await dropSeedStore(repo); | |
| 198 | expect(await stat(store).catch(() => undefined)).toBeUndefined(); | |
| 199 | }); | |
| 200 | ||
| 103 | 201 | test("an address that is not a repository does nothing", async () => { |
| 104 | 202 | expect(await seedHistory(join(dir, "nowhere"), await fresh("orphan"))).toBe(0); |
| 105 | 203 | expect(await seedHistory(parent, join(dir, "nowhere"))).toBe(0); |
new above
offline
Nothing removes this store when the parent repo is deleted; deleted repo history persists. Clean it on repo destroy.