Public: anyone can read this; the crew can change it.

A held fork reader cannot monopolize the git gate or borrow lender objects through a thin pack #14975

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14955-a-held-fork-reader-cannot-monopolize-the → main 2 commits fork

  • Merged. scout/14955-a-held-fork-reader-cannot-monopolize-the was merged into main as 52f0dc3.

Review

Sign in to review this change.
3 files changed +322 −17
src/git/own_refs.ts +167 −16
@@ -31,7 +31,9 @@
3131 */
3232
3333 import { AsyncLocalStorage } from "node:async_hooks";
34 import { gitEnv, gitGate } from "./env.ts";
34 import { closeSync, openSync, readdirSync, readSync } from "node:fs";
35 import { join } from "node:path";
36 import { Gate, gitEnv, gitGate } from "./env.ts";
3537
3638 /** What a held reader is told at the git door, and what to do. */
3739 export const OWN_REFS_ONLY =
@@ -307,6 +309,45 @@ function knownReached(dir: string, oid: string, now: number): boolean {
307309 return until !== undefined && until > now;
308310 }
309311
312 /**
313 * Ids a walk did not meet, kept in `reached` beside the ids it did, under a
314 * key that names the state of the dir's refs: a refusal is asked again only
315 * once a ref has moved, so a stranger repeating one id costs one walk, not one
316 * a request. A refusal never expires by time; the size cap clears it.
317 */
318 function refusedKey(refsKey: string, oid: string): string {
319 return `!${refsKey}:${oid}`;
320 }
321
322 function knownRefused(dir: string, refsKey: string, oid: string): boolean {
323 return reached.get(dir)?.has(refusedKey(refsKey, oid)) === true;
324 }
325
326 function keepRefused(dir: string, refsKey: string, oids: readonly string[]): void {
327 keepReached(
328 dir,
329 oids.map((oid) => refusedKey(refsKey, oid)),
330 Number.POSITIVE_INFINITY,
331 );
332 }
333
334 /**
335 * How far one walk for a held reader goes before it answers "not reachable":
336 * objects met and milliseconds spent. A walk costs about 20 µs an object, so
337 * the bound is a few seconds; a fork larger than it serves its trees and
338 * files by id to a held reader no more, and its branches and tags as ever.
339 */
340 export const WALK_BOUND = { objects: 400_000, ms: 8_000 };
341
342 /**
343 * The walks' own gate, apart from `gitGate`: two at once, so walks a stranger
344 * starts wait on each other and never hold the permits every page needs.
345 */
346 let walkGate: Gate | undefined;
347
348 /** How many walks this process ran: what a test reads to see one was not repeated. */
349 export const walkCount = { runs: 0 };
350
310351 function keepReached(dir: string, oids: Iterable<string>, now: number): void {
311352 let kept = reached.get(dir);
312353 if (kept === undefined) {
@@ -341,6 +382,8 @@ export async function reachableFromOwnRefs(
341382 ]);
342383 if (tips.code !== 0) return false;
343384 const atTip = new Set(tips.out.split(/\s+/).filter((t) => t !== ""));
385 const refsKey = String(Bun.hash(tips.out));
386 if (left.some((oid) => knownRefused(dir, refsKey, oid))) return false;
344387 keepReached(
345388 dir,
346389 left.filter((oid) => atTip.has(oid)),
@@ -365,7 +408,10 @@ export async function reachableFromOwnRefs(
365408 keepReached(dir, commits, now);
366409 }
367410 if (others.length > 0) {
368 if (!(await walkHolds(dir, others))) return false;
411 if (!(await walkHolds(dir, others))) {
412 keepRefused(dir, refsKey, others);
413 return false;
414 }
369415 keepReached(dir, others, now);
370416 }
371417 return true;
@@ -373,13 +419,18 @@ export async function reachableFromOwnRefs(
373419
374420 /**
375421 * Whether a walk of everything `dir`'s refs reach meets every one of `oids`;
376 * it stops at the last one found.
422 * it stops at the last one found, and answers no past `WALK_BOUND`.
377423 */
378424 async function walkHolds(
379425 dir: string,
380426 oids: readonly string[],
381427 ): Promise<boolean> {
382 return gitGate.run(async () => {
428 // Made at first use: `env.ts` imports this module, so `Gate` is not yet there as it loads.
429 walkGate ??= new Gate(2);
430 return walkGate.run(async () => {
431 walkCount.runs++;
432 const until = Date.now() + WALK_BOUND.ms;
433 let met = 0;
383434 const want = new Set(oids);
384435 const proc = Bun.spawn(
385436 ["git", "rev-list", "--objects", "--no-object-names", "--all"],
@@ -391,7 +442,7 @@ async function walkHolds(
391442 stderr: "ignore",
392443 },
393444 );
394 const timer = killAfter(proc, 120_000);
445 const timer = killAfter(proc, WALK_BOUND.ms);
395446 const decoder = new TextDecoder();
396447 let carry = "";
397448 try {
@@ -400,10 +451,14 @@ async function walkHolds(
400451 "\n",
401452 );
402453 carry = lines.pop() ?? "";
403 for (const line of lines)
454 for (const line of lines) {
404455 if (want.delete(line) && want.size === 0) return true;
456 if (++met >= WALK_BOUND.objects) return false;
457 }
458 if (Date.now() >= until) return false;
405459 }
406 return want.size === 0;
460 const code = await proc.exited;
461 return code === 0 && want.size === 0;
407462 } finally {
408463 clearTimeout(timer);
409464 proc.kill("SIGKILL");
@@ -576,6 +631,21 @@ export async function pushCarriesWhatItNames(
576631 }
577632 });
578633 if (listed === undefined) return false;
634 if (quarantine !== undefined) {
635 // A thin pack's delta against a base it does not carry is completed by
636 // git from the whole store, the lender's included, and the base is then
637 // written into the quarantine as if sent; the delta's own object is that
638 // base's content too. So a base the lender holds must be one the fork's
639 // refs already reach, as every base an honest client chose from what the
640 // fork advertised is.
641 const bases = await refDeltaBases(hold.dir, quarantine);
642 if (bases === undefined) return false;
643 if (bases.length > 0) {
644 const borrowed = [...(await typesIn(hold.lenderDir, bases)).keys()];
645 if (borrowed.length > 0 && !(await reachableFromOwnRefs(hold.dir, borrowed)))
646 return false;
647 }
648 }
579649 if (listed.length === 0) return true;
580650 const lent = [...(await typesIn(hold.lenderDir, listed)).keys()];
581651 if (lent.length === 0) return true;
@@ -589,14 +659,95 @@ export async function pushCarriesWhatItNames(
589659 stdout: "pipe",
590660 stderr: "ignore",
591661 });
592 const out = await new Response(proc.stdout).text();
593 await proc.exited;
594 return new Set(
595 out
596 .split("\n")
597 .filter((line) => !line.endsWith(" missing"))
598 .map((line) => line.split(" ")[0]!),
599 );
662 const timer = killAfter(proc, 30_000);
663 try {
664 const [out, code] = await Promise.all([
665 new Response(proc.stdout).text(),
666 proc.exited,
667 ]);
668 if (code !== 0) return undefined;
669 return new Set(
670 out
671 .split("\n")
672 .filter((line) => !line.endsWith(" missing"))
673 .map((line) => line.split(" ")[0]!),
674 );
675 } finally {
676 clearTimeout(timer);
677 }
600678 });
601 return lent.every((oid) => sent.has(oid));
679 return sent !== undefined && lent.every((oid) => sent.has(oid));
680 }
681
682 /** A pack entry's type that names its base by id: the only kind a thin pack leaves out. */
683 const REF_DELTA = 7;
684
685 /**
686 * The base ids of every by-id delta in the packs of a push's quarantine,
687 * read from each entry's header at the offset its index gives: no object is
688 * inflated. Undefined when a pack cannot be read.
689 */
690 async function refDeltaBases(
691 dir: string,
692 quarantine: string,
693 ): Promise<string[] | undefined> {
694 const packDir = join(quarantine, "pack");
695 let names: string[];
696 try {
697 names = readdirSync(packDir).filter((name) => name.endsWith(".idx"));
698 } catch {
699 return [];
700 }
701 const bases = new Set<string>();
702 for (const name of names) {
703 const idx = join(packDir, name);
704 const listed = await gitGate.run(async () => {
705 const proc = Bun.spawn(["git", "show-index"], {
706 cwd: dir,
707 env: gitEnv(),
708 stdin: Bun.file(idx),
709 stdout: "pipe",
710 stderr: "ignore",
711 });
712 const timer = killAfter(proc, 30_000);
713 try {
714 const [out, code] = await Promise.all([
715 new Response(proc.stdout).text(),
716 proc.exited,
717 ]);
718 return code === 0 ? out : undefined;
719 } finally {
720 clearTimeout(timer);
721 }
722 });
723 if (listed === undefined) return undefined;
724 const entries: { offset: number; hashBytes: number }[] = [];
725 for (const line of listed.split("\n")) {
726 const [offset, oid] = line.split(" ");
727 if (offset === undefined || oid === undefined || !FULL.test(oid)) continue;
728 entries.push({ offset: Number(offset), hashBytes: oid.length / 2 });
729 }
730 let fd: number;
731 try {
732 fd = openSync(join(packDir, `${name.slice(0, -4)}.pack`), "r");
733 } catch {
734 return undefined;
735 }
736 try {
737 const head = new Uint8Array(16 + 32);
738 for (const { offset, hashBytes } of entries) {
739 const read = readSync(fd, head, 0, head.length, offset);
740 let at = 0;
741 let byte = head[at++]!;
742 const type = (byte >> 4) & 7;
743 while (byte & 0x80 && at < 16) byte = head[at++]!;
744 if (type !== REF_DELTA) continue;
745 if (byte & 0x80 || read < at + hashBytes) return undefined;
746 bases.add(Buffer.from(head.subarray(at, at + hashBytes)).toString("hex"));
747 }
748 } finally {
749 closeSync(fd);
750 }
751 }
752 return [...bases];
602753 }
test/held-fork-bounds.test.ts added +154
@@ -0,0 +1,154 @@
1 /**
2 * A reader held to a fork's own refs (`src/git/own_refs.ts`, #14955): the walk
3 * that answers for a tree or a file by id is bounded and its refusal kept, and
4 * a push cannot pin the lender's objects by letting a thin pack's base
5 * completion copy them into its quarantine.
6 */
7
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
9 import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { gitClientEnv } from "./helpers/git-client.ts";
13 import {
14 pushCarriesWhatItNames,
15 reachableFromOwnRefs,
16 WALK_BOUND,
17 walkCount,
18 } from "../src/git/own_refs.ts";
19
20 let root: string;
21 let lender: string;
22 let fork: string;
23 let work: string;
24 let shared: { commit: string; blob: string };
25 let secret: { commit: string; blob: string };
26
27 async function git(cwd: string, args: string[], input?: Uint8Array | string, env: Record<string, string> = {}) {
28 const proc = Bun.spawn(["git", ...args], {
29 cwd,
30 env: { ...gitClientEnv(), ...env },
31 stdin: input === undefined ? "ignore" : new Blob([input]),
32 stdout: "pipe",
33 stderr: "pipe",
34 });
35 const [out, err, code] = await Promise.all([
36 new Response(proc.stdout).arrayBuffer(),
37 new Response(proc.stderr).text(),
38 proc.exited,
39 ]);
40 if (code !== 0) throw new Error(`git ${args.join(" ")}: ${err}`);
41 return new Uint8Array(out);
42 }
43 const text = async (cwd: string, args: string[], input?: string) =>
44 new TextDecoder().decode(await git(cwd, args, input)).trim();
45
46 /** A file long enough that git deltas a small change of it. */
47 const lines = (word: string) => Array.from({ length: 200 }, (_, i) => `${word} line ${i}\n`).join("");
48
49 async function commitFile(content: string, message: string): Promise<{ commit: string; blob: string }> {
50 await writeFile(join(work, "f.txt"), content);
51 await git(work, ["add", "f.txt"]);
52 await git(work, ["commit", "-qm", message]);
53 return { commit: await text(work, ["rev-parse", "HEAD"]), blob: await text(work, ["rev-parse", "HEAD:f.txt"]) };
54 }
55
56 beforeAll(async () => {
57 root = await mkdtemp(join(tmpdir(), "held-bounds-"));
58 lender = join(root, "lender.git");
59 fork = join(root, "fork.git");
60 work = join(root, "work");
61 await git(root, ["init", "-q", "--bare", lender]);
62 await git(root, ["init", "-q", "-b", "main", work]);
63 shared = await commitFile(lines("shared"), "shared");
64 await git(work, ["push", "-q", lender, "HEAD:refs/heads/main"]);
65 // The parent's later work, which the fork's refs never reach.
66 secret = await commitFile(lines("secret"), "secret");
67 await git(work, ["push", "-q", lender, "HEAD:refs/heads/private"]);
68 fork = await makeFork("fork.git");
69 });
70
71 /** A fork of the lender whose only ref is the shared commit. */
72 async function makeFork(name: string): Promise<string> {
73 const dir = join(root, name);
74 await git(root, ["init", "-q", "--bare", dir]);
75 await writeFile(join(dir, "objects/info/alternates"), `${join(lender, "objects")}\n`);
76 await git(dir, ["update-ref", "refs/heads/main", shared.commit]);
77 return dir;
78 }
79
80 afterAll(async () => {
81 await rm(root, { recursive: true, force: true });
82 });
83
84 describe("a held reader's walk", () => {
85 test("goes no further than its bound and answers no past it", async () => {
86 const was = WALK_BOUND.objects;
87 try {
88 // The shared file is reachable; a bound of one object is met before it.
89 WALK_BOUND.objects = 1;
90 expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(false);
91 } finally {
92 WALK_BOUND.objects = was;
93 }
94 // The refusal stands for these refs; once one moves the file is walked to again.
95 expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(false);
96 await git(fork, ["update-ref", "refs/heads/again", shared.commit]);
97 expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(true);
98 });
99
100 test("a refused id is not walked again until a ref moves", async () => {
101 const before = walkCount.runs;
102 expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(false);
103 expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(false);
104 expect(walkCount.runs - before).toBe(1);
105 await git(fork, ["update-ref", "refs/heads/later", secret.commit]);
106 expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(true);
107 // The pushes below start from a fork that never had the secret.
108 fork = await makeFork("pushed.git");
109 });
110 });
111
112 /** A push's quarantine as receive-pack makes it, the pack indexed with `--fix-thin`. */
113 async function quarantined(pack: Uint8Array): Promise<Record<string, string>> {
114 const quarantine = await mkdtemp(join(fork, "objects", "incoming-"));
115 await mkdir(join(quarantine, "pack"));
116 const env = {
117 GIT_OBJECT_DIRECTORY: quarantine,
118 GIT_ALTERNATE_OBJECT_DIRECTORIES: join(fork, "objects"),
119 GIT_QUARANTINE_PATH: quarantine,
120 };
121 await git(fork, ["index-pack", "--stdin", "--fix-thin"], pack, env);
122 return env;
123 }
124
125 /** A thin pack of `tip` against `base`: its file a delta of the base's, left out. */
126 const thinPack = (tip: string, base: string) =>
127 git(work, ["pack-objects", "--thin", "--stdout", "--revs", "-q"], `${tip}\n^${base}\n`);
128
129 describe("a held push", () => {
130 const hold = () => ({ dir: fork, lenderDir: lender });
131
132 test("cannot pin a lender's file by sending only a delta against it", async () => {
133 // One commit on the secret one, so its pack's file is a delta whose base is the secret file.
134 await git(work, ["checkout", "-q", secret.commit]);
135 const pinned = await commitFile(`${lines("secret")}one more\n`, "pin");
136 const env = await quarantined(await thinPack(pinned.commit, secret.commit));
137 // The base completion copied the secret file into the quarantine.
138 const inQuarantine = await git(fork, ["cat-file", "-t", secret.blob], undefined, {
139 GIT_OBJECT_DIRECTORY: env.GIT_OBJECT_DIRECTORY!,
140 });
141 expect(new TextDecoder().decode(inQuarantine).trim()).toBe("blob");
142 expect(await pushCarriesWhatItNames(hold(), [secret.blob], env)).toBe(false);
143 // Nor a tip whose file is the secret one's content by delta.
144 expect(await pushCarriesWhatItNames(hold(), [pinned.commit], env)).toBe(false);
145 });
146
147 test("an honest thin push, its delta against what the fork's refs reach, lands", async () => {
148 await git(work, ["checkout", "-q", shared.commit]);
149 const next = await commitFile(`${lines("shared")}one more\n`, "next");
150 const pack = await thinPack(next.commit, shared.commit);
151 const env = await quarantined(pack);
152 expect(await pushCarriesWhatItNames(hold(), [next.commit], env)).toBe(true);
153 });
154 });
test/spawn-pipes.test.ts +1 −1
@@ -34,7 +34,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
3434 "src/git/env.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end, finally clears the timer" },
3535 "src/git/combined.ts": { sites: ["stdout"], reason: "spawnSync buffers rev-parse stdout; stderr ignored" },
3636 "src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" },
37 "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (three as text alongside exited, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" },
37 "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (four as text alongside exited, cat-file and show-index under a kill timer, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" },
3838 "src/git/read.ts": { sites: ["stdout", "stdout", "stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed), by blobRange until its range is in (then killed), and by blobTail until the blob ends, keeping a ring of its last bytes; stderr ignored" },
3939 "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" },
4040 "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs