Public: anyone can read this; the crew can change it.
A held fork reader cannot monopolize the git gate or borrow lender objects through a thin pack #14975
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14955-a-held-fork-reader-cannot-monopolize-the → main 2 commits fork
3 files
changed +322
−17
src/git/own_refs.ts +167 −16
| @@ -31,7 +31,9 @@ | ||
| 31 | 31 | */ |
| 32 | 32 | |
| 33 | 33 | import { AsyncLocalStorage } from "node:async_hooks"; |
| 34 | import { gitEnv, gitGate } from "./env.ts"; | |
| 34 | import { closeSync, openSync, readdirSync, readSync } from "node:fs"; | |
| 35 | import { join } from "node:path"; | |
| 36 | import { Gate, gitEnv, gitGate } from "./env.ts"; | |
| 35 | 37 | |
| 36 | 38 | /** What a held reader is told at the git door, and what to do. */ |
| 37 | 39 | export const OWN_REFS_ONLY = |
| @@ -307,6 +309,45 @@ function knownReached(dir: string, oid: string, now: number): boolean { | ||
| 307 | 309 | return until !== undefined && until > now; |
| 308 | 310 | } |
| 309 | 311 | |
| 312 | /** | |
| 313 | * Ids a walk did not meet, kept in `reached` beside the ids it did, under a | |
| 314 | * key that names the state of the dir's refs: a refusal is asked again only | |
| 315 | * once a ref has moved, so a stranger repeating one id costs one walk, not one | |
| 316 | * a request. A refusal never expires by time; the size cap clears it. | |
| 317 | */ | |
| 318 | function refusedKey(refsKey: string, oid: string): string { | |
| 319 | return `!${refsKey}:${oid}`; | |
| 320 | } | |
| 321 | ||
| 322 | function knownRefused(dir: string, refsKey: string, oid: string): boolean { | |
| 323 | return reached.get(dir)?.has(refusedKey(refsKey, oid)) === true; | |
| 324 | } | |
| 325 | ||
| 326 | function keepRefused(dir: string, refsKey: string, oids: readonly string[]): void { | |
| 327 | keepReached( | |
| 328 | dir, | |
| 329 | oids.map((oid) => refusedKey(refsKey, oid)), | |
| 330 | Number.POSITIVE_INFINITY, | |
| 331 | ); | |
| 332 | } | |
| 333 | ||
| 334 | /** | |
| 335 | * How far one walk for a held reader goes before it answers "not reachable": | |
| 336 | * objects met and milliseconds spent. A walk costs about 20 µs an object, so | |
| 337 | * the bound is a few seconds; a fork larger than it serves its trees and | |
| 338 | * files by id to a held reader no more, and its branches and tags as ever. | |
| 339 | */ | |
| 340 | export const WALK_BOUND = { objects: 400_000, ms: 8_000 }; | |
| 341 | ||
| 342 | /** | |
| 343 | * The walks' own gate, apart from `gitGate`: two at once, so walks a stranger | |
| 344 | * starts wait on each other and never hold the permits every page needs. | |
| 345 | */ | |
| 346 | let walkGate: Gate | undefined; | |
| 347 | ||
| 348 | /** How many walks this process ran: what a test reads to see one was not repeated. */ | |
| 349 | export const walkCount = { runs: 0 }; | |
| 350 | ||
| 310 | 351 | function keepReached(dir: string, oids: Iterable<string>, now: number): void { |
| 311 | 352 | let kept = reached.get(dir); |
| 312 | 353 | if (kept === undefined) { |
| @@ -341,6 +382,8 @@ export async function reachableFromOwnRefs( | ||
| 341 | 382 | ]); |
| 342 | 383 | if (tips.code !== 0) return false; |
| 343 | 384 | const atTip = new Set(tips.out.split(/\s+/).filter((t) => t !== "")); |
| 385 | const refsKey = String(Bun.hash(tips.out)); | |
| 386 | if (left.some((oid) => knownRefused(dir, refsKey, oid))) return false; | |
| 344 | 387 | keepReached( |
| 345 | 388 | dir, |
| 346 | 389 | left.filter((oid) => atTip.has(oid)), |
| @@ -365,7 +408,10 @@ export async function reachableFromOwnRefs( | ||
| 365 | 408 | keepReached(dir, commits, now); |
| 366 | 409 | } |
| 367 | 410 | if (others.length > 0) { |
| 368 | if (!(await walkHolds(dir, others))) return false; | |
| 411 | if (!(await walkHolds(dir, others))) { | |
| 412 | keepRefused(dir, refsKey, others); | |
| 413 | return false; | |
| 414 | } | |
| 369 | 415 | keepReached(dir, others, now); |
| 370 | 416 | } |
| 371 | 417 | return true; |
| @@ -373,13 +419,18 @@ export async function reachableFromOwnRefs( | ||
| 373 | 419 | |
| 374 | 420 | /** |
| 375 | 421 | * Whether a walk of everything `dir`'s refs reach meets every one of `oids`; |
| 376 | * it stops at the last one found. | |
| 422 | * it stops at the last one found, and answers no past `WALK_BOUND`. | |
| 377 | 423 | */ |
| 378 | 424 | async function walkHolds( |
| 379 | 425 | dir: string, |
| 380 | 426 | oids: readonly string[], |
| 381 | 427 | ): Promise<boolean> { |
| 382 | return gitGate.run(async () => { | |
| 428 | // Made at first use: `env.ts` imports this module, so `Gate` is not yet there as it loads. | |
| 429 | walkGate ??= new Gate(2); | |
| 430 | return walkGate.run(async () => { | |
| 431 | walkCount.runs++; | |
| 432 | const until = Date.now() + WALK_BOUND.ms; | |
| 433 | let met = 0; | |
| 383 | 434 | const want = new Set(oids); |
| 384 | 435 | const proc = Bun.spawn( |
| 385 | 436 | ["git", "rev-list", "--objects", "--no-object-names", "--all"], |
| @@ -391,7 +442,7 @@ async function walkHolds( | ||
| 391 | 442 | stderr: "ignore", |
| 392 | 443 | }, |
| 393 | 444 | ); |
| 394 | const timer = killAfter(proc, 120_000); | |
| 445 | const timer = killAfter(proc, WALK_BOUND.ms); | |
| 395 | 446 | const decoder = new TextDecoder(); |
| 396 | 447 | let carry = ""; |
| 397 | 448 | try { |
| @@ -400,10 +451,14 @@ async function walkHolds( | ||
| 400 | 451 | "\n", |
| 401 | 452 | ); |
| 402 | 453 | carry = lines.pop() ?? ""; |
| 403 | for (const line of lines) | |
| 454 | for (const line of lines) { | |
| 404 | 455 | if (want.delete(line) && want.size === 0) return true; |
| 456 | if (++met >= WALK_BOUND.objects) return false; | |
| 457 | } | |
| 458 | if (Date.now() >= until) return false; | |
| 405 | 459 | } |
| 406 | return want.size === 0; | |
| 460 | const code = await proc.exited; | |
| 461 | return code === 0 && want.size === 0; | |
| 407 | 462 | } finally { |
| 408 | 463 | clearTimeout(timer); |
| 409 | 464 | proc.kill("SIGKILL"); |
| @@ -576,6 +631,21 @@ export async function pushCarriesWhatItNames( | ||
| 576 | 631 | } |
| 577 | 632 | }); |
| 578 | 633 | if (listed === undefined) return false; |
| 634 | if (quarantine !== undefined) { | |
| 635 | // A thin pack's delta against a base it does not carry is completed by | |
| 636 | // git from the whole store, the lender's included, and the base is then | |
| 637 | // written into the quarantine as if sent; the delta's own object is that | |
| 638 | // base's content too. So a base the lender holds must be one the fork's | |
| 639 | // refs already reach, as every base an honest client chose from what the | |
| 640 | // fork advertised is. | |
| 641 | const bases = await refDeltaBases(hold.dir, quarantine); | |
| 642 | if (bases === undefined) return false; | |
| 643 | if (bases.length > 0) { | |
| 644 | const borrowed = [...(await typesIn(hold.lenderDir, bases)).keys()]; | |
| 645 | if (borrowed.length > 0 && !(await reachableFromOwnRefs(hold.dir, borrowed))) | |
| 646 | return false; | |
| 647 | } | |
| 648 | } | |
| 579 | 649 | if (listed.length === 0) return true; |
| 580 | 650 | const lent = [...(await typesIn(hold.lenderDir, listed)).keys()]; |
| 581 | 651 | if (lent.length === 0) return true; |
| @@ -589,14 +659,95 @@ export async function pushCarriesWhatItNames( | ||
| 589 | 659 | stdout: "pipe", |
| 590 | 660 | stderr: "ignore", |
| 591 | 661 | }); |
| 592 | const out = await new Response(proc.stdout).text(); | |
| 593 | await proc.exited; | |
| 594 | return new Set( | |
| 595 | out | |
| 596 | .split("\n") | |
| 597 | .filter((line) => !line.endsWith(" missing")) | |
| 598 | .map((line) => line.split(" ")[0]!), | |
| 599 | ); | |
| 662 | const timer = killAfter(proc, 30_000); | |
| 663 | try { | |
| 664 | const [out, code] = await Promise.all([ | |
| 665 | new Response(proc.stdout).text(), | |
| 666 | proc.exited, | |
| 667 | ]); | |
| 668 | if (code !== 0) return undefined; | |
| 669 | return new Set( | |
| 670 | out | |
| 671 | .split("\n") | |
| 672 | .filter((line) => !line.endsWith(" missing")) | |
| 673 | .map((line) => line.split(" ")[0]!), | |
| 674 | ); | |
| 675 | } finally { | |
| 676 | clearTimeout(timer); | |
| 677 | } | |
| 600 | 678 | }); |
| 601 | return lent.every((oid) => sent.has(oid)); | |
| 679 | return sent !== undefined && lent.every((oid) => sent.has(oid)); | |
| 680 | } | |
| 681 | ||
| 682 | /** A pack entry's type that names its base by id: the only kind a thin pack leaves out. */ | |
| 683 | const REF_DELTA = 7; | |
| 684 | ||
| 685 | /** | |
| 686 | * The base ids of every by-id delta in the packs of a push's quarantine, | |
| 687 | * read from each entry's header at the offset its index gives: no object is | |
| 688 | * inflated. Undefined when a pack cannot be read. | |
| 689 | */ | |
| 690 | async function refDeltaBases( | |
| 691 | dir: string, | |
| 692 | quarantine: string, | |
| 693 | ): Promise<string[] | undefined> { | |
| 694 | const packDir = join(quarantine, "pack"); | |
| 695 | let names: string[]; | |
| 696 | try { | |
| 697 | names = readdirSync(packDir).filter((name) => name.endsWith(".idx")); | |
| 698 | } catch { | |
| 699 | return []; | |
| 700 | } | |
| 701 | const bases = new Set<string>(); | |
| 702 | for (const name of names) { | |
| 703 | const idx = join(packDir, name); | |
| 704 | const listed = await gitGate.run(async () => { | |
| 705 | const proc = Bun.spawn(["git", "show-index"], { | |
| 706 | cwd: dir, | |
| 707 | env: gitEnv(), | |
| 708 | stdin: Bun.file(idx), | |
| 709 | stdout: "pipe", | |
| 710 | stderr: "ignore", | |
| 711 | }); | |
| 712 | const timer = killAfter(proc, 30_000); | |
| 713 | try { | |
| 714 | const [out, code] = await Promise.all([ | |
| 715 | new Response(proc.stdout).text(), | |
| 716 | proc.exited, | |
| 717 | ]); | |
| 718 | return code === 0 ? out : undefined; | |
| 719 | } finally { | |
| 720 | clearTimeout(timer); | |
| 721 | } | |
| 722 | }); | |
| 723 | if (listed === undefined) return undefined; | |
| 724 | const entries: { offset: number; hashBytes: number }[] = []; | |
| 725 | for (const line of listed.split("\n")) { | |
| 726 | const [offset, oid] = line.split(" "); | |
| 727 | if (offset === undefined || oid === undefined || !FULL.test(oid)) continue; | |
| 728 | entries.push({ offset: Number(offset), hashBytes: oid.length / 2 }); | |
| 729 | } | |
| 730 | let fd: number; | |
| 731 | try { | |
| 732 | fd = openSync(join(packDir, `${name.slice(0, -4)}.pack`), "r"); | |
| 733 | } catch { | |
| 734 | return undefined; | |
| 735 | } | |
| 736 | try { | |
| 737 | const head = new Uint8Array(16 + 32); | |
| 738 | for (const { offset, hashBytes } of entries) { | |
| 739 | const read = readSync(fd, head, 0, head.length, offset); | |
| 740 | let at = 0; | |
| 741 | let byte = head[at++]!; | |
| 742 | const type = (byte >> 4) & 7; | |
| 743 | while (byte & 0x80 && at < 16) byte = head[at++]!; | |
| 744 | if (type !== REF_DELTA) continue; | |
| 745 | if (byte & 0x80 || read < at + hashBytes) return undefined; | |
| 746 | bases.add(Buffer.from(head.subarray(at, at + hashBytes)).toString("hex")); | |
| 747 | } | |
| 748 | } finally { | |
| 749 | closeSync(fd); | |
| 750 | } | |
| 751 | } | |
| 752 | return [...bases]; | |
| 602 | 753 | } |
test/held-fork-bounds.test.ts added +154
| @@ -0,0 +1,154 @@ | ||
| 1 | /** | |
| 2 | * A reader held to a fork's own refs (`src/git/own_refs.ts`, #14955): the walk | |
| 3 | * that answers for a tree or a file by id is bounded and its refusal kept, and | |
| 4 | * a push cannot pin the lender's objects by letting a thin pack's base | |
| 5 | * completion copy them into its quarantine. | |
| 6 | */ | |
| 7 | ||
| 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 9 | import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { gitClientEnv } from "./helpers/git-client.ts"; | |
| 13 | import { | |
| 14 | pushCarriesWhatItNames, | |
| 15 | reachableFromOwnRefs, | |
| 16 | WALK_BOUND, | |
| 17 | walkCount, | |
| 18 | } from "../src/git/own_refs.ts"; | |
| 19 | ||
| 20 | let root: string; | |
| 21 | let lender: string; | |
| 22 | let fork: string; | |
| 23 | let work: string; | |
| 24 | let shared: { commit: string; blob: string }; | |
| 25 | let secret: { commit: string; blob: string }; | |
| 26 | ||
| 27 | async function git(cwd: string, args: string[], input?: Uint8Array | string, env: Record<string, string> = {}) { | |
| 28 | const proc = Bun.spawn(["git", ...args], { | |
| 29 | cwd, | |
| 30 | env: { ...gitClientEnv(), ...env }, | |
| 31 | stdin: input === undefined ? "ignore" : new Blob([input]), | |
| 32 | stdout: "pipe", | |
| 33 | stderr: "pipe", | |
| 34 | }); | |
| 35 | const [out, err, code] = await Promise.all([ | |
| 36 | new Response(proc.stdout).arrayBuffer(), | |
| 37 | new Response(proc.stderr).text(), | |
| 38 | proc.exited, | |
| 39 | ]); | |
| 40 | if (code !== 0) throw new Error(`git ${args.join(" ")}: ${err}`); | |
| 41 | return new Uint8Array(out); | |
| 42 | } | |
| 43 | const text = async (cwd: string, args: string[], input?: string) => | |
| 44 | new TextDecoder().decode(await git(cwd, args, input)).trim(); | |
| 45 | ||
| 46 | /** A file long enough that git deltas a small change of it. */ | |
| 47 | const lines = (word: string) => Array.from({ length: 200 }, (_, i) => `${word} line ${i}\n`).join(""); | |
| 48 | ||
| 49 | async function commitFile(content: string, message: string): Promise<{ commit: string; blob: string }> { | |
| 50 | await writeFile(join(work, "f.txt"), content); | |
| 51 | await git(work, ["add", "f.txt"]); | |
| 52 | await git(work, ["commit", "-qm", message]); | |
| 53 | return { commit: await text(work, ["rev-parse", "HEAD"]), blob: await text(work, ["rev-parse", "HEAD:f.txt"]) }; | |
| 54 | } | |
| 55 | ||
| 56 | beforeAll(async () => { | |
| 57 | root = await mkdtemp(join(tmpdir(), "held-bounds-")); | |
| 58 | lender = join(root, "lender.git"); | |
| 59 | fork = join(root, "fork.git"); | |
| 60 | work = join(root, "work"); | |
| 61 | await git(root, ["init", "-q", "--bare", lender]); | |
| 62 | await git(root, ["init", "-q", "-b", "main", work]); | |
| 63 | shared = await commitFile(lines("shared"), "shared"); | |
| 64 | await git(work, ["push", "-q", lender, "HEAD:refs/heads/main"]); | |
| 65 | // The parent's later work, which the fork's refs never reach. | |
| 66 | secret = await commitFile(lines("secret"), "secret"); | |
| 67 | await git(work, ["push", "-q", lender, "HEAD:refs/heads/private"]); | |
| 68 | fork = await makeFork("fork.git"); | |
| 69 | }); | |
| 70 | ||
| 71 | /** A fork of the lender whose only ref is the shared commit. */ | |
| 72 | async function makeFork(name: string): Promise<string> { | |
| 73 | const dir = join(root, name); | |
| 74 | await git(root, ["init", "-q", "--bare", dir]); | |
| 75 | await writeFile(join(dir, "objects/info/alternates"), `${join(lender, "objects")}\n`); | |
| 76 | await git(dir, ["update-ref", "refs/heads/main", shared.commit]); | |
| 77 | return dir; | |
| 78 | } | |
| 79 | ||
| 80 | afterAll(async () => { | |
| 81 | await rm(root, { recursive: true, force: true }); | |
| 82 | }); | |
| 83 | ||
| 84 | describe("a held reader's walk", () => { | |
| 85 | test("goes no further than its bound and answers no past it", async () => { | |
| 86 | const was = WALK_BOUND.objects; | |
| 87 | try { | |
| 88 | // The shared file is reachable; a bound of one object is met before it. | |
| 89 | WALK_BOUND.objects = 1; | |
| 90 | expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(false); | |
| 91 | } finally { | |
| 92 | WALK_BOUND.objects = was; | |
| 93 | } | |
| 94 | // The refusal stands for these refs; once one moves the file is walked to again. | |
| 95 | expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(false); | |
| 96 | await git(fork, ["update-ref", "refs/heads/again", shared.commit]); | |
| 97 | expect(await reachableFromOwnRefs(fork, [shared.blob])).toBe(true); | |
| 98 | }); | |
| 99 | ||
| 100 | test("a refused id is not walked again until a ref moves", async () => { | |
| 101 | const before = walkCount.runs; | |
| 102 | expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(false); | |
| 103 | expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(false); | |
| 104 | expect(walkCount.runs - before).toBe(1); | |
| 105 | await git(fork, ["update-ref", "refs/heads/later", secret.commit]); | |
| 106 | expect(await reachableFromOwnRefs(fork, [secret.blob])).toBe(true); | |
| 107 | // The pushes below start from a fork that never had the secret. | |
| 108 | fork = await makeFork("pushed.git"); | |
| 109 | }); | |
| 110 | }); | |
| 111 | ||
| 112 | /** A push's quarantine as receive-pack makes it, the pack indexed with `--fix-thin`. */ | |
| 113 | async function quarantined(pack: Uint8Array): Promise<Record<string, string>> { | |
| 114 | const quarantine = await mkdtemp(join(fork, "objects", "incoming-")); | |
| 115 | await mkdir(join(quarantine, "pack")); | |
| 116 | const env = { | |
| 117 | GIT_OBJECT_DIRECTORY: quarantine, | |
| 118 | GIT_ALTERNATE_OBJECT_DIRECTORIES: join(fork, "objects"), | |
| 119 | GIT_QUARANTINE_PATH: quarantine, | |
| 120 | }; | |
| 121 | await git(fork, ["index-pack", "--stdin", "--fix-thin"], pack, env); | |
| 122 | return env; | |
| 123 | } | |
| 124 | ||
| 125 | /** A thin pack of `tip` against `base`: its file a delta of the base's, left out. */ | |
| 126 | const thinPack = (tip: string, base: string) => | |
| 127 | git(work, ["pack-objects", "--thin", "--stdout", "--revs", "-q"], `${tip}\n^${base}\n`); | |
| 128 | ||
| 129 | describe("a held push", () => { | |
| 130 | const hold = () => ({ dir: fork, lenderDir: lender }); | |
| 131 | ||
| 132 | test("cannot pin a lender's file by sending only a delta against it", async () => { | |
| 133 | // One commit on the secret one, so its pack's file is a delta whose base is the secret file. | |
| 134 | await git(work, ["checkout", "-q", secret.commit]); | |
| 135 | const pinned = await commitFile(`${lines("secret")}one more\n`, "pin"); | |
| 136 | const env = await quarantined(await thinPack(pinned.commit, secret.commit)); | |
| 137 | // The base completion copied the secret file into the quarantine. | |
| 138 | const inQuarantine = await git(fork, ["cat-file", "-t", secret.blob], undefined, { | |
| 139 | GIT_OBJECT_DIRECTORY: env.GIT_OBJECT_DIRECTORY!, | |
| 140 | }); | |
| 141 | expect(new TextDecoder().decode(inQuarantine).trim()).toBe("blob"); | |
| 142 | expect(await pushCarriesWhatItNames(hold(), [secret.blob], env)).toBe(false); | |
| 143 | // Nor a tip whose file is the secret one's content by delta. | |
| 144 | expect(await pushCarriesWhatItNames(hold(), [pinned.commit], env)).toBe(false); | |
| 145 | }); | |
| 146 | ||
| 147 | test("an honest thin push, its delta against what the fork's refs reach, lands", async () => { | |
| 148 | await git(work, ["checkout", "-q", shared.commit]); | |
| 149 | const next = await commitFile(`${lines("shared")}one more\n`, "next"); | |
| 150 | const pack = await thinPack(next.commit, shared.commit); | |
| 151 | const env = await quarantined(pack); | |
| 152 | expect(await pushCarriesWhatItNames(hold(), [next.commit], env)).toBe(true); | |
| 153 | }); | |
| 154 | }); |
test/spawn-pipes.test.ts +1 −1
| @@ -34,7 +34,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 34 | 34 | "src/git/env.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end, finally clears the timer" }, |
| 35 | 35 | "src/git/combined.ts": { sites: ["stdout"], reason: "spawnSync buffers rev-parse stdout; stderr ignored" }, |
| 36 | 36 | "src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" }, |
| 37 | "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (three as text alongside exited, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" }, | |
| 37 | "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (four as text alongside exited, cat-file and show-index under a kill timer, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" }, | |
| 38 | 38 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed), by blobRange until its range is in (then killed), and by blobTail until the blob ends, keeping a ring of its last bytes; stderr ignored" }, |
| 39 | 39 | "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" }, |
| 40 | 40 | "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" }, |
new above
offline