Public: anyone can read this; the crew can change it.

A held reader can learn whether an id exists in a fork's private lender: `have` ACKs and `shallow` errors #14870

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14850-a-held-reader-can-learn-whether-an-id-ex → main 8 commits fork 1 thread · 1 unresolved

  • Merged. scout/14850-a-held-reader-can-learn-whether-an-id-ex was merged into main as 57f7c33.

Review

Sign in to review this change.
5 files changed +311 −12
docs/features/held-reader-negotiation.md added +26
@@ -0,0 +1,26 @@
1 ---
2 topic: Agents: MCP, git, or the terminal
3 row: What a fork's held reader's fetch does with have and shallow
4 ---
5 # What a fork's held reader's fetch does with `have` and `shallow`
6
7 A fork shares its parent's object store. A reader of a fork who may not read the
8 parent (it became private) is held to what the fork's own refs reach
9 (`src/git/own_refs.ts`): the `want` lines of a fetch, over HTTP and over ssh.
10
11 The `have` and `shallow` lines are held the same way, because git would ACK a
12 `have` of an id the parent holds and die on a `shallow` that is a parent's tree
13 or file, and that tells the reader which ids the parent has.
14
15 - A `have` or `shallow` of an id the store holds but the fork's refs do not
16 reach is dropped before git sees it, so git answers as for an id nothing
17 holds: no ACK, no error. Dropping a `have` costs a bigger pack, nothing more.
18 - Ids the fork's refs reach, and ids the store does not hold, pass unchanged.
19 - Later rounds count too: over ssh every chunk the client sends passes the same
20 filter, in order, before git's input is closed.
21 - At most 64 held ids are checked per fetch; the rest are dropped.
22 - A malformed pkt-line from a held reader ends the fetch with the same sentence
23 as a refused `want` (HTTP: the refusal body; ssh: stderr and the channel's
24 git is killed).
25
26 Readers of the parent are never held and see none of this.
src/git/own_refs.ts +85
@@ -627,3 +627,88 @@ export async function pushCarriesWhatItNames(
627627 });
628628 return lent.every((oid) => sent.has(oid));
629629 }
630
631 // ---- what a held reader's negotiation may ask about ----
632
633 /** More held ids than this in one fetch's `have` and `shallow` lines are dropped unasked. */
634 const ASKED_MAX = 64;
635
636 /**
637 * The negotiation of a held reader, line by line, before git sees it.
638 *
639 * Git looks ids up through the lender's objects, so a `have` it holds is
640 * ACKed and a `shallow` that is a held tree or file makes it die: either one
641 * tells a reader who has an id whether the lender holds it. A `have` or a
642 * `shallow` whose id the store holds but the fork's own refs do not reach is
643 * dropped, so git answers it as it answers an id nothing holds. An id the
644 * store does not hold is passed on (git ignores it), and so is one in
645 * `allow`, which the reader was already given (a runner's job commit, whose
646 * shallow clone it deepens). Dropping a `have` only costs a bigger pack.
647 *
648 * `push` takes the bytes as they arrive, in any pieces, and gives back what
649 * git may be fed now; a half packet waits for the rest.
650 */
651 export function heldNegotiation(
652 dir: string,
653 allow: readonly string[] = [],
654 ): { push(bytes: Uint8Array): Promise<Uint8Array> } {
655 const allowed = new Set(allow);
656 const verdicts = new Map<string, boolean>();
657 let carry = new Uint8Array(0);
658 let asked = 0;
659 // One push at a time: a later piece must not overtake an earlier one.
660 let queue: Promise<unknown> = Promise.resolve();
661
662 const judge = async (ids: Set<string>): Promise<void> => {
663 const fresh = [...ids].filter((id) => !verdicts.has(id) && !allowed.has(id));
664 const held = await typesIn(dir, fresh.filter((id) => FULL.test(id)));
665 for (const id of fresh) {
666 if (!held.has(id)) verdicts.set(id, FULL.test(id));
667 else if (asked >= ASKED_MAX) verdicts.set(id, false);
668 else {
669 asked++;
670 verdicts.set(id, await reachableFromOwnRefs(dir, [id]));
671 }
672 }
673 };
674
675 const run = async (bytes: Uint8Array): Promise<Uint8Array> => {
676 const all = new Uint8Array(carry.length + bytes.length);
677 all.set(carry);
678 all.set(bytes, carry.length);
679 const packets: { raw: Uint8Array; id?: string }[] = [];
680 let at = 0;
681 for (;;) {
682 if (all.length < at + 4) break;
683 const head = String.fromCharCode(...all.subarray(at, at + 4));
684 if (!/^[0-9a-fA-F]{4}$/.test(head)) throw new Error("bad pkt-line");
685 const length = parseInt(head, 16);
686 if (length === 3 || length > 65_520) throw new Error("bad pkt-line");
687 const size = length < 4 ? 4 : length;
688 if (all.length < at + size) break;
689 const raw = all.subarray(at, at + size);
690 const text = size > 4 ? new TextDecoder("latin1").decode(raw.subarray(4)) : "";
691 const m = /^(?:have|shallow) ([^\s\0]*)/.exec(text);
692 packets.push(m === null ? { raw } : { raw, id: m[1]!.toLowerCase() });
693 at += size;
694 }
695 carry = all.slice(at);
696 await judge(new Set(packets.flatMap((p) => (p.id === undefined ? [] : [p.id]))));
697 const kept = packets.filter((p) => p.id === undefined || allowed.has(p.id) || verdicts.get(p.id) === true);
698 const out = new Uint8Array(kept.reduce((n, p) => n + p.raw.length, 0));
699 let w = 0;
700 for (const p of kept) {
701 out.set(p.raw, w);
702 w += p.raw.length;
703 }
704 return out;
705 };
706
707 return {
708 push(bytes) {
709 const next = queue.then(() => run(bytes));
710 queue = next.catch(() => undefined);
711 return next;
712 },
713 };
714 }
src/git/smart-http.ts +10 −1
@@ -21,7 +21,7 @@ import { pullPipe, type PullPipe } from "./pull-pipe.ts";
2121 import { transferDeadline, type Stall } from "./progress.ts";
2222 import { pacer } from "./pace.ts";
2323 import { bundleConfig } from "./bundle_uri.ts";
24 import { OWN_REFS_ONLY, reachableFromOwnRefs, refusalLine, wantsOf } from "./own_refs.ts";
24 import { OWN_REFS_ONLY, heldNegotiation, reachableFromOwnRefs, refusalLine, wantsOf } from "./own_refs.ts";
2525
2626 export type GitService = "git-upload-pack" | "git-receive-pack";
2727
@@ -179,6 +179,15 @@ export async function serviceRpc(
179179 },
180180 });
181181 }
182 // What git is told it has and what it is told is shallow is held too.
183 try {
184 read = await heldNegotiation(repoDir, only).push(read!);
185 } catch {
186 options.release?.();
187 return new Response(refusalLine(OWN_REFS_ONLY), {
188 headers: { "content-type": `application/x-${service}-result`, "cache-control": "no-cache" },
189 });
190 }
182191 }
183192 const { proc, stdout } = await spawnService(
184193 repoDir,
src/git/smart-ssh.ts +33 −11
@@ -14,7 +14,7 @@ import { gitEnv } from "./env.ts";
1414 import { pullPipeSync } from "./pull-pipe.ts";
1515 import { transferDeadline, type Stall } from "./progress.ts";
1616 import { serviceCommand, type GitService } from "./smart-http.ts";
17 import { OWN_REFS_ONLY, reachableFromOwnRefs, wantsOf } from "./own_refs.ts";
17 import { OWN_REFS_ONLY, heldNegotiation, reachableFromOwnRefs, wantsOf } from "./own_refs.ts";
1818
1919 export type { GitService };
2020
@@ -184,6 +184,13 @@ export function runGitService(
184184 kill,
185185 );
186186
187 // Set once the transfer is stopped and git killed: nothing is ended after that.
188 let halted = false;
189 const halt = (): void => {
190 halted = true;
191 stop();
192 kill();
193 };
187194 const toGit = (chunk: Uint8Array): void => {
188195 try {
189196 proc.stdin.write(chunk);
@@ -204,6 +211,25 @@ export function runGitService(
204211 let kept: Uint8Array[] | undefined = ownRefsOnly ? [] : undefined;
205212 let asking = false;
206213 let ended = false;
214 // Everything a held reader sends, `have` lines of later rounds included, passes
215 // through the scrub before git sees it (`heldNegotiation`).
216 const negotiation = ownRefsOnly ? heldNegotiation(dir) : undefined;
217 let fed: Promise<unknown> = Promise.resolve();
218 const feed = (chunk: Uint8Array): void => {
219 if (negotiation === undefined) return toGit(chunk);
220 const piece = chunk.slice();
221 fed = fed.then(() =>
222 negotiation.push(piece).then(toGit, () => {
223 channel.writeStderr(`forge: ${OWN_REFS_ONLY}\n`);
224 halt();
outdated

After a failed push the chain is already caught, so endInput still runs on a killed child. Guard with a flag set in the failure path.

225 }),
226 );
227 };
228 const finish = (): void => {
229 void fed.then(() => {
230 if (!halted) endInput();
231 });
232 };
207233 const ask = (): void => {
208234 if (kept === undefined || asking) return;
209235 let size = 0;
@@ -223,14 +249,13 @@ export function runGitService(
223249 .then((own) => {
224250 if (!own) {
225251 channel.writeStderr(`forge: ${OWN_REFS_ONLY}\n`);
226 stop();
227 kill();
252 halt();
228253 return;
229254 }
230255 const passed = kept ?? [];
231256 kept = undefined;
232 for (const chunk of passed) toGit(chunk);
233 if (ended) endInput();
257 for (const chunk of passed) feed(chunk);
258 if (ended) finish();
234259 });
235260 };
236261 channel.onData((chunk) => {
@@ -240,7 +265,7 @@ export function runGitService(
240265 ask();
241266 return;
242267 }
243 toGit(chunk);
268 feed(chunk);
244269 });
245270 // The peer's EOF means "I have sent everything", not "we are done": a
246271 // clone's whole pack arrives *after* the client half-closes. So this ends
@@ -249,16 +274,13 @@ export function runGitService(
249274 channel.onEof(() => {
250275 ended = true;
251276 if (kept !== undefined) ask();
252 else endInput();
277 else finish();
253278 });
254279 // A *close* is different: the peer is gone, so every byte git still has to
255280 // give has nowhere to go. Without this the child sits in `upload-pack`
256281 // writing into a dead channel until its deadline reaps it — one idle git
257282 // per disconnected clone.
258 channel.onClose(() => {
259 stop();
260 kill();
261 });
283 channel.onClose(halt);
262284
263285 const pump = async (
264286 stream: ReadableStream<Uint8Array>,
test/boundary-fork-objects.test.ts +157
@@ -42,6 +42,7 @@ import { createToken } from "../src/core/tokens.ts";
4242 import { createUser, type User } from "../src/core/users.ts";
4343 import { runGit } from "../src/git/env.ts";
4444 import {
45 heldNegotiation,
4546 idsNamed,
4647 OWN_REFS_ONLY,
4748 reachableFromOwnRefs,
@@ -353,6 +354,101 @@ describe("a public fork of a repository that is then made private", () => {
353354 expect(await res.text()).toContain(OWN_REFS_ONLY);
354355 }, 60_000);
355356
357 test("a have or a shallow of the parent's is answered as an id nothing holds: no ACK, no error", async () => {
358 const nothing = "1".repeat(40);
359 const post = async (lines: string[], tail: string[]): Promise<string> => {
360 const res = await fetch(`${servers.url}/bob/lib.git/git-upload-pack`, {
361 method: "POST",
362 headers: { "content-type": "application/x-git-upload-pack-request" },
363 body: `${lines.map(pkt).join("")}0000${tail.map(pkt).join("")}`,
364 });
365 return new TextDecoder("latin1").decode(await res.arrayBuffer());
366 };
367 const want = `want ${oid["later"]} multi_ack_detailed side-band-64k ofs-delta\n`;
368 // What git says before its pack, whose progress is chunked as the clock pleases.
369 const head = (t: string) => t.split(/[\u0001\u0002]/)[0]!.slice(0, -4);
370 const haven = async (id: string) =>
371 post([want], [`have ${id}\n`, "done\n"]);
372 // The fork's own commit is ACKed, as ever; the parent's is not, and reads as nothing.
373 expect(await haven(oid["shared"]!)).toContain(`ACK ${oid["shared"]} common`);
374 const asked = await haven(oid["private"]!);
375 expect(asked).not.toContain(`ACK ${oid["private"]}`);
376 expect(asked).not.toContain("ACK ");
377 expect(head(asked)).toBe(head(await haven(nothing)));
378 const shallowed = (id: string) =>
379 post([want, `shallow ${id}\n`, "deepen 1\n"], ["done\n"]);
380 const none = await shallowed(nothing);
381 expect(none).not.toContain("invalid shallow");
382 for (const id of [oid["private's tree"]!, oid["private's file"]!]) {
383 const got = await shallowed(id);
384 expect(got).not.toContain("invalid shallow");
385 expect(head(got)).toBe(head(none));
386 }
387 }, 60_000);
388
389 test("what is held out of a negotiation is the have and the shallow of an id the fork's refs do not reach, and nothing else", async () => {
390 const neg = heldNegotiation(dirOf(fork), [oid["private"]!]);
391 const body = [
392 `want ${oid["later"]} multi_ack\n`,
393 `shallow ${oid["private"]}\n`,
394 `shallow ${oid["private's tree"]}\n`,
395 `shallow ${oid["shared"]}\n`,
396 "deepen 1\n",
397 ]
398 .map(pkt)
399 .join("");
400 const rest = ["0000", pkt(`have ${oid["dropped"]}\n`), pkt(`have ${oid["shared"]}\n`), pkt("done\n")].join("");
401 const whole = new TextEncoder().encode(body + rest);
402 // Delivered in two awkward pieces: the half packet waits.
403 const a = await neg.push(whole.slice(0, 120));
404 const b = await neg.push(whole.slice(120));
405 const out = new TextDecoder().decode(new Uint8Array([...a, ...b]));
406 expect(out).toContain(`shallow ${oid["private"]}`); // the job's own, allowed
407 expect(out).toContain(`shallow ${oid["shared"]}`);
408 expect(out).not.toContain(oid["private's tree"]!);
409 expect(out).not.toContain(oid["dropped"]!);
410 expect(out).toContain(`have ${oid["shared"]}`);
411 expect(out).toContain("deepen 1");
412 expect(out).toContain("done");
413 }, 60_000);
414
415 test("a malformed pkt-line from a held reader over HTTP is refused with the sentence, and git is not run", async () => {
416 const res = await fetch(`${servers.url}/bob/lib.git/git-upload-pack`, {
417 method: "POST",
418 headers: { "content-type": "application/x-git-upload-pack-request" },
419 body: `${pkt(`want ${oid["later"]}\n`)}0000zzzz`,
420 });
421 const text = new TextDecoder("latin1").decode(await res.arrayBuffer());
422 expect([text.includes(OWN_REFS_ONLY), text.includes("PACK")]).toEqual([true, false]);
423 }, 60_000);
424
425 test("more held ids than the cap in one negotiation are dropped, and the rest of it still passes", async () => {
426 const blobs: string[] = [];
427 for (let i = 0; i < 70; i++) {
428 const w = Bun.spawn(["git", "hash-object", "-w", "--stdin"], {
429 cwd: dirOf(parent),
430 stdin: new Blob([`lender only ${i}\n`]),
431 stdout: "pipe",
432 stderr: "pipe",
433 });
434 blobs.push((await new Response(w.stdout).text()).trim());
435 await w.exited;
436 }
437 const neg = heldNegotiation(dirOf(fork));
438 const body =
439 pkt(`want ${oid["later"]}\n`) +
440 "0000" +
441 blobs.map((b) => pkt(`have ${b}\n`)).join("") +
442 pkt(`have ${oid["shared"]}\n`) +
443 pkt("done\n");
444 const out = new TextDecoder().decode(await neg.push(new TextEncoder().encode(body)));
445 for (const b of blobs) expect(out).not.toContain(b);
446 // The 71st held id is past the cap and dropped unasked; the lines that name no id pass.
447 expect(out).not.toContain(`have ${oid["shared"]}`);
448 expect(out).toContain(`want ${oid["later"]}`);
449 expect(out).toContain("done");
450 }, 120_000);
451
356452 test("a held reader is answered in protocol v0, and a reader of the parent in the v2 they asked for", async () => {
357453 const advert = async (token?: string): Promise<string> => {
358454 const res = await fetch(
@@ -577,6 +673,67 @@ describe("a public fork of a repository that is then made private", () => {
577673 ]).toEqual([false, false, true, 0]);
578674 }, 60_000);
579675
676 test("over ssh a later round's have of a lender-only id gets no ACK, and the input is ended after it was fed", async () => {
677 const drive = async (
678 chunks: string[],
679 ): Promise<{ out: string; err: string; code: number }> => {
680 const sent: Uint8Array[] = [];
681 let err = "";
682 let onData: (data: Uint8Array) => void = () => {};
683 let onEof: () => void = () => {};
684 const channel: GitChannel = {
685 write: (data) =>
686 void sent.push(typeof data === "string" ? new TextEncoder().encode(data) : data.slice()),
687 writeStderr: (data) =>
688 void (err += typeof data === "string" ? data : new TextDecoder().decode(data)),
689 onData: (cb) => void (onData = cb),
690 onEof: (cb) => void (onEof = cb),
691 onClose: () => {},
692 exit: () => {},
693 drain: () => Promise.resolve(),
694 };
695 const done = runGitService(dirOf(fork), "git-upload-pack", channel, { ownRefsOnly: true });
696 for (const c of chunks) {
697 onData(new TextEncoder().encode(c));
698 }
699 onEof();
700 const code = await done;
701 return { out: Buffer.concat(sent).toString("latin1"), err, code };
702 };
703 const wants = `${pkt(`want ${oid["later"]} multi_ack_detailed side-band-64k\n`)}0000`;
704 // Wants, then the haves and the end in later pieces, then EOF: git must be
705 // fed all of it before its input is closed, or "done" never reaches it.
706 const own = await drive([wants, pkt(`have ${oid["shared"]}\n`), pkt("done\n")]);
707 expect([own.out.includes(`ACK ${oid["shared"]}`), own.out.includes("PACK"), own.code]).toEqual([true, true, 0]);
708 const lent = await drive([wants, pkt(`have ${oid["private"]}\n`), pkt("done\n")]);
709 expect([lent.out.includes("ACK "), lent.out.includes("PACK"), lent.err, lent.code]).toEqual([false, true, "", 0]);
710 }, 60_000);
711
712 test("over ssh a malformed pkt-line from a held reader ends the fetch with the sentence on stderr", async () => {
713 const sent: Uint8Array[] = [];
714 let err = "";
715 let onData: (data: Uint8Array) => void = () => {};
716 let onEof: () => void = () => {};
717 const channel: GitChannel = {
718 write: (data) =>
719 void sent.push(typeof data === "string" ? new TextEncoder().encode(data) : data.slice()),
720 writeStderr: (data) =>
721 void (err += typeof data === "string" ? data : new TextDecoder().decode(data)),
722 onData: (cb) => void (onData = cb),
723 onEof: (cb) => void (onEof = cb),
724 onClose: () => {},
725 exit: () => {},
726 drain: () => Promise.resolve(),
727 };
728 const done = runGitService(dirOf(fork), "git-upload-pack", channel, { ownRefsOnly: true });
729 onData(new TextEncoder().encode(`${pkt(`want ${oid["later"]}\n`)}0000`));
730 onData(new TextEncoder().encode("zzzz"));
731 onEof();
732 const code = await done;
733 expect([err.includes(OWN_REFS_ONLY), Buffer.concat(sent).toString("latin1").includes("PACK")]).toEqual([true, false]);
734 expect(code).not.toBe(0);
735 }, 60_000);
736
580737 // `fetch --unshallow` by hand: a want the fork's refs reach, then `shallow`
581738 // naming a commit only the parent holds and an infinite `deepen`. Git looks
582739 // the shallow id up through the shared store, answers `unshallow <id>` and

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs