Public: anyone can read this; the crew can change it.

Mirror fetches are pinned to the vetted address and use migrate's git environment #3050

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2994-mirror-fetches-are-pinned-to-the-vetted → main 3 commits fork 5 threads · 5 unresolved

  • Merged. scout/2994-mirror-fetches-are-pinned-to-the-vetted was merged into main as 48f9c34.

Review

Sign in to review this change.
6 files changed +178 −18
README.md +6
@@ -660,6 +660,12 @@ and the next sync brings back the merge commit it actually made. Pushes go
660660 through too — the pre-receive hook forwards the push upstream first, so a push
661661 the origin refuses never lands here either.
662662
663 Mirror fetches are vetted and pinned: the origin's host is resolved once, refused
664 if it is a loopback, LAN or other private address, and git is pinned to the
665 address that passed. Git follows no redirects for a mirror, so give the clone
666 URL in its final form (https, with the `.git` suffix the host expects); a URL
667 that redirects fails with git's redirect error.
668
663669 Only the person who connected an origin can write through it. Another writer
664670 gets `origin_owner_only` rather than a comment appearing upstream under
665671 someone else's name with someone else's token.
src/actions/migrate.ts +2 −16
@@ -50,6 +50,7 @@ import {
5050 type CreateRepoActionError,
5151 } from "./repos.ts";
5252 import { tell, toldWords } from "./tell.ts";
53 import { remoteGitEnv } from "../git/remote-env.ts";
5354
5455 /** The whole clone — the HEAD probe and the fetch — gets this long. */
5556 export const MIGRATE_TIMEOUT_MS = 120_000;
@@ -266,21 +267,6 @@ export function isPrivateHost(hostname: string): boolean {
266267 return false;
267268 }
268269
269 /**
270 * Whatever the source's own git config says, a clone follows no redirect:
271 * the address that was checked is the only address that is fetched. Added
272 * after any entries the credential brought, at the next free index.
273 */
274 function noRedirects(env: Record<string, string>): Record<string, string> {
275 const n = Number(env["GIT_CONFIG_COUNT"] ?? "0");
276 return {
277 ...env,
278 GIT_CONFIG_COUNT: String(n + 1),
279 [`GIT_CONFIG_KEY_${n}`]: "http.followRedirects",
280 [`GIT_CONFIG_VALUE_${n}`]: "false",
281 };
282 }
283
284270 /** Dotted IPv4 → inside the network?; `undefined` when not IPv4 at all. */
285271 function privateV4(h: string): boolean | undefined {
286272 const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h);
@@ -490,7 +476,7 @@ async function cloneInto(
490476 const deadline = Date.now() + MIGRATE_TIMEOUT_MS;
491477 const dir = repoDir(app.reposRoot, repo.ownerName, repo.name);
492478 const authEnv = await source.authEnv();
493 const env = { ...noRedirects(authEnv), GIT_ALLOW_PROTOCOL: ALLOWED_PROTOCOL };
479 const env = remoteGitEnv(authEnv, undefined, ALLOWED_PROTOCOL);
494480 const said = (stderr: string, fallback: string): string =>
495481 redact(stderr.trim(), ...secretsIn(authEnv))
496482 .split("\n")
src/git/remote-env.ts added +48
@@ -0,0 +1,48 @@
1 /**
2 * The environment git gets whenever it talks to somebody else's server: no
3 * redirects, only the transports a remote may use, and — when the host has
4 * been vetted — the vetted address pinned, so git's own lookup cannot answer
5 * differently from the one that was judged. One function, shared by the
6 * migration clone and the mirror fetch.
7 */
8
9 /** The transports a mirror will speak. */
10 export const REMOTE_PROTOCOLS = "https:http";
11
12 export interface Pin {
13 readonly host: string;
14 readonly port: number;
15 readonly ip: string;
16 }
17
18 /** `host:port` and the vetted address for a clone URL, ready to pin. */
19 export function pinFor(url: string, ip: string): Pin {
20 const parsed = new URL(url);
21 const port = parsed.port !== "" ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80;
22 return { host: parsed.hostname.replace(/^\[|\]$/g, ""), port, ip };
23 }
24
25 /**
26 * `authEnv` plus `http.followRedirects=false`, `GIT_ALLOW_PROTOCOL`, and a
27 * `http.curloptResolve` entry when `pin` is given. Config entries are added
28 * after any the credential brought, at the next free indexes.
29 */
30 export function remoteGitEnv(
31 authEnv: Record<string, string>,
32 pin?: Pin,
33 protocols: string = REMOTE_PROTOCOLS,
34 ): Record<string, string> {
35 const entries: [string, string][] = [["http.followRedirects", "false"]];
36 if (pin !== undefined) {
37 const ip = pin.ip.includes(":") ? `[${pin.ip}]` : pin.ip;
38 entries.push(["http.curloptResolve", `${pin.host}:${pin.port}:${ip}`]);
39 }
40 const env: Record<string, string> = { ...authEnv, GIT_ALLOW_PROTOCOL: protocols };
41 const n = Number(authEnv["GIT_CONFIG_COUNT"] ?? "0");
42 entries.forEach(([key, value], i) => {

Pin is only proved as env text. Add an end-to-end test where git must reach a fake origin through the pinned address, and check that proxy env vars cannot make curl skip the resolve entry.

43 env[`GIT_CONFIG_KEY_${n + i}`] = key;
44 env[`GIT_CONFIG_VALUE_${n + i}`] = value;
45 });
46 env["GIT_CONFIG_COUNT"] = String(n + entries.length);
47 return env;
48 }
src/origins/git-mirror.ts +35 −2
@@ -24,6 +24,8 @@ import { runGitCapture, type RunOptions } from "../git/env.ts";
2424 import { listRefs } from "../git/read.ts";
2525 import { ZERO_OID } from "../git/hooks.ts";
2626 import { err, ok, type Result } from "../kernel/result.ts";
27 import { vetHost, type OutboundOptions } from "../kernel/outbound.ts";
28 import { pinFor, remoteGitEnv } from "../git/remote-env.ts";
2729 import { redact, safeRemoteUrl, secretsIn } from "./credentials.ts";
2830
2931 /** A mirror fetch can be genuinely slow; Forgejo allows its own 300s. */
@@ -40,6 +42,33 @@ export const MIRROR_REFSPECS = [
4042 "+refs/tags/*:refs/tags/*",
4143 ] as const;
4244
45 let outbound: OutboundOptions = {};
46
47 /** Test seam: what `vetHost` is told (resolver, exempt addresses). Never read from the environment. */
48 export function provideMirrorOutboundForTest(options: OutboundOptions): void {
49 outbound = options;
50 }

Nit: a mutable module-level seam in production code. It matches provideCpusForTest, so fine, but keep it the only one.

51
52 export const MIRROR_REFUSED = "refused: the remote's host is a private address";
53 export const MIRROR_UNRESOLVED = "the remote's host could not be resolved";
54
55 /**
56 * Vet the clone URL's host and return git's environment pinned to the
57 * address that passed — or the one sentence that ends the sync.
58 */
59 async function vettedEnv(
60 url: string,
61 authEnv: Record<string, string>,
62 ): Promise<Result<Record<string, string>, GitMirrorError>> {
63 const host = new URL(url).hostname;
64 const vetted = await vetHost(host, outbound);
65 if (!vetted.ok)
66 return err({
67 message: vetted.error === "refused" ? MIRROR_REFUSED : MIRROR_UNRESOLVED,
68 });
69 return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!)));
70 }
71
4372 export interface RefSnapshot {
4473 /** ref name → object id. */
4574 readonly refs: ReadonlyMap<string, string>;
@@ -159,6 +188,8 @@ export async function fetchMirror(
159188 if (url === undefined)
160189 return err({ message: "refusing an unusable clone URL" });
161190
191 const pinned = await vettedEnv(url, authEnv);
192 if (!pinned.ok) return pinned;
162193 const before = await snapshotRefs(dir);
163194 const run = await runGitCapture(
164195 [
@@ -170,7 +201,7 @@ export async function fetchMirror(
170201 url,
171202 ...MIRROR_REFSPECS,
172203 ],
173 { cwd: dir, env: authEnv, timeoutMs: FETCH_TIMEOUT_MS },
204 { cwd: dir, env: pinned.value, timeoutMs: FETCH_TIMEOUT_MS },
174205 );
175206 if (run.code !== 0) {
176207 return err({
@@ -209,6 +240,8 @@ export async function fetchRefFrom(
209240 if (!isSafeRef(srcRef))

fetchRefFrom is vetted here but has no test that would fail without it; add one next to the fetchMirror refusal test.

210241 return err({ message: `refusing an unsafe ref: ${srcRef}` });
211242
243 const pinned = await vettedEnv(url, authEnv);

This vettedEnv call in fetchRefFrom has no test that fails without it. Add a refusal test for fetchRefFrom.

244 if (!pinned.ok) return pinned;
212245 const run = await runGitCapture(
213246 [
214247 "fetch",
@@ -220,7 +253,7 @@ export async function fetchRefFrom(
220253 // without any local ref moving.
221254 srcRef,
222255 ],
223 { cwd: dir, env: authEnv, timeoutMs: FETCH_TIMEOUT_MS },
256 { cwd: dir, env: pinned.value, timeoutMs: FETCH_TIMEOUT_MS },
224257 );
225258 if (run.code !== 0) {
226259 return err({
test/helpers/preload.ts +5
@@ -4,12 +4,17 @@
44 */
55
66 import { provideCpusForTest } from "../../src/core/capacity.ts";
7 import { provideMirrorOutboundForTest } from "../../src/origins/git-mirror.ts";
78 import { installCheapHash } from "./cheap-hash.ts";
89
910 installCheapHash();
1011 // Defaults sized from CPUs (#2549) read as on a 4-CPU pod — 4 shards, one
1112 // check at once — whatever machine runs the suite.
1213 provideCpusForTest(4);
14 // Suites mirror from fake origins on loopback; the refusal itself is proved
15 // in mirror-outbound.test.ts, which turns this off.
16 provideMirrorOutboundForTest({ allow: ["127.0.0.1", "::1"] });
17
1318 // Loopback is reachable for origin discovery only in dev (#2993); the suite
1419 // serves its fake forges from this machine.
1520 process.env.FORGE_DEV = "1";
test/mirror-outbound.test.ts added +82
@@ -0,0 +1,82 @@
1 import { afterAll, beforeAll, expect, test } from "bun:test";
2 import { mkdtemp, rm } from "node:fs/promises";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import {
6 fetchMirror,
7 fetchRefFrom,
8 MIRROR_REFUSED,
9 MIRROR_UNRESOLVED,
10 provideMirrorOutboundForTest,
11 } from "../src/origins/git-mirror.ts";
12 import { remoteGitEnv, pinFor } from "../src/git/remote-env.ts";
13
14 let dir: string;
15 beforeAll(async () => {
16 dir = await mkdtemp(join(tmpdir(), "mirror-outbound-"));
17 provideMirrorOutboundForTest({});
18 });
19 afterAll(async () => {
20 provideMirrorOutboundForTest({ allow: ["127.0.0.1", "::1"] });
21 await rm(dir, { recursive: true, force: true });
22 });
23
24 test("a remote that resolves to 127.0.0.1 is refused before git starts", async () => {
25 // `dir` is not a repository: had git started, the message would be git's.
26 provideMirrorOutboundForTest({ resolve: async () => ["127.0.0.1"] });
27 const refused = await fetchMirror(dir, "https://example.test/a/b.git", {});
28 expect(refused.ok).toBe(false);
29 if (!refused.ok) expect(refused.error.message).toBe(MIRROR_REFUSED);
30 const literal = await fetchMirror(dir, "http://127.0.0.1:9/a/b.git", {});
31 expect(literal.ok).toBe(false);
32 if (!literal.ok) expect(literal.error.message).toBe(MIRROR_REFUSED);
33 });
34
outdated

Cover pinFor with an IPv6 address and an explicit port, so the bracket wrapping and the port default are both checked.

35 test("git's environment carries the pin, no redirects and the protocol list", () => {
36 const env = remoteGitEnv(
37 { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "a.b", GIT_CONFIG_VALUE_0: "c" },
38 pinFor("https://git.example.test/a/b.git", "93.184.216.34"),
39 );
40 expect(env["GIT_ALLOW_PROTOCOL"]).toBe("https:http");
41 const pairs = Array.from({ length: Number(env["GIT_CONFIG_COUNT"]) }, (_, i) => [
42 env[`GIT_CONFIG_KEY_${i}`],
43 env[`GIT_CONFIG_VALUE_${i}`],
44 ]);
45 expect(pairs).toEqual([
46 ["a.b", "c"],
47 ["http.followRedirects", "false"],
48 ["http.curloptResolve", "git.example.test:443:93.184.216.34"],
49 ]);
50 });
51
52 test("fetchRefFrom refuses a private host and reports an unresolvable one before git starts", async () => {
53 provideMirrorOutboundForTest({ resolve: async () => ["127.0.0.1"] });
54 const refused = await fetchRefFrom(dir, "https://example.test/a/b.git", "refs/heads/main", {});
55 expect(refused.ok).toBe(false);
56 if (!refused.ok) expect(refused.error.message).toBe(MIRROR_REFUSED);
57
58 provideMirrorOutboundForTest({ resolve: async () => [] });
59 const gone = await fetchRefFrom(dir, "https://example.test/a/b.git", "refs/heads/main", {});
60 expect(gone.ok).toBe(false);
61 if (!gone.ok) expect(gone.error.message).toBe(MIRROR_UNRESOLVED);
62 const goneMirror = await fetchMirror(dir, "https://example.test/a/b.git", {});
63 expect(goneMirror.ok).toBe(false);
64 if (!goneMirror.ok) expect(goneMirror.error.message).toBe(MIRROR_UNRESOLVED);
65 });
66
67 test("pinFor brackets an IPv6 address and honours an explicit port", () => {
68 const pin = (url: string, ip: string) =>
69 remoteGitEnv({}, pinFor(url, ip))["GIT_CONFIG_VALUE_1"];
70 expect(pin("https://git.example.test/a.git", "2001:db8::1")).toBe(
71 "git.example.test:443:[2001:db8::1]",
72 );
73 expect(pin("http://git.example.test:8080/a.git", "93.184.216.34")).toBe(
74 "git.example.test:8080:93.184.216.34",
75 );
76 expect(pin("http://git.example.test/a.git", "93.184.216.34")).toBe(
77 "git.example.test:80:93.184.216.34",
78 );
79 expect(pin("https://[2001:db8::2]:444/a.git", "2001:db8::2")).toBe(
80 "2001:db8::2:444:[2001:db8::2]",
81 );
82 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs