Public: anyone can read this; the crew can change it.
Mirror fetches are pinned to the vetted address and use migrate's git environment #3050
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2994-mirror-fetches-are-pinned-to-the-vetted → main 3 commits fork 5 threads · 5 unresolved
README.md +6
| @@ -660,6 +660,12 @@ and the next sync brings back the merge commit it actually made. Pushes go | ||
| 660 | 660 | through too — the pre-receive hook forwards the push upstream first, so a push |
| 661 | 661 | the origin refuses never lands here either. |
| 662 | 662 | |
| 663 | Mirror fetches are vetted and pinned: the origin's host is resolved once, refused | |
| 664 | if it is a loopback, LAN or other private address, and git is pinned to the | |
| 665 | address that passed. Git follows no redirects for a mirror, so give the clone | |
| 666 | URL in its final form (https, with the `.git` suffix the host expects); a URL | |
| 667 | that redirects fails with git's redirect error. | |
| 668 | ||
| 663 | 669 | Only the person who connected an origin can write through it. Another writer |
| 664 | 670 | gets `origin_owner_only` rather than a comment appearing upstream under |
| 665 | 671 | someone else's name with someone else's token. |
src/actions/migrate.ts +2 −16
| @@ -50,6 +50,7 @@ import { | ||
| 50 | 50 | type CreateRepoActionError, |
| 51 | 51 | } from "./repos.ts"; |
| 52 | 52 | import { tell, toldWords } from "./tell.ts"; |
| 53 | import { remoteGitEnv } from "../git/remote-env.ts"; | |
| 53 | 54 | |
| 54 | 55 | /** The whole clone — the HEAD probe and the fetch — gets this long. */ |
| 55 | 56 | export const MIGRATE_TIMEOUT_MS = 120_000; |
| @@ -266,21 +267,6 @@ export function isPrivateHost(hostname: string): boolean { | ||
| 266 | 267 | return false; |
| 267 | 268 | } |
| 268 | 269 | |
| 269 | /** | |
| 270 | * Whatever the source's own git config says, a clone follows no redirect: | |
| 271 | * the address that was checked is the only address that is fetched. Added | |
| 272 | * after any entries the credential brought, at the next free index. | |
| 273 | */ | |
| 274 | function noRedirects(env: Record<string, string>): Record<string, string> { | |
| 275 | const n = Number(env["GIT_CONFIG_COUNT"] ?? "0"); | |
| 276 | return { | |
| 277 | ...env, | |
| 278 | GIT_CONFIG_COUNT: String(n + 1), | |
| 279 | [`GIT_CONFIG_KEY_${n}`]: "http.followRedirects", | |
| 280 | [`GIT_CONFIG_VALUE_${n}`]: "false", | |
| 281 | }; | |
| 282 | } | |
| 283 | ||
| 284 | 270 | /** Dotted IPv4 → inside the network?; `undefined` when not IPv4 at all. */ |
| 285 | 271 | function privateV4(h: string): boolean | undefined { |
| 286 | 272 | const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h); |
| @@ -490,7 +476,7 @@ async function cloneInto( | ||
| 490 | 476 | const deadline = Date.now() + MIGRATE_TIMEOUT_MS; |
| 491 | 477 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); |
| 492 | 478 | const authEnv = await source.authEnv(); |
| 493 | const env = { ...noRedirects(authEnv), GIT_ALLOW_PROTOCOL: ALLOWED_PROTOCOL }; | |
| 479 | const env = remoteGitEnv(authEnv, undefined, ALLOWED_PROTOCOL); | |
| 494 | 480 | const said = (stderr: string, fallback: string): string => |
| 495 | 481 | redact(stderr.trim(), ...secretsIn(authEnv)) |
| 496 | 482 | .split("\n") |
src/git/remote-env.ts added +48
| @@ -0,0 +1,48 @@ | ||
| 1 | /** | |
| 2 | * The environment git gets whenever it talks to somebody else's server: no | |
| 3 | * redirects, only the transports a remote may use, and — when the host has | |
| 4 | * been vetted — the vetted address pinned, so git's own lookup cannot answer | |
| 5 | * differently from the one that was judged. One function, shared by the | |
| 6 | * migration clone and the mirror fetch. | |
| 7 | */ | |
| 8 | ||
| 9 | /** The transports a mirror will speak. */ | |
| 10 | export const REMOTE_PROTOCOLS = "https:http"; | |
| 11 | ||
| 12 | export interface Pin { | |
| 13 | readonly host: string; | |
| 14 | readonly port: number; | |
| 15 | readonly ip: string; | |
| 16 | } | |
| 17 | ||
| 18 | /** `host:port` and the vetted address for a clone URL, ready to pin. */ | |
| 19 | export function pinFor(url: string, ip: string): Pin { | |
| 20 | const parsed = new URL(url); | |
| 21 | const port = parsed.port !== "" ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80; | |
| 22 | return { host: parsed.hostname.replace(/^\[|\]$/g, ""), port, ip }; | |
| 23 | } | |
| 24 | ||
| 25 | /** | |
| 26 | * `authEnv` plus `http.followRedirects=false`, `GIT_ALLOW_PROTOCOL`, and a | |
| 27 | * `http.curloptResolve` entry when `pin` is given. Config entries are added | |
| 28 | * after any the credential brought, at the next free indexes. | |
| 29 | */ | |
| 30 | export function remoteGitEnv( | |
| 31 | authEnv: Record<string, string>, | |
| 32 | pin?: Pin, | |
| 33 | protocols: string = REMOTE_PROTOCOLS, | |
| 34 | ): Record<string, string> { | |
| 35 | const entries: [string, string][] = [["http.followRedirects", "false"]]; | |
| 36 | if (pin !== undefined) { | |
| 37 | const ip = pin.ip.includes(":") ? `[${pin.ip}]` : pin.ip; | |
| 38 | entries.push(["http.curloptResolve", `${pin.host}:${pin.port}:${ip}`]); | |
| 39 | } | |
| 40 | const env: Record<string, string> = { ...authEnv, GIT_ALLOW_PROTOCOL: protocols }; | |
| 41 | const n = Number(authEnv["GIT_CONFIG_COUNT"] ?? "0"); | |
| 42 | entries.forEach(([key, value], i) => { | |
| 43 | env[`GIT_CONFIG_KEY_${n + i}`] = key; | |
| 44 | env[`GIT_CONFIG_VALUE_${n + i}`] = value; | |
| 45 | }); | |
| 46 | env["GIT_CONFIG_COUNT"] = String(n + entries.length); | |
| 47 | return env; | |
| 48 | } |
src/origins/git-mirror.ts +35 −2
| @@ -24,6 +24,8 @@ import { runGitCapture, type RunOptions } from "../git/env.ts"; | ||
| 24 | 24 | import { listRefs } from "../git/read.ts"; |
| 25 | 25 | import { ZERO_OID } from "../git/hooks.ts"; |
| 26 | 26 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 27 | import { vetHost, type OutboundOptions } from "../kernel/outbound.ts"; | |
| 28 | import { pinFor, remoteGitEnv } from "../git/remote-env.ts"; | |
| 27 | 29 | import { redact, safeRemoteUrl, secretsIn } from "./credentials.ts"; |
| 28 | 30 | |
| 29 | 31 | /** A mirror fetch can be genuinely slow; Forgejo allows its own 300s. */ |
| @@ -40,6 +42,33 @@ export const MIRROR_REFSPECS = [ | ||
| 40 | 42 | "+refs/tags/*:refs/tags/*", |
| 41 | 43 | ] as const; |
| 42 | 44 | |
| 45 | let outbound: OutboundOptions = {}; | |
| 46 | ||
| 47 | /** Test seam: what `vetHost` is told (resolver, exempt addresses). Never read from the environment. */ | |
| 48 | export function provideMirrorOutboundForTest(options: OutboundOptions): void { | |
| 49 | outbound = options; | |
| 50 | } | |
Nit: a mutable module-level seam in production code. It matches provideCpusForTest, so fine, but keep it the only one. |
||
| 51 | ||
| 52 | export const MIRROR_REFUSED = "refused: the remote's host is a private address"; | |
| 53 | export const MIRROR_UNRESOLVED = "the remote's host could not be resolved"; | |
| 54 | ||
| 55 | /** | |
| 56 | * Vet the clone URL's host and return git's environment pinned to the | |
| 57 | * address that passed — or the one sentence that ends the sync. | |
| 58 | */ | |
| 59 | async function vettedEnv( | |
| 60 | url: string, | |
| 61 | authEnv: Record<string, string>, | |
| 62 | ): Promise<Result<Record<string, string>, GitMirrorError>> { | |
| 63 | const host = new URL(url).hostname; | |
| 64 | const vetted = await vetHost(host, outbound); | |
| 65 | if (!vetted.ok) | |
| 66 | return err({ | |
| 67 | message: vetted.error === "refused" ? MIRROR_REFUSED : MIRROR_UNRESOLVED, | |
| 68 | }); | |
| 69 | return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!))); | |
| 70 | } | |
| 71 | ||
| 43 | 72 | export interface RefSnapshot { |
| 44 | 73 | /** ref name → object id. */ |
| 45 | 74 | readonly refs: ReadonlyMap<string, string>; |
| @@ -159,6 +188,8 @@ export async function fetchMirror( | ||
| 159 | 188 | if (url === undefined) |
| 160 | 189 | return err({ message: "refusing an unusable clone URL" }); |
| 161 | 190 | |
| 191 | const pinned = await vettedEnv(url, authEnv); | |
| 192 | if (!pinned.ok) return pinned; | |
| 162 | 193 | const before = await snapshotRefs(dir); |
| 163 | 194 | const run = await runGitCapture( |
| 164 | 195 | [ |
| @@ -170,7 +201,7 @@ export async function fetchMirror( | ||
| 170 | 201 | url, |
| 171 | 202 | ...MIRROR_REFSPECS, |
| 172 | 203 | ], |
| 173 | { cwd: dir, env: authEnv, timeoutMs: FETCH_TIMEOUT_MS }, | |
| 204 | { cwd: dir, env: pinned.value, timeoutMs: FETCH_TIMEOUT_MS }, | |
| 174 | 205 | ); |
| 175 | 206 | if (run.code !== 0) { |
| 176 | 207 | return err({ |
| @@ -209,6 +240,8 @@ export async function fetchRefFrom( | ||
| 209 | 240 | if (!isSafeRef(srcRef)) |
fetchRefFrom is vetted here but has no test that would fail without it; add one next to the fetchMirror refusal test. |
||
| 210 | 241 | return err({ message: `refusing an unsafe ref: ${srcRef}` }); |
| 211 | 242 | |
| 243 | const pinned = await vettedEnv(url, authEnv); | |
This vettedEnv call in fetchRefFrom has no test that fails without it. Add a refusal test for fetchRefFrom. |
||
| 244 | if (!pinned.ok) return pinned; | |
| 212 | 245 | const run = await runGitCapture( |
| 213 | 246 | [ |
| 214 | 247 | "fetch", |
| @@ -220,7 +253,7 @@ export async function fetchRefFrom( | ||
| 220 | 253 | // without any local ref moving. |
| 221 | 254 | srcRef, |
| 222 | 255 | ], |
| 223 | { cwd: dir, env: authEnv, timeoutMs: FETCH_TIMEOUT_MS }, | |
| 256 | { cwd: dir, env: pinned.value, timeoutMs: FETCH_TIMEOUT_MS }, | |
| 224 | 257 | ); |
| 225 | 258 | if (run.code !== 0) { |
| 226 | 259 | return err({ |
test/helpers/preload.ts +5
| @@ -4,12 +4,17 @@ | ||
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | import { provideCpusForTest } from "../../src/core/capacity.ts"; |
| 7 | import { provideMirrorOutboundForTest } from "../../src/origins/git-mirror.ts"; | |
| 7 | 8 | import { installCheapHash } from "./cheap-hash.ts"; |
| 8 | 9 | |
| 9 | 10 | installCheapHash(); |
| 10 | 11 | // Defaults sized from CPUs (#2549) read as on a 4-CPU pod — 4 shards, one |
| 11 | 12 | // check at once — whatever machine runs the suite. |
| 12 | 13 | provideCpusForTest(4); |
| 14 | // Suites mirror from fake origins on loopback; the refusal itself is proved | |
| 15 | // in mirror-outbound.test.ts, which turns this off. | |
| 16 | provideMirrorOutboundForTest({ allow: ["127.0.0.1", "::1"] }); | |
| 17 | ||
| 13 | 18 | // Loopback is reachable for origin discovery only in dev (#2993); the suite |
| 14 | 19 | // serves its fake forges from this machine. |
| 15 | 20 | process.env.FORGE_DEV = "1"; |
test/mirror-outbound.test.ts added +82
| @@ -0,0 +1,82 @@ | ||
| 1 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { | |
| 6 | fetchMirror, | |
| 7 | fetchRefFrom, | |
| 8 | MIRROR_REFUSED, | |
| 9 | MIRROR_UNRESOLVED, | |
| 10 | provideMirrorOutboundForTest, | |
| 11 | } from "../src/origins/git-mirror.ts"; | |
| 12 | import { remoteGitEnv, pinFor } from "../src/git/remote-env.ts"; | |
| 13 | ||
| 14 | let dir: string; | |
| 15 | beforeAll(async () => { | |
| 16 | dir = await mkdtemp(join(tmpdir(), "mirror-outbound-")); | |
| 17 | provideMirrorOutboundForTest({}); | |
| 18 | }); | |
| 19 | afterAll(async () => { | |
| 20 | provideMirrorOutboundForTest({ allow: ["127.0.0.1", "::1"] }); | |
| 21 | await rm(dir, { recursive: true, force: true }); | |
| 22 | }); | |
| 23 | ||
| 24 | test("a remote that resolves to 127.0.0.1 is refused before git starts", async () => { | |
| 25 | // `dir` is not a repository: had git started, the message would be git's. | |
| 26 | provideMirrorOutboundForTest({ resolve: async () => ["127.0.0.1"] }); | |
| 27 | const refused = await fetchMirror(dir, "https://example.test/a/b.git", {}); | |
| 28 | expect(refused.ok).toBe(false); | |
| 29 | if (!refused.ok) expect(refused.error.message).toBe(MIRROR_REFUSED); | |
| 30 | const literal = await fetchMirror(dir, "http://127.0.0.1:9/a/b.git", {}); | |
| 31 | expect(literal.ok).toBe(false); | |
| 32 | if (!literal.ok) expect(literal.error.message).toBe(MIRROR_REFUSED); | |
| 33 | }); | |
| 34 | ||
outdated
Cover pinFor with an IPv6 address and an explicit port, so the bracket wrapping and the port default are both checked. |
||
| 35 | test("git's environment carries the pin, no redirects and the protocol list", () => { | |
| 36 | const env = remoteGitEnv( | |
| 37 | { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "a.b", GIT_CONFIG_VALUE_0: "c" }, | |
| 38 | pinFor("https://git.example.test/a/b.git", "93.184.216.34"), | |
| 39 | ); | |
| 40 | expect(env["GIT_ALLOW_PROTOCOL"]).toBe("https:http"); | |
| 41 | const pairs = Array.from({ length: Number(env["GIT_CONFIG_COUNT"]) }, (_, i) => [ | |
| 42 | env[`GIT_CONFIG_KEY_${i}`], | |
| 43 | env[`GIT_CONFIG_VALUE_${i}`], | |
| 44 | ]); | |
| 45 | expect(pairs).toEqual([ | |
| 46 | ["a.b", "c"], | |
| 47 | ["http.followRedirects", "false"], | |
| 48 | ["http.curloptResolve", "git.example.test:443:93.184.216.34"], | |
| 49 | ]); | |
| 50 | }); | |
| 51 | ||
| 52 | test("fetchRefFrom refuses a private host and reports an unresolvable one before git starts", async () => { | |
| 53 | provideMirrorOutboundForTest({ resolve: async () => ["127.0.0.1"] }); | |
| 54 | const refused = await fetchRefFrom(dir, "https://example.test/a/b.git", "refs/heads/main", {}); | |
| 55 | expect(refused.ok).toBe(false); | |
| 56 | if (!refused.ok) expect(refused.error.message).toBe(MIRROR_REFUSED); | |
| 57 | ||
| 58 | provideMirrorOutboundForTest({ resolve: async () => [] }); | |
| 59 | const gone = await fetchRefFrom(dir, "https://example.test/a/b.git", "refs/heads/main", {}); | |
| 60 | expect(gone.ok).toBe(false); | |
| 61 | if (!gone.ok) expect(gone.error.message).toBe(MIRROR_UNRESOLVED); | |
| 62 | const goneMirror = await fetchMirror(dir, "https://example.test/a/b.git", {}); | |
| 63 | expect(goneMirror.ok).toBe(false); | |
| 64 | if (!goneMirror.ok) expect(goneMirror.error.message).toBe(MIRROR_UNRESOLVED); | |
| 65 | }); | |
| 66 | ||
| 67 | test("pinFor brackets an IPv6 address and honours an explicit port", () => { | |
| 68 | const pin = (url: string, ip: string) => | |
| 69 | remoteGitEnv({}, pinFor(url, ip))["GIT_CONFIG_VALUE_1"]; | |
| 70 | expect(pin("https://git.example.test/a.git", "2001:db8::1")).toBe( | |
| 71 | "git.example.test:443:[2001:db8::1]", | |
| 72 | ); | |
| 73 | expect(pin("http://git.example.test:8080/a.git", "93.184.216.34")).toBe( | |
| 74 | "git.example.test:8080:93.184.216.34", | |
| 75 | ); | |
| 76 | expect(pin("http://git.example.test/a.git", "93.184.216.34")).toBe( | |
| 77 | "git.example.test:80:93.184.216.34", | |
| 78 | ); | |
| 79 | expect(pin("https://[2001:db8::2]:444/a.git", "2001:db8::2")).toBe( | |
| 80 | "2001:db8::2:444:[2001:db8::2]", | |
| 81 | ); | |
| 82 | }); |
Pin is only proved as env text. Add an end-to-end test where git must reach a fake origin through the pinned address, and check that proxy env vars cannot make curl skip the resolve entry.