Public: anyone can read this; the crew can change it.

Checking an upstream fetches only the vetted address, sends a token only to its exact https origin, and runs one check at a time #4044

Parked opened this
Closed without merging

scout/forge:scout/3977-checking-an-upstream-fetches-only-the-ve → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
3 files changed +304 −3
src/actions/upstream_follow.ts added +152
@@ -0,0 +1,152 @@
1 /**
2 * Checking a followed upstream (#3442, layer 2): one `ls-remote`, a fetch
3 * only when the tip moved, into `refs/upstream-follow/<branch>` — not a
4 * branch, so nothing of ours moves. What it found is noted on the follow row
5 * (upstream tip, commits behind). It opens nothing.
6 *
7 * The fetch is the mirror's: the host is vetted and the vetted address
8 * pinned (`vettedEnv`), https and http only. A connected origin's token goes
9 * only to the exact https origin it was connected at. One check runs per
10 * repository at a time, whoever asks.
11 */
12
13 import type { App } from "../app.ts";
14 import { getOriginFor } from "../core/origins.ts";
15 import { canWrite, type Repo } from "../core/repos.ts";
16 import { storageStale } from "../core/storage.ts";
17 import { getFollow, noteCheck } from "../core/upstream_follow.ts";
18 import { getUserById } from "../core/users.ts";
19 import { runGitCapture } from "../git/env.ts";
20 import { isSafeOid } from "../git/guard.ts";
21 import { aheadCount, resolveOid } from "../git/merge.ts";
22 import { REMOTE_PROTOCOLS } from "../git/remote-env.ts";
23 import { repoDir } from "../git/repos.ts";
24 import { err, ok, type Result } from "../kernel/result.ts";
25 import { providerForOrigin } from "../origins/credential.ts";
26 import { redact, secretsIn } from "../origins/credentials.ts";
27 import { vettedEnv } from "../origins/git-mirror.ts";
28
29 /** One check's whole budget: ls-remote and fetch together. */
30 const CHECK_TIMEOUT_MS = 120_000;
31
32 /** Where the upstream tip is staged: not a branch, so a check moves nothing of ours. */
33 export const stagedRef = (branch: string): string => `refs/upstream-follow/${branch}`;
34
35 export type CheckError = "off" | "nobody" | "unreadable" | "no_branch" | "git" | "busy";
36
37 /** What each refusal says, at every door. */
38 export const CHECK_REFUSALS: Record<CheckError, string> = {
39 off: "Following an upstream is off for this repository.",
40 nobody: "The person who turned following on can no longer write here; turn it on again.",
41 unreadable: "The upstream could not be read.",
42 no_branch: "The upstream has no such branch.",
43 git: "Git could not read the staged upstream.",
44 busy: "Already checking this upstream",
45 };
46
47 /** What one check read. */
48 export interface Checked {
49 readonly tip: string;
50 readonly behind: number;
51 }
52
53 /**
54 * Whether an origin's token may travel to this upstream: only to the exact
55 * https origin (scheme, host and port) the person connected. An `http://`
56 * upstream on the same host is fetched anonymously.
57 */
58 export function mayCarryCredential(upstreamUrl: string, webUrl: string): boolean {
59 try {
60 const up = new URL(upstreamUrl);
61 return up.protocol === "https:" && up.origin === new URL(webUrl).origin;
62 } catch {
63 return false;
64 }
65 }
66
67 /** Repositories with a check in flight: one at a time each, whoever asked. */
68 const checking = new Set<string>();
69
70 /**
71 * One check, of one repository. The claim is taken before the first await,
72 * so a double click and the clock never both run: the second is `busy`.
73 */
74 export async function checkUpstream(app: App, repo: Repo): Promise<Result<Checked, CheckError>> {
75 if (checking.has(repo.id)) return err("busy");
76 checking.add(repo.id);
77 try {
78 return await checkOnce(app, repo);
79 } finally {
80 checking.delete(repo.id);
81 }
82 }
83
84 async function checkOnce(app: App, repo: Repo): Promise<Result<Checked, CheckError>> {
85 const follow = getFollow(app.db, repo.id);
86 if (follow === undefined || !follow.enabled || repo.archivedAt !== null || repo.origin !== null)
87 return err("off");
88 const at = new Date().toISOString();
89 const failed = (error: CheckError, why = ""): Result<Checked, CheckError> => {
90 noteCheck(app.db, repo.id, { at, error: why || CHECK_REFUSALS[error] });
91 return err(error);
92 };
93 const by = getUserById(app.db, follow.byId);
94 if (by === undefined || !canWrite(app.db, repo, by)) return failed("nobody");
95
96 // The credential of a private upstream: the connected origin's, scoped to
97 // this one git invocation, and only when the upstream is that origin's https origin.
98 let authEnv: Record<string, string> = {};
99 if (follow.originId !== "") {
100 const origin = getOriginFor(app.db, by.id, follow.originId);
101 const provider = origin === undefined ? undefined : providerForOrigin(app, origin);
102 if (origin !== undefined && provider?.ok && mayCarryCredential(follow.url, origin.webUrl))
103 authEnv = await provider.value.gitAuthEnv(follow.url);
104 }
105 // Vetted as the mirror fetch is, and pinned to the address that passed.
106 const pinned = await vettedEnv(follow.url, authEnv, REMOTE_PROTOCOLS);
107 if (!pinned.ok) return failed("unreadable", pinned.error.message);
108 const env = pinned.value;
109 const said = (stderr: string): string =>
110 redact(stderr.trim(), ...secretsIn(authEnv)).split("\n").find((l) => l.trim() !== "") ?? "";
111
112 const dir = repoDir(app.reposRoot, repo.ownerName, repo.name);
113 const deadline = Date.now() + CHECK_TIMEOUT_MS;
114 const remaining = (): number => Math.max(1_000, deadline - Date.now());
115
116 const listed = await runGitCapture(
117 ["ls-remote", "--", follow.url, `refs/heads/${follow.branch}`],
118 { cwd: dir, env, timeoutMs: remaining() },
119 );
120 if (listed.code !== 0)
121 return failed("unreadable", `The upstream could not be read: ${said(listed.stderr) || "no reason given"}`);
122 const tip = listed.stdout.split("\t")[0]?.trim() ?? "";
123 if (!isSafeOid(tip)) return failed("no_branch");
124
125 // Fetch only when the tip moved: the staged ref is what the last fetch brought.
126 const ref = stagedRef(follow.branch);
127 if ((await resolveOid(dir, ref)) !== tip) {
128 const fetched = await runGitCapture(
129 [
130 "fetch",
131 "--no-tags",
132 "--no-write-fetch-head",
133 "--no-recurse-submodules",
134 "--quiet",
135 "--",
136 follow.url,
137 `+refs/heads/${follow.branch}:${ref}`,
138 ],
139 { cwd: dir, env, timeoutMs: remaining() },
140 );
141 if (fetched.code !== 0)
142 return failed("unreadable", `The upstream could not be fetched: ${said(fetched.stderr) || "no reason given"}`);
143 storageStale(app, repo.ownerId);
144 }
145 const staged = await resolveOid(dir, ref);
146 if (staged === undefined) return failed("git");
147 const main = await resolveOid(dir, `refs/heads/${repo.defaultBranch}`);
148 if (main === undefined) return failed("git", "The default branch has no commits yet.");
149 const behind = await aheadCount(dir, main, staged);
150 noteCheck(app.db, repo.id, { at, tip: staged, behind, error: "" });
151 return ok({ tip: staged, behind });
152 }
src/origins/git-mirror.ts +3 −2
@@ -57,9 +57,10 @@ export const MIRROR_UNRESOLVED = "the remote's host could not be resolved";
5757 * Vet the clone URL's host and return git's environment pinned to the
5858 * address that passed — or the one sentence that ends the sync.
5959 */
60 async function vettedEnv(
60 export async function vettedEnv(
6161 url: string,
6262 authEnv: Record<string, string>,
63 protocols?: string,
6364 ): Promise<Result<Record<string, string>, GitMirrorError>> {
6465 const host = new URL(url).hostname;
6566 const vetted = await vetHost(host, { ...outbound, trusted: trustedHosts() });
@@ -67,7 +68,7 @@ async function vettedEnv(
6768 return err({
6869 message: vetted.error === "refused" ? MIRROR_REFUSED : MIRROR_UNRESOLVED,
6970 });
70 return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!)));
71 return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!), protocols));
7172 }
7273
7374 export interface RefSnapshot {
test/upstream_follow.test.ts +149 −1
@@ -1,6 +1,7 @@
11 /**
22 * A repository's upstream setting (#3442, layer 1): stored as a row, https
3 * or http only, and an admin's to read.
3 * or http only, and an admin's to read. Layer 2: its check — one at a time,
4 * the vetted address, a token only to its exact https origin.
45 */
56
67 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
@@ -13,6 +14,12 @@ import { audienceFilter } from "../src/core/event_audience.ts";
1314 import { listEvents } from "../src/core/events.ts";
1415 import type { Repo } from "../src/core/repos.ts";
1516 import { createUser, type User } from "../src/core/users.ts";
17 import { checkUpstream, mayCarryCredential } from "../src/actions/upstream_follow.ts";
18 import { createOrigin } from "../src/core/origins.ts";
19 import { runGit } from "../src/git/env.ts";
20 import { REMOTE_PROTOCOLS } from "../src/git/remote-env.ts";
21 import { repoDir } from "../src/git/repos.ts";
22 import { MIRROR_REFUSED, provideMirrorOutboundForTest, vettedEnv } from "../src/origins/git-mirror.ts";
1623 import {
1724 getFollow,
1825 listFollowing,
@@ -26,6 +33,8 @@ let app: App;
2633 let alice: User;
2734 let guest: User;
2835 let repo: Repo;
36 let target: Repo;
37 let dir = "";
2938
3039 const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => {
3140 if (!r.ok) throw new Error(JSON.stringify(r.error));
@@ -128,3 +137,142 @@ describe("who hears of it", () => {
128137 expect(seen(guest)).not.toContain(secret);
129138 });
130139 });
140
141 describe("checking an upstream (layer 2)", () => {
142 beforeAll(async () => {
143 target = ok(await createRepo(app, alice, { name: "checked", visibility: "public" }));
144 dir = repoDir(app.reposRoot, target.ownerName, target.name);
145 provideMirrorOutboundForTest({ allow: ["127.0.0.1"] });
146 });
147 afterAll(() => provideMirrorOutboundForTest({}));
148 /** A server that records every request and answers with `refs` as a dumb-http `info/refs`. */
149 function upstream(refs: () => string, delayMs = 0) {
150 const seen: Request[] = [];
151 const server = Bun.serve({
152 port: 0,
153 hostname: "127.0.0.1",
154 async fetch(req) {
155 seen.push(req);
156 if (delayMs > 0) await Bun.sleep(delayMs);
157 return new URL(req.url).pathname.endsWith("/info/refs")
158 ? new Response(refs(), { headers: { "content-type": "text/plain" } })
159 : new Response("no", { status: 404 });
160 },
161 });
162 return { seen, server, url: `http://127.0.0.1:${server.port}/o/r.git` };
163 }
164
165 const follow = (url: string, originId = "") =>
166 putFollow(app.db, { repoId: target.id, url, branch: "main", enabled: true, byId: alice.id, originId });
167
168 describe("one check at a time", () => {
169 test("a second caller while one runs is refused", async () => {
170 const up = upstream(() => "", 150);
171 try {
172 follow(up.url);
173 const [a, b] = await Promise.all([checkUpstream(app, target), checkUpstream(app, target)]);
174 expect(!b.ok && b.error).toBe("busy");
175 expect(a.ok).toBe(false);
176 expect(!a.ok && a.error).not.toBe("busy");
177 // The claim is let go: the next one runs.
178 const c = await checkUpstream(app, target);
179 expect(!c.ok && c.error).not.toBe("busy");
180 } finally {
181 up.server.stop(true);
182 }
183 });
184 });
185
186 describe("the vetted address", () => {
187 test("is pinned in git's environment, with only https and http", async () => {
188 const env = ok(await vettedEnv("http://127.0.0.1:8123/o/r.git", {}, REMOTE_PROTOCOLS));
189 expect(env["GIT_ALLOW_PROTOCOL"]).toBe("https:http");
190 const entries = Object.entries(env).filter(([, v]) => v === "127.0.0.1:8123:127.0.0.1");
191 expect(entries.length).toBe(1);
192 expect(env[entries[0]![0].replace("VALUE", "KEY")]).toBe("http.curloptResolve");
193 });
194
195 test("a private host is refused unless trusted, and nothing is fetched", async () => {
196 provideMirrorOutboundForTest({});
197 const up = upstream(() => "");
198 try {
199 follow(up.url);
200 const r = await checkUpstream(app, target);
201 expect(!r.ok && r.error).toBe("unreadable");
202 expect(getFollow(app.db, target.id)?.error).toBe(MIRROR_REFUSED);
203 expect(up.seen.length).toBe(0);
204 } finally {
205 provideMirrorOutboundForTest({ allow: ["127.0.0.1"] });
206 up.server.stop(true);
207 }
208 });
209 });
210
211 describe("the credential", () => {
212 test("goes only to the same scheme, host and port, over https", () => {
213 const web = "https://git.example.com";
214 expect(mayCarryCredential("https://git.example.com/a/b.git", web)).toBe(true);
215 expect(mayCarryCredential("http://git.example.com/a/b.git", web)).toBe(false);
216 expect(mayCarryCredential("https://git.example.com:8443/a/b.git", web)).toBe(false);
217 expect(mayCarryCredential("https://evil.example.com/a/b.git", web)).toBe(false);
218 expect(mayCarryCredential("https://git.example.com.evil.com/a", web)).toBe(false);
219 expect(mayCarryCredential("https://git.example.com/a", "http://git.example.com")).toBe(false);
220 expect(mayCarryCredential("not a url", web)).toBe(false);
221 });
222
223 test("an http upstream on a connected host is fetched with no Authorization", async () => {
224 const up = upstream(() => "");
225 try {
226 const origin = ok(
227 createOrigin(app.db, dataDir, {
228 userId: alice.id,
229 kind: "forgejo",
230 apiUrl: `http://127.0.0.1:${up.server.port}/api/v1`,
231 webUrl: `http://127.0.0.1:${up.server.port}`,
232 login: "alice",
233 token: "s3cret-token",
234 }),
235 );
236 follow(up.url, origin.id);
237 await checkUpstream(app, target);
238 expect(up.seen.length).toBeGreaterThan(0);
239 for (const req of up.seen) expect(req.headers.get("authorization")).toBeNull();
240 } finally {
241 up.server.stop(true);
242 }
243 });
244 });
245
246 describe("what a check records", () => {
247 test("the upstream tip and how far behind, without opening anything", async () => {
248 const git = async (args: string[]) => ok(await runGit(args, { cwd: dir, env: ident })).trim();
249 const ident = {
250 GIT_AUTHOR_NAME: "t",
251 GIT_AUTHOR_EMAIL: "t@x",
252 GIT_COMMITTER_NAME: "t",
253 GIT_COMMITTER_EMAIL: "t@x",
254 };
255 const empty = await git(["hash-object", "-t", "tree", "-w", "--stdin"]).catch(() => "");
256 const tree = empty || "4b825dc642cb9eb9a060e54bf8d69288fbee4904";
257 const c1 = await git(["commit-tree", tree, "-m", "one"]);
258 const c2 = await git(["commit-tree", tree, "-p", c1, "-m", "two"]);
259 await git(["update-ref", `refs/heads/${target.defaultBranch}`, c1]);
260 // Already staged, so the check reads the tip and has nothing to fetch.
261 await git(["update-ref", "refs/upstream-follow/main", c2]);
262 const up = upstream(() => `${c2}\trefs/heads/main\n`);
263 try {
264 follow(up.url);
265 const r = ok(await checkUpstream(app, target));
266 expect(r).toEqual({ tip: c2, behind: 1 });
267 const row = getFollow(app.db, target.id)!;
268 expect(row.tip).toBe(c2);
269 expect(row.behind).toBe(1);
270 expect(row.error).toBe("");
271 expect(row.pendingId).toBeNull();
272 expect(row.opened).toBe(0);
273 } finally {
274 up.server.stop(true);
275 }
276 });
277 });
278 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs