Public: anyone can read this; the crew can change it.

Cloning from outside is slow enough to fail: serve the bulk of a clone from object storage, not the home uplink #14410

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14409-cloning-from-outside-is-slow-enough-to-f → main 5 commits fork 3 threads · 3 unresolved

  • Merged. scout/14409-cloning-from-outside-is-slow-enough-to-f was merged into main as 1b8007e.

Review

Sign in to review this change.
14 files changed +682 −14
docs/features/clone-from-storage.md added +14
@@ -0,0 +1,14 @@
1 ---
2 topic: Your first app in five minutes
3 row: Clones from storage
4 ---
5 # Clones from storage
6
7 A clone left through the home uplink, which is shared and slow, and a big pack cut partway failed with `early EOF` (#14409).
8
9 - **What the forge does** (`src/git/bundle_uri.ts`): for each public repository it publishes a git bundle of the default branch under `bundles/<owner>/<name>/<commit>.bundle` in the bucket the forge backs up to (`S3_*`), and advertises it with git's `bundle-uri` capability. It is made after a push to the default branch and on a clone's ref advertisement when missing or behind, one build per repository, none more often than a minute, and two at once at most across the forge. The newest and the one before it are kept, the one before only while it is part of the new history (a force-push drops it).
10 - **Taking it back**: a repository that goes private, is renamed, moved or deleted has every object under its `bundles/<owner>/<name>/` deleted from the bucket and its `forge-bundle.json` removed first, since the bucket is publicly readable.
11 - **To turn it on**: set `FORGE_BUNDLE_BASE_URL` to the address the bucket (or a CDN in front of it) is read from without credentials, and make `bundles/` publicly readable there. Unset, nothing is published or advertised. A private repository is never bundled or advertised.
12 - **For a client** (git 2.38+): `git -c transfer.bundleURI=true clone <url>` downloads the bundle from storage, then fetches only what is newer from the forge. git does not ask for bundles unless told to; a client that does not clones whole from the forge as before.
13 - **On a slow link without that**: `git clone --filter=blob:none <url>`, one sentence and a copy field on the repository page, and a `clone (slow link)` line in the terminal's repository About.
14 - Not done: the uplink shaper's priority for git over model traffic is host configuration (`docs/features/bottleneck.md`), not in this repository; a bundle is the default branch only, so other branches come from the forge; a private repository's bundle would need signed addresses.
src/actions/accounts.ts +2
@@ -10,6 +10,7 @@
1010 * and scrubs the account; the row stays so the name stays taken.
1111 */
1212
13 import { retractBundles } from "../git/bundle_uri.ts";
1314 import { removeDrop } from "../drop/store.ts";
1415 import type { Database } from "bun:sqlite";
1516 import type { App } from "../app.ts";
@@ -185,6 +186,7 @@ async function removeOwnedRepos(
185186 for (const repo of listReposOwnedBy(app.db, owner.id)) {
186187 if (runtime !== undefined) await runtime.teardownAll(repo.id);
187188 const label = `${owner.username}/${repo.name}`;
189 await retractBundles(app, repo);
188190 const deleted = await destroyRepo(app.db, app.reposRoot, repo);
189191 if (!deleted.ok) return label;
190192 await removeInstallCache(app.config.dataDir, repo.id);
src/actions/claim.ts +2
@@ -8,6 +8,7 @@
88 * repository that will not move stays with the agent and is reported; when
99 * none moves the link is given back.
1010 */
11 import { retractBundles } from "../git/bundle_uri.ts";
1112 import type { App } from "../app.ts";
1213 import { setSuspended } from "../core/accounts.ts";
1314 import { forgetListings } from "../core/listings.ts";
@@ -70,6 +71,7 @@ export async function claimStart(
7071 const moved: Repo[] = [];
7172 const stayed: string[] = [];
7273 for (const repo of listReposOwnedBy(app.db, agent.id)) {
74 await retractBundles(app, repo);
7375 const done = await transferRepo(app.db, app.reposRoot, repo, org.value, claimer.id);
7476 if (done.ok) {
7577 moved.push(done.value);
src/actions/repos.ts +5
@@ -4,6 +4,7 @@
44 * write path — committing files to a branch without a git client.
55 */
66
7 import { retractBundles } from "../git/bundle_uri.ts";
78 import { removeDrop } from "../drop/store.ts";
89 import { getOriginById } from "../core/origins.ts";
910 import { isContributionFork, type GuestRefusal } from "../core/contribute.ts";
@@ -456,6 +457,9 @@ export async function updateRepoSettings(
456457 }
457458 const { archived, ...rest } = settings;
458459 writeSettings(app.db, repo, rest);
460 // A bundle sits in a publicly readable bucket: a repository that stops being public takes it back
461 // (#14409).
462 if (repo.visibility === "public" && settings.visibility === "private") await retractBundles(app, repo);
459463 const moved =
460464 archived === undefined ? false : await moveArchive(app, repo.id, archived);
461465 const updated = getRepoById(app.db, repo.id);
@@ -587,6 +591,7 @@ export async function deleteRepo(
587591 ? ok(actor)
588592 : requireRepoAdmin(app, actor, repo);
589593 if (!admin.ok) return admin;
594 await retractBundles(app, repo);
590595 const deleted = await destroyRepo(app.db, app.reposRoot, repo);
591596 if (!deleted.ok) return deleted;
592597 await removeInstallCache(app.config.dataDir, repo.id);
src/actions/transfer.ts +3
@@ -8,6 +8,7 @@
88 * where.
99 */
1010
11 import { retractBundles } from "../git/bundle_uri.ts";
1112 import type { App } from "../app.ts";
1213 import { getUserByName, type User } from "../core/users.ts";
1314 import { canAdmin, type Repo } from "../core/repos.ts";
@@ -50,6 +51,7 @@ export async function transferRepository(
5051 if (target === undefined || target.kind === "agent") return err("unknown_owner");
5152 const ownerId = resolveOwner(app, actor, target.username);
5253 if (ownerId !== target.id && !actor.isAdmin) return err("forbidden_owner");
54 await retractBundles(app, repo);
5355 const moved = await transferRepo(app.db, app.reposRoot, repo, target, actor.id);
5456 if (!moved.ok) return moved;
5557 storageChanged(app, repo.ownerId);
@@ -80,6 +82,7 @@ export async function renameRepository(
8082 newName: string,
8183 ): Promise<Result<Repo, TransferActionError>> {
8284 if (actor === undefined || actor.kind === "agent" || !canAdmin(app.db, repo, actor)) return err("not_found");
85 await retractBundles(app, repo);
8386 const moved = await renameRepo(app.db, app.reposRoot, repo, newName, actor.id);
8487 if (!moved.ok) return moved;
8588 app.bus.append({
src/app.ts +3
@@ -34,6 +34,7 @@ import {
3434 import { RefusedLedger } from "./kernel/internal_refused.ts";
3535 import { closeReadersUnder } from "./git/read.ts";
3636 import { dropAllPins } from "./git/pins_kept.ts";
37 import { onPushEvent } from "./git/bundle_uri.ts";
3738
3839 export interface ForgeConfig {
3940 /** The engine shape this boot chose and why (`engine/mode.ts`), for `/system`; none in a test. */
@@ -296,6 +297,8 @@ export function createApp(config: ForgeConfig): App {
296297 // sends them. Listener throws are swallowed by the bus, so this can never
297298 // break the action that produced the event.
298299 bus.subscribe((event) => enqueueForEvent(db, event), "local");
300 // A push to a public repository's default branch makes its clone bundle again (#14409).
301 bus.subscribe((event) => onPushEvent({ db, reposRoot }, event), "local");
299302 // A pull opened or merged, a release served, an issue dropped or split:
300303 // estimates settle (#11955).
301304 bus.subscribe((event) => settleEstimates(db, event), "local");
src/git/bundle_uri.ts added +251
@@ -0,0 +1,251 @@
1 /**
2 * The bulk of a clone from object storage, not the home uplink (#14409).
3 *
4 * A public repository's default branch is published as one git bundle to the
5 * bucket the forge already backs up to (`S3_*`), and upload-pack is told to
6 * advertise it with git's `bundle-uri` capability (protocol v2, git 2.38+). A
7 * client that asks (`git -c transfer.bundleURI=true clone`) downloads the
8 * bundle from `FORGE_BUNDLE_BASE_URL` (the bucket's public read address or a
9 * CDN in front of it) and then fetches only what is newer from the forge. A
10 * client without bundle-uri never sees the capability and clones as before.
11 *
12 * State: one small file per repository, `<repo>/forge-bundle.json`
13 * (`{ oid, url }`: the commit the bundle holds and where it is), written only
14 * after the upload succeeded. Nothing is advertised for a repository that is
15 * not public, whatever the file says.
16 *
17 * When one is made again: after a push to the default branch, and on a clone's ref advertisement
18 * when the pointer is missing or behind. One build per repository at a time, and none tried twice
19 * within `COOLDOWN_MS`, so a busy branch or a bucket that refuses costs one bundle a minute, never
20 * a pile. Held in memory: the ids building and when each was last tried, at most the repositories
21 * touched in the last minute (pruned); gone on restart, and the next clone or push asks again.
22 */
23
24 import { readFile, rename, rm, writeFile } from "node:fs/promises";
25 import { join } from "node:path";
26 import { runGit, runGitRaw } from "./env.ts";
27 import type { App } from "../app.ts";
28 import type { ForgeEvent } from "../core/events.ts";
29 import { getRepoById, type Repo } from "../core/repos.ts";
30 import { createLog } from "../kernel/log.ts";
31 import { repoDir } from "./repos.ts";
32 import { s3FromEnv, s3Store, type Store } from "../snapshot/store.ts";
33
34 export interface Published {
35 readonly oid: string;
36 readonly url: string;
37 }
38
39 /** The most a bundle may be in memory while it is built and sent: 512 MiB. */
40 const BUNDLE_MAX = 512 * 1024 * 1024;
41 const POINTER = "forge-bundle.json";
42 const OID = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
43 const URL_SHAPE = /^https?:\/\/[^\s"'\\]{1,400}$/;
44
45 /** The public address the bucket is read from, or undefined (the feature is off). */
46 export function bundleBase(env: Record<string, string | undefined> = process.env): string | undefined {
47 const raw = env["FORGE_BUNDLE_BASE_URL"]?.trim().replace(/\/+$/, "");
48 if (raw === undefined || !/^https?:\/\/[^\s"'\\]{1,200}$/.test(raw)) return undefined;
49 return raw;
50 }
51
52 /**
53 * The object key of a repository's bundle at one commit. Names are validated elsewhere; encoded
54 * again here.
55 */
56 export const bundleKey = (owner: string, name: string, oid: string): string =>
57 `bundles/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/${oid}.bundle`;
58
59 /**
60 * What a repository's pointer file holds, or undefined when it is absent or not what it should be.
61 */
62 export async function pointerOf(dir: string): Promise<Published | undefined> {
63 try {
64 const v: unknown = JSON.parse(await readFile(join(dir, POINTER), "utf8"));
65 if (typeof v !== "object" || v === null) return undefined;
66 const { oid, url } = v as Record<string, unknown>;
67 return typeof oid === "string" && OID.test(oid) && typeof url === "string" && URL_SHAPE.test(url)
68 ? { oid, url }
69 : undefined;
70 } catch {
71 return undefined;
72 }
73 }
74
75 /** The `git -c` arguments that make upload-pack advertise `url` as the clone's first download. */
76 export function bundleConfig(url: string | undefined): string[] {
77 if (url === undefined || !URL_SHAPE.test(url)) return [];
78 return [
79 "-c", "uploadpack.advertiseBundleURIs=true",
80 "-c", "bundle.version=1",
81 "-c", "bundle.mode=all",
82 "-c", `bundle.default.uri=${url}`,
83 ];
84 }
85
86 /**
87 * The address to advertise for a repository: its published bundle if it is public and the feature
88 * is on.
89 */
90 export async function bundleUriFor(
91 dir: string,
92 visibility: "public" | "private",
93 env: Record<string, string | undefined> = process.env,
94 ): Promise<string | undefined> {
95 if (visibility !== "public" || bundleBase(env) === undefined) return undefined;
96 return (await pointerOf(dir))?.url;
97 }
98
99 /**
100 * Bundle `branch` and put it in `store`, then point at it. Nothing is done when the pointer already
101 * names the branch's tip. The one older bundle is kept for a client mid-download; the rest go.
102 * Returns what is published, or undefined when there was nothing to publish.
103 */
104 export async function publishBundle(input: {
105 readonly store: Store;
106 readonly base: string;
107 readonly dir: string;
108 readonly owner: string;
109 readonly name: string;
110 readonly branch: string;
111 }): Promise<Published | undefined> {
112 const { store, base, dir, owner, name, branch } = input;
113 const ref = `refs/heads/${branch}`;
114 const tip = await runGit(["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd: dir });
115 if (!tip.ok || !OID.test(tip.value)) return undefined;
116 const before = await pointerOf(dir);
117 if (before?.oid === tip.value) return before;
118
119 // The bundle goes to memory, not a file: nothing is left behind to clean up, and
120 // `BUNDLE_MAX` bounds it.
121 const made = await runGitRaw(["bundle", "create", "-", ref], {
122 cwd: dir,
123 timeoutMs: 600_000,
124 maxBytes: BUNDLE_MAX,
125 });
126 if (!made.ok) throw new Error(`git bundle: ${made.error.stderr.slice(0, 200)}`);
127 const key = bundleKey(owner, name, tip.value);
128 await store.put(key, made.value);
129 const now: Published = { oid: tip.value, url: `${base}/${key}` };
130 const tmp = join(dir, `${POINTER}.tmp`);
131 await writeFile(tmp, JSON.stringify(now));
132 await rename(tmp, join(dir, POINTER));
133 // The one before is kept for a client mid-download, but only while it is part of the new history:
134 // after a force-push it holds what the push meant to remove.
135 const kept =
136 before !== undefined &&
137 (await runGit(["merge-base", "--is-ancestor", before.oid, tip.value], { cwd: dir })).ok;
138 const keep = new Set([key, kept && before !== undefined ? bundleKey(owner, name, before.oid) : ""]);
139 for (const old of await store.list(`bundles/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/`))
140 if (!keep.has(old)) await store.delete(old).catch(() => {});
141 return now;
142 }
143
144 // ---- when ----
145
146 const log = createLog("bundle");
147 export const COOLDOWN_MS = 60_000;
148 /**
149 * Builds in flight at once in one forge, whatever the repositories: each holds up to `BUNDLE_MAX`
150 * in memory.
151 */
152 export const MAX_BUILDS = 2;
153
154 interface Held {
155 readonly building: Map<string, Promise<void>>;
156 readonly triedAt: Map<string, number>;
157 /** Settled builds, for tests to wait on. */
158 readonly settled: Set<Promise<void>>;
159 }
160 const held = new WeakMap<object, Held>();
161 const heldBy = (app: Pick<App, "db">): Held => {
162 let h = held.get(app.db);
163 if (h === undefined) held.set(app.db, (h = { building: new Map(), triedAt: new Map(), settled: new Set() }));
164 return h;
165 };
166 let store: Store | undefined;
167
168 /** The builds in flight for this forge, for tests to wait on. */
169 export const settledOf = (app: Pick<App, "db">): ReadonlySet<Promise<void>> => heldBy(app).settled;
170
171 function bucket(): Store | undefined {
172 const target = s3FromEnv();
173 if (target === undefined) return undefined;
174 return (store ??= s3Store(target, 2));
175 }
176
177 /** Make this repository's bundle current in the background; a no-op when it cannot or need not. */
178 export function noteBundle(
179 app: Pick<App, "db" | "reposRoot">,
180 repo: Pick<Repo, "id" | "ownerName" | "name" | "visibility" | "defaultBranch">,
181 using: { store?: Store; base?: string; now?: number } = {},
182 ): void {
183 const base = using.base ?? bundleBase();
184 const to = using.store ?? bucket();
185 if (base === undefined || to === undefined || repo.visibility !== "public") return;
186 const now = using.now ?? Date.now();
187 const { building, triedAt, settled } = heldBy(app);
188 if (building.size >= MAX_BUILDS || building.has(repo.id) || now - (triedAt.get(repo.id) ?? -Infinity) < COOLDOWN_MS) return;
189 for (const [id, at] of triedAt) if (now - at >= COOLDOWN_MS) triedAt.delete(id);
190 triedAt.set(repo.id, now);
191 const job = publishBundle({
192 store: to,
193 base,
194 dir: repoDir(app.reposRoot, repo.ownerName, repo.name),
195 owner: repo.ownerName,
196 name: repo.name,
197 branch: repo.defaultBranch,
198 })
199 .then(() => {})
200 .catch((error: unknown) => {
201 log.warn("bundle not published", { repo: `${repo.ownerName}/${repo.name}`, why: String(error).slice(0, 200) });
202 })
203 .finally(() => {
204 building.delete(repo.id);
205 settled.delete(job);
206 });
207 building.set(repo.id, job);
208 settled.add(job);
209 }
210
211 /**
212 * Take a repository's bundles back: before it goes private, is renamed, moved or deleted, every
213 * object under its prefix is deleted from the bucket (it is publicly readable and the address is
214 * guessable) and the pointer is removed. A build in flight is waited for first, so it cannot put
215 * one back. A bucket that refuses is logged; the pointer is gone either way, so nothing is
216 * advertised.
217 */
218 export async function retractBundles(
219 app: Pick<App, "db" | "reposRoot">,
220 repo: Pick<Repo, "id" | "ownerName" | "name">,
221 using: { store?: Store } = {},
222 ): Promise<void> {
223 await heldBy(app).building.get(repo.id);
224 await rm(join(repoDir(app.reposRoot, repo.ownerName, repo.name), POINTER), { force: true });
225 heldBy(app).triedAt.delete(repo.id);
226 const to = using.store ?? bucket();
227 if (to === undefined) return;
228 const prefix = `bundles/${encodeURIComponent(repo.ownerName)}/${encodeURIComponent(repo.name)}/`;
229 try {
230 for (const key of await to.list(prefix)) await to.delete(key);
231 } catch (error) {
232 log.warn("bundles not retracted", { repo: `${repo.ownerName}/${repo.name}`, why: String(error).slice(0, 200) });
233 }
234 }
235
236 /** A push that moved the default branch makes the bundle again. */
237 export function onPushEvent(
238 app: Pick<App, "db" | "reposRoot">,
239 event: ForgeEvent,
240 using: { store?: Store; base?: string } = {},
241 ): void {
242 if (event.kind !== "push" || event.repoId === null) return;
243 const updates = event.payload["updates"];
244 if (!Array.isArray(updates)) return;
245 const repo = getRepoById(app.db, event.repoId);
246 if (!repo) return;
247 const moved = updates.some(
248 (u: unknown) => typeof u === "object" && u !== null && (u as { ref?: unknown }).ref === `refs/heads/${repo.defaultBranch}`,
249 );
250 if (moved) noteBundle(app, repo, using);
251 }
src/git/smart-http.ts +13 −2
@@ -19,6 +19,7 @@ import { gitEnv, transferGate } from "./env.ts";
1919 import { pullPipe, type PullPipe } from "./pull-pipe.ts";
2020 import { transferDeadline, type Stall } from "./progress.ts";
2121 import { pacer } from "./pace.ts";
22 import { bundleConfig } from "./bundle_uri.ts";
2223 import { OWN_REFS_ONLY, reachableFromOwnRefs, refusalLine, wantsOf } from "./own_refs.ts";
2324
2425 export type GitService = "git-upload-pack" | "git-receive-pack";
@@ -67,12 +68,15 @@ export function serviceCommand(
6768 service: GitService,
6869 args: readonly string[],
6970 pushOptions = true,
71 bundleUri?: string,
7072 ): string[] {
7173 // "git-upload-pack" the service is "upload-pack" the subcommand.
7274 const subcommand = service.slice(4);
7375 const config =
7476 service === "git-upload-pack" ? UPLOAD_PACK_CONFIG : pushOptions ? RECEIVE_PACK_CONFIG : [];
75 return ["git", ...config, subcommand, ...args];
77 // A published bundle is advertised to a fetch only (`bundle_uri.ts`).
78 const bundle = service === "git-upload-pack" ? bundleConfig(bundleUri) : [];
79 return ["git", ...config, ...bundle, subcommand, ...args];
7680 }
7781
7882 export interface SmartHttpOptions {
@@ -93,6 +97,8 @@ export interface SmartHttpOptions {
9397 idleMs?: number;
9498 /** Offer push options (`git push -o`); a mirror's push never takes them. @default true */
9599 pushOptions?: boolean;
100 /** Where this repository's published bundle is, advertised as `bundle-uri` (`bundle_uri.ts`). */
101 bundleUri?: string;
96102 /** Given back when the transfer ends, however it ends (a stranger's share). */
97103 release?: () => void;
98104 /**
@@ -241,7 +247,12 @@ async function spawnService(
241247 try {
242248 pipe = await pullPipe(() => proc.kill("SIGKILL"));
243249 proc = Bun.spawn(
244 serviceCommand(service, ["--stateless-rpc", ...extraArgs, repoDir], options.pushOptions),
250 serviceCommand(
251 service,
252 ["--stateless-rpc", ...extraArgs, repoDir],
253 options.pushOptions,
254 held ? undefined : options.bundleUri,
255 ),
245256 {
246257 env: gitEnv(env),
247258 stdin: "pipe",
src/http/server.ts +30 −9
@@ -125,6 +125,7 @@ import { PUSH_HOLDS_SECRET, pushHolds } from "../crew/leak.ts";
125125 import { heldLongForms } from "../kernel/log.ts";
126126 import { asReader, heldPushConfig, pushCarriesWhatItNames, SEND_WHAT_YOU_NAME } from "../git/own_refs.ts";
127127 import { holdFor, lendersBeyond } from "../core/fork_reach.ts";
128 import { bundleUriFor, noteBundle } from "../git/bundle_uri.ts";
128129 import {
129130 advertiseRefs,
130131 isGitService,
@@ -1326,24 +1327,31 @@ export function buildRouter(): Router<Handler> {
13261327 const service = ctx.url.searchParams.get("service");
13271328 if (!isGitService(service))
13281329 return new Response("smart http only", { status: 400 });
1329 return withGitAccess(ctx, service, (dir, _repoId, _cap, held) =>
1330 asStranger(ctx, (options) =>
1330 return withGitAccess(ctx, service, async (dir, _repoId, _cap, held) => {
1331 const bundleUri = await bundleUriOf(ctx, dir, service);

Anonymous GET triggers noteBundle for any public repo (see finding 2): needs a global concurrency cap. No test goes through this route (finding 3).

1332 return asStranger(ctx, (options) =>
13311333 advertiseRefs(dir, service, ctx.request, {
13321334 ...options,
1335 ...(bundleUri === undefined ? {} : { bundleUri }),
13331336 // A caller who may not read what this fork borrows from (`git/own_refs.ts`).
13341337 ownRefsOnly: held && service === "git-upload-pack",
13351338 ...(held && service === "git-receive-pack" ? { env: heldPushConfig({}) } : {}),
13361339 pushOptions: getRepo(ctx.app.db, ctx.params.owner!, ctx.params.name!)?.origin === null,
13371340 }),
1338 ),
1339 );
1341 );
1342 });
13401343 });
13411344 router.add("POST", "/:owner/:name.git/git-upload-pack", (ctx) =>
1342 withGitAccess(ctx, "git-upload-pack", (dir, _repoId, _cap, held) =>
1343 asStranger(ctx, (options) =>
1344 serviceRpc(dir, "git-upload-pack", ctx.request, { ...options, ownRefsOnly: held }),
1345 ),
1346 ),
1345 withGitAccess(ctx, "git-upload-pack", async (dir, _repoId, _cap, held) => {
1346 const bundleUri = await bundleUriOf(ctx, dir, "git-upload-pack");
1347 return asStranger(ctx, (options) =>
1348 serviceRpc(dir, "git-upload-pack", ctx.request, {
1349 ...options,
1350 ownRefsOnly: held,
1351 ...(bundleUri === undefined ? {} : { bundleUri }),
1352 }),
1353 );
1354 }),
13471355 );
13481356 router.add("POST", "/:owner/:name.git/git-receive-pack", (ctx) =>
13491357 withGitAccess(ctx, "git-receive-pack", (dir, repoId, cap) =>
@@ -2023,6 +2031,19 @@ async function withGitAccess(
20232031 : served;
20242032 }
20252033
2034 /**
2035 * The published bundle to advertise to this fetch (`git/bundle_uri.ts`), if the repository is
2036 * public and has one; a fetch's ref advertisement also asks for the bundle to be made or brought up
2037 * to date.
2038 */
2039 async function bundleUriOf(ctx: Ctx, dir: string, service: GitService): Promise<string | undefined> {
2040 if (service !== "git-upload-pack") return undefined;
2041 const repo = getRepo(ctx.app.db, ctx.params.owner!, ctx.params.name!);
2042 if (!repo) return undefined;
2043 if (ctx.request.method === "GET") noteBundle(ctx.app, repo);
2044 return bundleUriFor(dir, repo.visibility);
2045 }
2046
20262047 /** `body`, unchanged, then `done` once it has ended or been given up. */
20272048 function afterBody(body: ReadableStream<Uint8Array>, done: () => void): ReadableStream<Uint8Array> {
20282049 const reader = body.getReader();
src/tui/screens/about.ts +3 −1
@@ -994,8 +994,10 @@ export function cloneFacts(f: Session, repo: Repo): [string, string][] {
994994 config.url !== undefined
995995 ? new URL(config.url).origin
996996 : `http://${config.host ?? "localhost"}:${config.port ?? 3000}`;
997 const url = `${origin}/${repo.ownerName}/${repo.name}.git`;
997998 const out: [string, string][] = [
998 ["clone (http)", `${origin}/${repo.ownerName}/${repo.name}.git`],
999 ["clone (http)", url],
1000 ["clone (slow link)", `git clone --filter=blob:none ${url}`],
9991001 ];
10001002 const ssh = sshCloneUrl(f.app, repo.ownerName, repo.name);
10011003 if (ssh !== undefined) out.push(["clone (ssh)", ssh]);
src/web/pages/repo.ts +2
@@ -1013,6 +1013,8 @@ function aboutCard(
10131013 )}
10141014 <p class="mt small muted">Clone over HTTPS</p>
10151015 ${copyField("clone-https", cloneUrl, "HTTPS clone address", "Copy clone URL")}
1016 <p class="mt small muted">On a slow link, clone without the file contents.</p>
outdated

Principles 1 and 8: hand-rolled paragraph instead of the copyField component, and a second explanation of cloning. Also missing from the terminal screen and untested (findings 5, 3).

1017 ${copyField("clone-slow", `git clone --filter=blob:none ${cloneUrl}`, "Clone command for a slow link", "Copy slow-link clone command")}
10161018 ${when(
10171019 sshUrl,
10181020 (url) =>
test/bundle-uri-wired.test.ts added +135
@@ -0,0 +1,135 @@
1 /**
2 * The bundle's wiring through the real router and the real actions (#14409): what a clone's ref
3 * advertisement says, what a push does, and what a repository that stops being public (or is
4 * renamed) leaves behind. `bundle-uri.test.ts` holds the module itself.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtemp, readFile, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { commitToBranch, createRepo, updateRepoSettings } from "../src/actions/repos.ts";
12 import { renameRepository } from "../src/actions/transfer.ts";
13 import type { ForgeEvent } from "../src/core/events.ts";
14 import type { Repo } from "../src/core/repos.ts";
15 import { createToken } from "../src/core/tokens.ts";
16 import type { User } from "../src/core/users.ts";
17 import { onPushEvent, pointerOf, settledOf } from "../src/git/bundle_uri.ts";
18 import { repoDir } from "../src/git/repos.ts";
19 import { serve, type Servers } from "../src/http/server.ts";
20 import { dirStore } from "../src/snapshot/store.ts";
21 import { seedAdmin } from "./helpers/admin.ts";
22 import { quietLogs } from "./helpers/globals.ts";
23
24 quietLogs();
25
26 const BASE = "https://cdn.example/b";
27 const OID = "a".repeat(40);
28 const POINTER = JSON.stringify({ oid: OID, url: `${BASE}/bundles/ada/notes/${OID}.bundle` });
29
30 describe("bundle-uri through the real router", () => {
31 let dataDir: string;
32 let app: App;
33 let servers: Servers;
34 let ada: User;
35 let token: string;
36 const was = process.env["FORGE_BUNDLE_BASE_URL"];
37
38 beforeAll(async () => {
39 process.env["FORGE_BUNDLE_BASE_URL"] = BASE;
40 dataDir = await mkdtemp(join(tmpdir(), "forge-bundle-wired-"));
41 app = createApp({ dataDir, port: 0, host: "forge.example", url: "https://forge.example" });
42 servers = await serve(app);
43 const seeded = await seedAdmin(app, { username: "ada", email: "[email protected]" });
44 if (!seeded.ok) throw new Error(seeded.error);
45 ada = seeded.value;
46 token = createToken(app.db, ada.id, "t", "rw").token;
47 });
48
49 afterAll(async () => {
50 if (was === undefined) delete process.env["FORGE_BUNDLE_BASE_URL"];
51 else process.env["FORGE_BUNDLE_BASE_URL"] = was;
52 servers.stop();
53 app.close();
54 await rm(dataDir, { recursive: true, force: true });
55 });
56
57 const made = async (name: string, visibility: "public" | "private"): Promise<Repo> => {
58 const r = await createRepo(app, ada, { name, visibility });
59 if (!r.ok) throw new Error(String(r.error));
60 await commitToBranch(app, ada, r.value, { message: "seed", files: [{ path: "a.txt", content: "a\n" }] });
61 return r.value;
62 };
63 const point = (repo: Repo) => Bun.write(join(repoDir(app.reposRoot, repo.ownerName, repo.name), "forge-bundle.json"), POINTER);
64 const advert = async (name: string, service: string, auth = ""): Promise<string> => {
65 const res = await fetch(`${servers.url.replace("://", `://${auth}`)}/ada/${name}.git/info/refs?service=${service}`, {
66 headers: { "git-protocol": "version=2" },
67 });
68 expect(res.status).toBe(200);
69 return res.text();
70 };
71
72 test("a public repository with a published bundle advertises it to a fetch, and only to a fetch", async () => {
73 const repo = await made("pub", "public");
74 expect(await advert("pub", "git-upload-pack")).not.toContain("bundle-uri");
75 await point(repo);
76 expect(await advert("pub", "git-upload-pack")).toContain("bundle-uri");
77 expect(await advert("pub", "git-receive-pack", `ada:${token}@`)).not.toContain("bundle-uri");
78 });
79
80 test("a private repository never advertises, even with a pointer left in it", async () => {
81 const repo = await made("vault", "private");
82 await point(repo);
83 expect(await advert("vault", "git-upload-pack", `ada:${token}@`)).not.toContain("bundle-uri");
84 });
85
86 test("a repository made private takes its pointer back", async () => {
87 const repo = await made("turning", "public");
88 await point(repo);
89 const dir = repoDir(app.reposRoot, "ada", "turning");
90 expect(await pointerOf(dir)).toBeDefined();
91 const done = await updateRepoSettings(app, ada, repo, { visibility: "private" });
92 expect(done.ok).toBe(true);
93 expect(await pointerOf(dir)).toBeUndefined();
94 });
95
96 test("a renamed repository takes its pointer back", async () => {
97 const repo = await made("old", "public");
98 await point(repo);
99 const moved = await renameRepository(app, ada, repo, "new");
100 expect(moved.ok).toBe(true);
101 expect(await pointerOf(repoDir(app.reposRoot, "ada", "new"))).toBeUndefined();
102 });
103
104 test("a push to the default branch makes the bundle; a push to another branch does not", async () => {
105 const repo = await made("pushed", "public");
106 const bucket = await mkdtemp(join(tmpdir(), "forge-bundle-wired-bucket-"));
107 try {
108 const using = { store: dirStore(bucket), base: BASE };
109 const push = (ref: string): ForgeEvent => ({
110 id: "e",
111 kind: "push",
112 repoId: repo.id,
113 actorId: null,
114 payload: { updates: [{ ref }] },
115 createdAt: new Date().toISOString(),
116 });
117 onPushEvent(app, push("refs/heads/side"), using);
118 expect(settledOf(app).size).toBe(0);
119 onPushEvent(app, push("refs/heads/main"), using);
120 await Promise.all([...settledOf(app)]);
121 const got = await pointerOf(repoDir(app.reposRoot, "ada", "pushed"));
122 expect(got?.url.startsWith(`${BASE}/bundles/ada/pushed/`)).toBe(true);
123 expect((await readFile(join(bucket, `bundles/ada/pushed/${got!.oid}.bundle`))).length).toBeGreaterThan(0);
124 } finally {
125 await rm(bucket, { recursive: true, force: true });
126 }
127 });
128
129 test("the repository page and the terminal give the slow-link clone command once each", async () => {
130 await made("page", "public");
131 const html = await (await fetch(`${servers.url}/ada/page`)).text();
132 expect(html.match(/git clone --filter=blob:none/g)?.length).toBe(1);
133 expect(html).toContain('id="clone-slow"');
134 });
135 });
test/bundle-uri.test.ts added +217
@@ -0,0 +1,217 @@
1 /**

Wiring through the real router, onPushEvent and the repo page are not covered; the timing-sensitive first test flakes under load (findings 3, 4).

2 * The bulk of a clone from object storage (#14409). A public repository's default branch is bundled
3 * into the bucket and advertised with `bundle-uri`: a client that asks downloads the bundle from
4 * storage and the forge sends only what is newer; a client that does not ask clones as before.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtemp, readFile, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { runGit } from "../src/git/env.ts";
11 import type { App } from "../src/app.ts";
12 import { MAX_BUILDS, noteBundle, retractBundles, settledOf } from "../src/git/bundle_uri.ts";
13 import { bundleBase, bundleConfig, bundleKey, bundleUriFor, pointerOf, publishBundle } from "../src/git/bundle_uri.ts";
14 import { initBareRepo, repoDir } from "../src/git/repos.ts";
15 import { advertiseRefs, isGitService, serviceCommand, serviceRpc } from "../src/git/smart-http.ts";
16 import { dirStore } from "../src/snapshot/store.ts";
17 import { unwrap } from "../src/kernel/result.ts";
18 import { gitClientEnv } from "./helpers/git-client.ts";
19
20 async function git(args: string[], cwd: string) {
21 const proc = Bun.spawn(["git", ...args], { cwd, env: gitClientEnv(), stdout: "pipe", stderr: "pipe" });
22 const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
23 return { code, out: out.trim(), err: err.trim() };
24 }
25 async function ok(args: string[], cwd: string): Promise<string> {
26 const r = await git(args, cwd);
27 if (r.code !== 0) throw new Error(`git ${args.join(" ")}: ${r.err}`);
28 return r.out;
29 }
30
31 describe("the arguments and the address", () => {
32 test("only a fetch, only a well-formed address", () => {
33 const url = "https://cdn.example/bundles/ada/arm/" + "a".repeat(40) + ".bundle";
34 expect(serviceCommand("git-upload-pack", ["d"], true, url).join(" ")).toContain(`bundle.default.uri=${url}`);
35 expect(serviceCommand("git-upload-pack", ["d"], true, url).join(" ")).toContain("uploadpack.advertiseBundleURIs=true");
36 expect(serviceCommand("git-receive-pack", ["d"], true, url).join(" ")).not.toContain("bundle");
37 expect(serviceCommand("git-upload-pack", ["d"]).join(" ")).not.toContain("bundle");
38 expect(bundleConfig("https://x/y z")).toEqual([]);
39 expect(bundleConfig("file:///etc/passwd")).toEqual([]);
40 });
41
42 test("the feature is off until a public base address is set", () => {
43 expect(bundleBase({})).toBeUndefined();
44 expect(bundleBase({ FORGE_BUNDLE_BASE_URL: "ftp://x" })).toBeUndefined();
45 expect(bundleBase({ FORGE_BUNDLE_BASE_URL: " https://cdn.example/b/ " })).toBe("https://cdn.example/b");
46 });
47 });
48
49 describe("a clone from the bundle", () => {
50 let root: string;
51 let work: string;
52 let bucketDir: string;
53 let server: ReturnType<typeof Bun.serve>;
54 let forge: string;
55 let storage: string;
56 let packBytes = 0;
57 let tip: string;
58 const dir = () => repoDir(root, "ada", "arm");
59 const store = () => dirStore(bucketDir);
60 /** A forge to the module: its own state, and the repositories' root. */
61 const home = () => ({ reposRoot: root, db: {} as App["db"] });
62
63 beforeAll(async () => {
64 root = await mkdtemp(join(tmpdir(), "forge-bundle-"));
65 work = await mkdtemp(join(tmpdir(), "forge-bundle-work-"));
66 bucketDir = await mkdtemp(join(tmpdir(), "forge-bundle-bucket-"));
67 unwrap(await initBareRepo(dir()));
68 const seed = join(work, "seed");
69 await ok(["init", "-q", "-b", "main", seed], work);
70 const big = new Uint8Array(256 * 1024);
71 for (let at = 0; at < big.length; at += 65_536) crypto.getRandomValues(big.subarray(at, at + 65_536));
72 await Bun.write(join(seed, "base.bin"), big);
73 await ok(["add", "."], seed);
74 await ok(["commit", "-q", "-m", "one"], seed);
75 await ok(["push", "-q", dir(), "main"], seed);
76
77 server = Bun.serve({
78 port: 0,
79 async fetch(request) {
80 const url = new URL(request.url);
81 if (url.pathname.startsWith("/store/")) {
82 const file = Bun.file(join(bucketDir, decodeURIComponent(url.pathname.slice("/store/".length))));
83 return (await file.exists()) ? new Response(file) : new Response("no", { status: 404 });
84 }
85 const bundleUri = await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage });
86 if (url.pathname === "/ada/arm.git/info/refs") {
87 const service = url.searchParams.get("service");
88 if (!isGitService(service)) return new Response("no", { status: 400 });
89 return advertiseRefs(dir(), service, request, bundleUri === undefined ? {} : { bundleUri });
90 }
91 if (url.pathname === "/ada/arm.git/git-upload-pack") {
92 const res = await serviceRpc(dir(), "git-upload-pack", request, bundleUri === undefined ? {} : { bundleUri });
93 const bytes = new Uint8Array(await new Response(res.body).arrayBuffer());
94 packBytes = bytes.length;
95 return new Response(bytes, { headers: res.headers });
96 }
97 return new Response("not found", { status: 404 });
98 },
99 });
100 forge = `http://127.0.0.1:${server.port}/ada/arm.git`;
101 storage = `http://127.0.0.1:${server.port}/store`;
102 tip = unwrap(await runGit(["rev-parse", "refs/heads/main"], { cwd: dir() }));
103 });
104
105 afterAll(async () => {
106 server.stop(true);
107 for (const d of [root, work, bucketDir]) await rm(d, { recursive: true, force: true });
108 });
109
110 test("nothing is advertised before a bundle exists, and a private repository never advertises", async () => {
111 expect(await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined();
112 packBytes = 0;
113 const run = await git(["-c", "transfer.bundleURI=true", "clone", "-q", forge, "before"], work);
114 expect(run.code).toBe(0);
115 expect(packBytes).toBeGreaterThan(200_000);
116 });
117
118 test("publishing puts one bundle in the bucket and points at it; again is a no-op", async () => {
119 const made = await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" });
120 expect(made?.oid).toBe(tip);
121 expect(made?.url).toBe(`${storage}/${bundleKey("ada", "arm", tip)}`);
122 expect(await pointerOf(dir())).toEqual(made!);
123 expect(await store().list("bundles/")).toEqual([bundleKey("ada", "arm", tip)]);
124 expect((await readFile(join(bucketDir, bundleKey("ada", "arm", tip)))).length).toBeGreaterThan(200_000);
125 expect(await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" })).toEqual(made!);
126 expect(await bundleUriFor(dir(), "private", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined();
127 expect(await bundleUriFor(dir(), "public", {})).toBeUndefined();
128 });
129
130 test("a client that asks takes the bulk from storage and only the newer from the forge", async () => {
131 const seed = join(work, "seed");
132 await Bun.write(join(seed, "two.txt"), "two\n");
133 await ok(["add", "."], seed);
134 await ok(["commit", "-q", "-m", "two"], seed);
135 await ok(["push", "-q", dir(), "main"], seed);
136 packBytes = 0;
137 const run = await git(["-c", "transfer.bundleURI=true", "clone", "-q", forge, "after"], work);
138 expect(run.err).toBe("");
139 expect(run.code).toBe(0);
140 expect(packBytes).toBeGreaterThan(0);
141 expect(packBytes).toBeLessThan(10_000);
142 expect(await ok(["log", "--format=%s"], join(work, "after"))).toBe("two\none");
143 expect(await ok(["fsck"], join(work, "after")).catch((e: Error) => e.message)).not.toContain("error");
144 });
145
146 test("a client without bundle-uri still clones the whole thing from the forge", async () => {
147 packBytes = 0;
148 const run = await git(["clone", "-q", forge, "plain"], work);
149 expect(run.code).toBe(0);
150 expect(packBytes).toBeGreaterThan(200_000);
151 expect(await ok(["log", "--format=%s"], join(work, "plain"))).toBe("two\none");
152 });
153
154 test("a push to the default branch makes the bundle current; the older one is kept, older still go", async () => {
155 const app1 = home();
156 const first = (await pointerOf(dir()))!;
157 noteBundle(app1, { id: "r1", ownerName: "ada", name: "arm", visibility: "public", defaultBranch: "main" }, { store: store(), base: storage });
158 await Promise.all([...settledOf(app1)]);
159 const now = (await pointerOf(dir()))!;
160 expect(now.oid).not.toBe(first.oid);
161 expect((await store().list("bundles/")).sort()).toEqual([bundleKey("ada", "arm", first.oid), bundleKey("ada", "arm", now.oid)].sort());
162 // a private repository is never bundled
163 const other = home();
164 noteBundle(other, { id: "r2", ownerName: "ada", name: "arm", visibility: "private", defaultBranch: "main" }, { store: store(), base: storage });
165 expect(settledOf(other).size).toBe(0);
166 });
167
168 test("a repository taken back leaves nothing in the bucket and no pointer", async () => {
169 await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" });
170 expect((await store().list("bundles/ada/arm/")).length).toBeGreaterThan(0);
171 await store().put("bundles/ada/armor/keep.bundle", new Uint8Array(1));
172 await retractBundles(home(), { id: "r1", ownerName: "ada", name: "arm" }, { store: store() });
173 expect(await store().list("bundles/ada/arm/")).toEqual([]);
174 expect(await store().list("bundles/ada/armor/")).toEqual(["bundles/ada/armor/keep.bundle"]);
175 expect(await pointerOf(dir())).toBeUndefined();
176 expect(await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined();
177 });
178
179 test("after a force-push the bundle before it is not kept; a fast-forward keeps it", async () => {
180 const seed = join(work, "seed");
181 await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" });
182 const old = (await pointerOf(dir()))!;
183 await ok(["commit", "-q", "--amend", "-m", "rewritten"], seed);
184 await ok(["push", "-q", "-f", dir(), "main"], seed);
185 const now = (await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }))!;
186 expect(await store().list("bundles/ada/arm/")).toEqual([bundleKey("ada", "arm", now.oid)]);
187 expect(now.oid).not.toBe(old.oid);
188 await Bun.write(join(seed, "three.txt"), "three\n");
189 await ok(["add", "."], seed);
190 await ok(["commit", "-q", "-m", "three"], seed);
191 await ok(["push", "-q", dir(), "main"], seed);
192 const next = (await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }))!;
193 expect((await store().list("bundles/ada/arm/")).sort()).toEqual(
194 [bundleKey("ada", "arm", now.oid), bundleKey("ada", "arm", next.oid)].sort(),
195 );
196 });
197
198 test("builds in flight are capped for the whole forge; a refused one is not marked tried", async () => {
199 const gated = store();
200 let open!: () => void;
201 const gate = new Promise<void>((r) => (open = r));
202 const slow = { ...gated, put: async (k: string, b: Uint8Array) => (await gate, gated.put(k, b)) } as typeof gated;
203 const app = home();
204 const repoOf = (n: number) => ({ id: `cap${n}`, ownerName: "ada", name: "arm", visibility: "public" as const, defaultBranch: "main" });
205 // A stale pointer is not needed: each build starts, and waits at its put.
206 await ok(["commit", "-q", "--allow-empty", "-m", "four"], join(work, "seed"));
207 await ok(["push", "-q", dir(), "main"], join(work, "seed"));
208 for (let n = 0; n <= MAX_BUILDS; n++) noteBundle(app, repoOf(n), { store: slow, base: storage, now: 1 });
209 expect(settledOf(app).size).toBe(MAX_BUILDS);
210 open();
211 await Promise.all([...settledOf(app)]);
212 // The refused one is tried at once the cap is free, not a minute later.
213 noteBundle(app, repoOf(MAX_BUILDS), { store: slow, base: storage, now: 2 });
214 expect(settledOf(app).size).toBe(1);
215 await Promise.all([...settledOf(app)]);
216 });
217 });
test/ssh-clone.test.ts +2 −2
@@ -128,7 +128,7 @@ describe("the clone address on the repository page, in the terminal and in the p
128128 test("with sshPort 0 neither shows an ssh line", async () => {
129129 app.config.sshPort = 0;
130130 expect(await page()).not.toContain('id="clone-ssh"');
131 expect(terminal().map(([k]) => k)).toEqual(["clone (http)"]);
131 expect(terminal().map(([k]) => k)).toEqual(["clone (http)", "clone (slow link)"]);
132132 expect(await pinned()).toBeNull();
133133 });
134134
@@ -147,7 +147,7 @@ describe("the clone address on the repository page, in the terminal and in the p
147147 app.config.appSsh = "ssh -p 2222 [email protected]";
148148 try {
149149 expect(await page()).not.toContain('id="clone-ssh"');
150 expect(terminal().map(([k]) => k)).toEqual(["clone (http)"]);
150 expect(terminal().map(([k]) => k)).toEqual(["clone (http)", "clone (slow link)"]);
151151 expect(await pinned()).toBeNull();
152152 } finally {
153153 delete app.config.appSsh;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs