Public: anyone can read this; the crew can change it.
Cloning from outside is slow enough to fail: serve the bulk of a clone from object storage, not the home uplink #14410
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14409-cloning-from-outside-is-slow-enough-to-f → main 5 commits fork 3 threads · 3 unresolved
14 files
changed +682
−14
docs/features/clone-from-storage.md added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | --- | |
| 2 | topic: Your first app in five minutes | |
| 3 | row: Clones from storage | |
| 4 | --- | |
| 5 | # Clones from storage | |
| 6 | ||
| 7 | A clone left through the home uplink, which is shared and slow, and a big pack cut partway failed with `early EOF` (#14409). | |
| 8 | ||
| 9 | - **What the forge does** (`src/git/bundle_uri.ts`): for each public repository it publishes a git bundle of the default branch under `bundles/<owner>/<name>/<commit>.bundle` in the bucket the forge backs up to (`S3_*`), and advertises it with git's `bundle-uri` capability. It is made after a push to the default branch and on a clone's ref advertisement when missing or behind, one build per repository, none more often than a minute, and two at once at most across the forge. The newest and the one before it are kept, the one before only while it is part of the new history (a force-push drops it). | |
| 10 | - **Taking it back**: a repository that goes private, is renamed, moved or deleted has every object under its `bundles/<owner>/<name>/` deleted from the bucket and its `forge-bundle.json` removed first, since the bucket is publicly readable. | |
| 11 | - **To turn it on**: set `FORGE_BUNDLE_BASE_URL` to the address the bucket (or a CDN in front of it) is read from without credentials, and make `bundles/` publicly readable there. Unset, nothing is published or advertised. A private repository is never bundled or advertised. | |
| 12 | - **For a client** (git 2.38+): `git -c transfer.bundleURI=true clone <url>` downloads the bundle from storage, then fetches only what is newer from the forge. git does not ask for bundles unless told to; a client that does not clones whole from the forge as before. | |
| 13 | - **On a slow link without that**: `git clone --filter=blob:none <url>`, one sentence and a copy field on the repository page, and a `clone (slow link)` line in the terminal's repository About. | |
| 14 | - Not done: the uplink shaper's priority for git over model traffic is host configuration (`docs/features/bottleneck.md`), not in this repository; a bundle is the default branch only, so other branches come from the forge; a private repository's bundle would need signed addresses. |
src/actions/accounts.ts +2
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | * and scrubs the account; the row stays so the name stays taken. |
| 11 | 11 | */ |
| 12 | 12 | |
| 13 | import { retractBundles } from "../git/bundle_uri.ts"; | |
| 13 | 14 | import { removeDrop } from "../drop/store.ts"; |
| 14 | 15 | import type { Database } from "bun:sqlite"; |
| 15 | 16 | import type { App } from "../app.ts"; |
| @@ -185,6 +186,7 @@ async function removeOwnedRepos( | ||
| 185 | 186 | for (const repo of listReposOwnedBy(app.db, owner.id)) { |
| 186 | 187 | if (runtime !== undefined) await runtime.teardownAll(repo.id); |
| 187 | 188 | const label = `${owner.username}/${repo.name}`; |
| 189 | await retractBundles(app, repo); | |
| 188 | 190 | const deleted = await destroyRepo(app.db, app.reposRoot, repo); |
| 189 | 191 | if (!deleted.ok) return label; |
| 190 | 192 | await removeInstallCache(app.config.dataDir, repo.id); |
src/actions/claim.ts +2
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | * repository that will not move stays with the agent and is reported; when |
| 9 | 9 | * none moves the link is given back. |
| 10 | 10 | */ |
| 11 | import { retractBundles } from "../git/bundle_uri.ts"; | |
| 11 | 12 | import type { App } from "../app.ts"; |
| 12 | 13 | import { setSuspended } from "../core/accounts.ts"; |
| 13 | 14 | import { forgetListings } from "../core/listings.ts"; |
| @@ -70,6 +71,7 @@ export async function claimStart( | ||
| 70 | 71 | const moved: Repo[] = []; |
| 71 | 72 | const stayed: string[] = []; |
| 72 | 73 | for (const repo of listReposOwnedBy(app.db, agent.id)) { |
| 74 | await retractBundles(app, repo); | |
| 73 | 75 | const done = await transferRepo(app.db, app.reposRoot, repo, org.value, claimer.id); |
| 74 | 76 | if (done.ok) { |
| 75 | 77 | moved.push(done.value); |
src/actions/repos.ts +5
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * write path — committing files to a branch without a git client. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { retractBundles } from "../git/bundle_uri.ts"; | |
| 7 | 8 | import { removeDrop } from "../drop/store.ts"; |
| 8 | 9 | import { getOriginById } from "../core/origins.ts"; |
| 9 | 10 | import { isContributionFork, type GuestRefusal } from "../core/contribute.ts"; |
| @@ -456,6 +457,9 @@ export async function updateRepoSettings( | ||
| 456 | 457 | } |
| 457 | 458 | const { archived, ...rest } = settings; |
| 458 | 459 | writeSettings(app.db, repo, rest); |
| 460 | // A bundle sits in a publicly readable bucket: a repository that stops being public takes it back | |
| 461 | // (#14409). | |
| 462 | if (repo.visibility === "public" && settings.visibility === "private") await retractBundles(app, repo); | |
| 459 | 463 | const moved = |
| 460 | 464 | archived === undefined ? false : await moveArchive(app, repo.id, archived); |
| 461 | 465 | const updated = getRepoById(app.db, repo.id); |
| @@ -587,6 +591,7 @@ export async function deleteRepo( | ||
| 587 | 591 | ? ok(actor) |
| 588 | 592 | : requireRepoAdmin(app, actor, repo); |
| 589 | 593 | if (!admin.ok) return admin; |
| 594 | await retractBundles(app, repo); | |
| 590 | 595 | const deleted = await destroyRepo(app.db, app.reposRoot, repo); |
| 591 | 596 | if (!deleted.ok) return deleted; |
| 592 | 597 | await removeInstallCache(app.config.dataDir, repo.id); |
src/actions/transfer.ts +3
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | * where. |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | import { retractBundles } from "../git/bundle_uri.ts"; | |
| 11 | 12 | import type { App } from "../app.ts"; |
| 12 | 13 | import { getUserByName, type User } from "../core/users.ts"; |
| 13 | 14 | import { canAdmin, type Repo } from "../core/repos.ts"; |
| @@ -50,6 +51,7 @@ export async function transferRepository( | ||
| 50 | 51 | if (target === undefined || target.kind === "agent") return err("unknown_owner"); |
| 51 | 52 | const ownerId = resolveOwner(app, actor, target.username); |
| 52 | 53 | if (ownerId !== target.id && !actor.isAdmin) return err("forbidden_owner"); |
| 54 | await retractBundles(app, repo); | |
| 53 | 55 | const moved = await transferRepo(app.db, app.reposRoot, repo, target, actor.id); |
| 54 | 56 | if (!moved.ok) return moved; |
| 55 | 57 | storageChanged(app, repo.ownerId); |
| @@ -80,6 +82,7 @@ export async function renameRepository( | ||
| 80 | 82 | newName: string, |
| 81 | 83 | ): Promise<Result<Repo, TransferActionError>> { |
| 82 | 84 | if (actor === undefined || actor.kind === "agent" || !canAdmin(app.db, repo, actor)) return err("not_found"); |
| 85 | await retractBundles(app, repo); | |
| 83 | 86 | const moved = await renameRepo(app.db, app.reposRoot, repo, newName, actor.id); |
| 84 | 87 | if (!moved.ok) return moved; |
| 85 | 88 | app.bus.append({ |
src/app.ts +3
| @@ -34,6 +34,7 @@ import { | ||
| 34 | 34 | import { RefusedLedger } from "./kernel/internal_refused.ts"; |
| 35 | 35 | import { closeReadersUnder } from "./git/read.ts"; |
| 36 | 36 | import { dropAllPins } from "./git/pins_kept.ts"; |
| 37 | import { onPushEvent } from "./git/bundle_uri.ts"; | |
| 37 | 38 | |
| 38 | 39 | export interface ForgeConfig { |
| 39 | 40 | /** The engine shape this boot chose and why (`engine/mode.ts`), for `/system`; none in a test. */ |
| @@ -296,6 +297,8 @@ export function createApp(config: ForgeConfig): App { | ||
| 296 | 297 | // sends them. Listener throws are swallowed by the bus, so this can never |
| 297 | 298 | // break the action that produced the event. |
| 298 | 299 | bus.subscribe((event) => enqueueForEvent(db, event), "local"); |
| 300 | // A push to a public repository's default branch makes its clone bundle again (#14409). | |
| 301 | bus.subscribe((event) => onPushEvent({ db, reposRoot }, event), "local"); | |
| 299 | 302 | // A pull opened or merged, a release served, an issue dropped or split: |
| 300 | 303 | // estimates settle (#11955). |
| 301 | 304 | bus.subscribe((event) => settleEstimates(db, event), "local"); |
src/git/bundle_uri.ts added +251
| @@ -0,0 +1,251 @@ | ||
| 1 | /** | |
| 2 | * The bulk of a clone from object storage, not the home uplink (#14409). | |
| 3 | * | |
| 4 | * A public repository's default branch is published as one git bundle to the | |
| 5 | * bucket the forge already backs up to (`S3_*`), and upload-pack is told to | |
| 6 | * advertise it with git's `bundle-uri` capability (protocol v2, git 2.38+). A | |
| 7 | * client that asks (`git -c transfer.bundleURI=true clone`) downloads the | |
| 8 | * bundle from `FORGE_BUNDLE_BASE_URL` (the bucket's public read address or a | |
| 9 | * CDN in front of it) and then fetches only what is newer from the forge. A | |
| 10 | * client without bundle-uri never sees the capability and clones as before. | |
| 11 | * | |
| 12 | * State: one small file per repository, `<repo>/forge-bundle.json` | |
| 13 | * (`{ oid, url }`: the commit the bundle holds and where it is), written only | |
| 14 | * after the upload succeeded. Nothing is advertised for a repository that is | |
| 15 | * not public, whatever the file says. | |
| 16 | * | |
| 17 | * When one is made again: after a push to the default branch, and on a clone's ref advertisement | |
| 18 | * when the pointer is missing or behind. One build per repository at a time, and none tried twice | |
| 19 | * within `COOLDOWN_MS`, so a busy branch or a bucket that refuses costs one bundle a minute, never | |
| 20 | * a pile. Held in memory: the ids building and when each was last tried, at most the repositories | |
| 21 | * touched in the last minute (pruned); gone on restart, and the next clone or push asks again. | |
| 22 | */ | |
| 23 | ||
| 24 | import { readFile, rename, rm, writeFile } from "node:fs/promises"; | |
| 25 | import { join } from "node:path"; | |
| 26 | import { runGit, runGitRaw } from "./env.ts"; | |
| 27 | import type { App } from "../app.ts"; | |
| 28 | import type { ForgeEvent } from "../core/events.ts"; | |
| 29 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 30 | import { createLog } from "../kernel/log.ts"; | |
| 31 | import { repoDir } from "./repos.ts"; | |
| 32 | import { s3FromEnv, s3Store, type Store } from "../snapshot/store.ts"; | |
| 33 | ||
| 34 | export interface Published { | |
| 35 | readonly oid: string; | |
| 36 | readonly url: string; | |
| 37 | } | |
| 38 | ||
| 39 | /** The most a bundle may be in memory while it is built and sent: 512 MiB. */ | |
| 40 | const BUNDLE_MAX = 512 * 1024 * 1024; | |
| 41 | const POINTER = "forge-bundle.json"; | |
| 42 | const OID = /^[0-9a-f]{40}([0-9a-f]{24})?$/; | |
| 43 | const URL_SHAPE = /^https?:\/\/[^\s"'\\]{1,400}$/; | |
| 44 | ||
| 45 | /** The public address the bucket is read from, or undefined (the feature is off). */ | |
| 46 | export function bundleBase(env: Record<string, string | undefined> = process.env): string | undefined { | |
| 47 | const raw = env["FORGE_BUNDLE_BASE_URL"]?.trim().replace(/\/+$/, ""); | |
| 48 | if (raw === undefined || !/^https?:\/\/[^\s"'\\]{1,200}$/.test(raw)) return undefined; | |
| 49 | return raw; | |
| 50 | } | |
| 51 | ||
| 52 | /** | |
| 53 | * The object key of a repository's bundle at one commit. Names are validated elsewhere; encoded | |
| 54 | * again here. | |
| 55 | */ | |
| 56 | export const bundleKey = (owner: string, name: string, oid: string): string => | |
| 57 | `bundles/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/${oid}.bundle`; | |
| 58 | ||
| 59 | /** | |
| 60 | * What a repository's pointer file holds, or undefined when it is absent or not what it should be. | |
| 61 | */ | |
| 62 | export async function pointerOf(dir: string): Promise<Published | undefined> { | |
| 63 | try { | |
| 64 | const v: unknown = JSON.parse(await readFile(join(dir, POINTER), "utf8")); | |
| 65 | if (typeof v !== "object" || v === null) return undefined; | |
| 66 | const { oid, url } = v as Record<string, unknown>; | |
| 67 | return typeof oid === "string" && OID.test(oid) && typeof url === "string" && URL_SHAPE.test(url) | |
| 68 | ? { oid, url } | |
| 69 | : undefined; | |
| 70 | } catch { | |
| 71 | return undefined; | |
| 72 | } | |
| 73 | } | |
| 74 | ||
| 75 | /** The `git -c` arguments that make upload-pack advertise `url` as the clone's first download. */ | |
| 76 | export function bundleConfig(url: string | undefined): string[] { | |
| 77 | if (url === undefined || !URL_SHAPE.test(url)) return []; | |
| 78 | return [ | |
| 79 | "-c", "uploadpack.advertiseBundleURIs=true", | |
| 80 | "-c", "bundle.version=1", | |
| 81 | "-c", "bundle.mode=all", | |
| 82 | "-c", `bundle.default.uri=${url}`, | |
| 83 | ]; | |
| 84 | } | |
| 85 | ||
| 86 | /** | |
| 87 | * The address to advertise for a repository: its published bundle if it is public and the feature | |
| 88 | * is on. | |
| 89 | */ | |
| 90 | export async function bundleUriFor( | |
| 91 | dir: string, | |
| 92 | visibility: "public" | "private", | |
| 93 | env: Record<string, string | undefined> = process.env, | |
| 94 | ): Promise<string | undefined> { | |
| 95 | if (visibility !== "public" || bundleBase(env) === undefined) return undefined; | |
| 96 | return (await pointerOf(dir))?.url; | |
| 97 | } | |
| 98 | ||
| 99 | /** | |
| 100 | * Bundle `branch` and put it in `store`, then point at it. Nothing is done when the pointer already | |
| 101 | * names the branch's tip. The one older bundle is kept for a client mid-download; the rest go. | |
| 102 | * Returns what is published, or undefined when there was nothing to publish. | |
| 103 | */ | |
| 104 | export async function publishBundle(input: { | |
| 105 | readonly store: Store; | |
| 106 | readonly base: string; | |
| 107 | readonly dir: string; | |
| 108 | readonly owner: string; | |
| 109 | readonly name: string; | |
| 110 | readonly branch: string; | |
| 111 | }): Promise<Published | undefined> { | |
| 112 | const { store, base, dir, owner, name, branch } = input; | |
| 113 | const ref = `refs/heads/${branch}`; | |
| 114 | const tip = await runGit(["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd: dir }); | |
| 115 | if (!tip.ok || !OID.test(tip.value)) return undefined; | |
| 116 | const before = await pointerOf(dir); | |
| 117 | if (before?.oid === tip.value) return before; | |
| 118 | ||
| 119 | // The bundle goes to memory, not a file: nothing is left behind to clean up, and | |
| 120 | // `BUNDLE_MAX` bounds it. | |
| 121 | const made = await runGitRaw(["bundle", "create", "-", ref], { | |
| 122 | cwd: dir, | |
| 123 | timeoutMs: 600_000, | |
| 124 | maxBytes: BUNDLE_MAX, | |
| 125 | }); | |
| 126 | if (!made.ok) throw new Error(`git bundle: ${made.error.stderr.slice(0, 200)}`); | |
| 127 | const key = bundleKey(owner, name, tip.value); | |
| 128 | await store.put(key, made.value); | |
| 129 | const now: Published = { oid: tip.value, url: `${base}/${key}` }; | |
| 130 | const tmp = join(dir, `${POINTER}.tmp`); | |
| 131 | await writeFile(tmp, JSON.stringify(now)); | |
| 132 | await rename(tmp, join(dir, POINTER)); | |
| 133 | // The one before is kept for a client mid-download, but only while it is part of the new history: | |
| 134 | // after a force-push it holds what the push meant to remove. | |
| 135 | const kept = | |
| 136 | before !== undefined && | |
| 137 | (await runGit(["merge-base", "--is-ancestor", before.oid, tip.value], { cwd: dir })).ok; | |
| 138 | const keep = new Set([key, kept && before !== undefined ? bundleKey(owner, name, before.oid) : ""]); | |
| 139 | for (const old of await store.list(`bundles/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/`)) | |
| 140 | if (!keep.has(old)) await store.delete(old).catch(() => {}); | |
| 141 | return now; | |
| 142 | } | |
| 143 | ||
| 144 | // ---- when ---- | |
| 145 | ||
| 146 | const log = createLog("bundle"); | |
| 147 | export const COOLDOWN_MS = 60_000; | |
| 148 | /** | |
| 149 | * Builds in flight at once in one forge, whatever the repositories: each holds up to `BUNDLE_MAX` | |
| 150 | * in memory. | |
| 151 | */ | |
| 152 | export const MAX_BUILDS = 2; | |
| 153 | ||
| 154 | interface Held { | |
| 155 | readonly building: Map<string, Promise<void>>; | |
| 156 | readonly triedAt: Map<string, number>; | |
| 157 | /** Settled builds, for tests to wait on. */ | |
| 158 | readonly settled: Set<Promise<void>>; | |
| 159 | } | |
| 160 | const held = new WeakMap<object, Held>(); | |
| 161 | const heldBy = (app: Pick<App, "db">): Held => { | |
| 162 | let h = held.get(app.db); | |
| 163 | if (h === undefined) held.set(app.db, (h = { building: new Map(), triedAt: new Map(), settled: new Set() })); | |
| 164 | return h; | |
| 165 | }; | |
| 166 | let store: Store | undefined; | |
| 167 | ||
| 168 | /** The builds in flight for this forge, for tests to wait on. */ | |
| 169 | export const settledOf = (app: Pick<App, "db">): ReadonlySet<Promise<void>> => heldBy(app).settled; | |
| 170 | ||
| 171 | function bucket(): Store | undefined { | |
| 172 | const target = s3FromEnv(); | |
| 173 | if (target === undefined) return undefined; | |
| 174 | return (store ??= s3Store(target, 2)); | |
| 175 | } | |
| 176 | ||
| 177 | /** Make this repository's bundle current in the background; a no-op when it cannot or need not. */ | |
| 178 | export function noteBundle( | |
| 179 | app: Pick<App, "db" | "reposRoot">, | |
| 180 | repo: Pick<Repo, "id" | "ownerName" | "name" | "visibility" | "defaultBranch">, | |
| 181 | using: { store?: Store; base?: string; now?: number } = {}, | |
| 182 | ): void { | |
| 183 | const base = using.base ?? bundleBase(); | |
| 184 | const to = using.store ?? bucket(); | |
| 185 | if (base === undefined || to === undefined || repo.visibility !== "public") return; | |
| 186 | const now = using.now ?? Date.now(); | |
| 187 | const { building, triedAt, settled } = heldBy(app); | |
| 188 | if (building.size >= MAX_BUILDS || building.has(repo.id) || now - (triedAt.get(repo.id) ?? -Infinity) < COOLDOWN_MS) return; | |
| 189 | for (const [id, at] of triedAt) if (now - at >= COOLDOWN_MS) triedAt.delete(id); | |
| 190 | triedAt.set(repo.id, now); | |
| 191 | const job = publishBundle({ | |
| 192 | store: to, | |
| 193 | base, | |
| 194 | dir: repoDir(app.reposRoot, repo.ownerName, repo.name), | |
| 195 | owner: repo.ownerName, | |
| 196 | name: repo.name, | |
| 197 | branch: repo.defaultBranch, | |
| 198 | }) | |
| 199 | .then(() => {}) | |
| 200 | .catch((error: unknown) => { | |
| 201 | log.warn("bundle not published", { repo: `${repo.ownerName}/${repo.name}`, why: String(error).slice(0, 200) }); | |
| 202 | }) | |
| 203 | .finally(() => { | |
| 204 | building.delete(repo.id); | |
| 205 | settled.delete(job); | |
| 206 | }); | |
| 207 | building.set(repo.id, job); | |
| 208 | settled.add(job); | |
| 209 | } | |
| 210 | ||
| 211 | /** | |
| 212 | * Take a repository's bundles back: before it goes private, is renamed, moved or deleted, every | |
| 213 | * object under its prefix is deleted from the bucket (it is publicly readable and the address is | |
| 214 | * guessable) and the pointer is removed. A build in flight is waited for first, so it cannot put | |
| 215 | * one back. A bucket that refuses is logged; the pointer is gone either way, so nothing is | |
| 216 | * advertised. | |
| 217 | */ | |
| 218 | export async function retractBundles( | |
| 219 | app: Pick<App, "db" | "reposRoot">, | |
| 220 | repo: Pick<Repo, "id" | "ownerName" | "name">, | |
| 221 | using: { store?: Store } = {}, | |
| 222 | ): Promise<void> { | |
| 223 | await heldBy(app).building.get(repo.id); | |
| 224 | await rm(join(repoDir(app.reposRoot, repo.ownerName, repo.name), POINTER), { force: true }); | |
| 225 | heldBy(app).triedAt.delete(repo.id); | |
| 226 | const to = using.store ?? bucket(); | |
| 227 | if (to === undefined) return; | |
| 228 | const prefix = `bundles/${encodeURIComponent(repo.ownerName)}/${encodeURIComponent(repo.name)}/`; | |
| 229 | try { | |
| 230 | for (const key of await to.list(prefix)) await to.delete(key); | |
| 231 | } catch (error) { | |
| 232 | log.warn("bundles not retracted", { repo: `${repo.ownerName}/${repo.name}`, why: String(error).slice(0, 200) }); | |
| 233 | } | |
| 234 | } | |
| 235 | ||
| 236 | /** A push that moved the default branch makes the bundle again. */ | |
| 237 | export function onPushEvent( | |
| 238 | app: Pick<App, "db" | "reposRoot">, | |
| 239 | event: ForgeEvent, | |
| 240 | using: { store?: Store; base?: string } = {}, | |
| 241 | ): void { | |
| 242 | if (event.kind !== "push" || event.repoId === null) return; | |
| 243 | const updates = event.payload["updates"]; | |
| 244 | if (!Array.isArray(updates)) return; | |
| 245 | const repo = getRepoById(app.db, event.repoId); | |
| 246 | if (!repo) return; | |
| 247 | const moved = updates.some( | |
| 248 | (u: unknown) => typeof u === "object" && u !== null && (u as { ref?: unknown }).ref === `refs/heads/${repo.defaultBranch}`, | |
| 249 | ); | |
| 250 | if (moved) noteBundle(app, repo, using); | |
| 251 | } |
src/git/smart-http.ts +13 −2
| @@ -19,6 +19,7 @@ import { gitEnv, transferGate } from "./env.ts"; | ||
| 19 | 19 | import { pullPipe, type PullPipe } from "./pull-pipe.ts"; |
| 20 | 20 | import { transferDeadline, type Stall } from "./progress.ts"; |
| 21 | 21 | import { pacer } from "./pace.ts"; |
| 22 | import { bundleConfig } from "./bundle_uri.ts"; | |
| 22 | 23 | import { OWN_REFS_ONLY, reachableFromOwnRefs, refusalLine, wantsOf } from "./own_refs.ts"; |
| 23 | 24 | |
| 24 | 25 | export type GitService = "git-upload-pack" | "git-receive-pack"; |
| @@ -67,12 +68,15 @@ export function serviceCommand( | ||
| 67 | 68 | service: GitService, |
| 68 | 69 | args: readonly string[], |
| 69 | 70 | pushOptions = true, |
| 71 | bundleUri?: string, | |
| 70 | 72 | ): string[] { |
| 71 | 73 | // "git-upload-pack" the service is "upload-pack" the subcommand. |
| 72 | 74 | const subcommand = service.slice(4); |
| 73 | 75 | const config = |
| 74 | 76 | service === "git-upload-pack" ? UPLOAD_PACK_CONFIG : pushOptions ? RECEIVE_PACK_CONFIG : []; |
| 75 | return ["git", ...config, subcommand, ...args]; | |
| 77 | // A published bundle is advertised to a fetch only (`bundle_uri.ts`). | |
| 78 | const bundle = service === "git-upload-pack" ? bundleConfig(bundleUri) : []; | |
| 79 | return ["git", ...config, ...bundle, subcommand, ...args]; | |
| 76 | 80 | } |
| 77 | 81 | |
| 78 | 82 | export interface SmartHttpOptions { |
| @@ -93,6 +97,8 @@ export interface SmartHttpOptions { | ||
| 93 | 97 | idleMs?: number; |
| 94 | 98 | /** Offer push options (`git push -o`); a mirror's push never takes them. @default true */ |
| 95 | 99 | pushOptions?: boolean; |
| 100 | /** Where this repository's published bundle is, advertised as `bundle-uri` (`bundle_uri.ts`). */ | |
| 101 | bundleUri?: string; | |
| 96 | 102 | /** Given back when the transfer ends, however it ends (a stranger's share). */ |
| 97 | 103 | release?: () => void; |
| 98 | 104 | /** |
| @@ -241,7 +247,12 @@ async function spawnService( | ||
| 241 | 247 | try { |
| 242 | 248 | pipe = await pullPipe(() => proc.kill("SIGKILL")); |
| 243 | 249 | proc = Bun.spawn( |
| 244 | serviceCommand(service, ["--stateless-rpc", ...extraArgs, repoDir], options.pushOptions), | |
| 250 | serviceCommand( | |
| 251 | service, | |
| 252 | ["--stateless-rpc", ...extraArgs, repoDir], | |
| 253 | options.pushOptions, | |
| 254 | held ? undefined : options.bundleUri, | |
| 255 | ), | |
| 245 | 256 | { |
| 246 | 257 | env: gitEnv(env), |
| 247 | 258 | stdin: "pipe", |
src/http/server.ts +30 −9
| @@ -125,6 +125,7 @@ import { PUSH_HOLDS_SECRET, pushHolds } from "../crew/leak.ts"; | ||
| 125 | 125 | import { heldLongForms } from "../kernel/log.ts"; |
| 126 | 126 | import { asReader, heldPushConfig, pushCarriesWhatItNames, SEND_WHAT_YOU_NAME } from "../git/own_refs.ts"; |
| 127 | 127 | import { holdFor, lendersBeyond } from "../core/fork_reach.ts"; |
| 128 | import { bundleUriFor, noteBundle } from "../git/bundle_uri.ts"; | |
| 128 | 129 | import { |
| 129 | 130 | advertiseRefs, |
| 130 | 131 | isGitService, |
| @@ -1326,24 +1327,31 @@ export function buildRouter(): Router<Handler> { | ||
| 1326 | 1327 | const service = ctx.url.searchParams.get("service"); |
| 1327 | 1328 | if (!isGitService(service)) |
| 1328 | 1329 | return new Response("smart http only", { status: 400 }); |
| 1329 | return withGitAccess(ctx, service, (dir, _repoId, _cap, held) => | |
| 1330 | asStranger(ctx, (options) => | |
| 1330 | return withGitAccess(ctx, service, async (dir, _repoId, _cap, held) => { | |
| 1331 | const bundleUri = await bundleUriOf(ctx, dir, service); | |
| 1332 | return asStranger(ctx, (options) => | |
| 1331 | 1333 | advertiseRefs(dir, service, ctx.request, { |
| 1332 | 1334 | ...options, |
| 1335 | ...(bundleUri === undefined ? {} : { bundleUri }), | |
| 1333 | 1336 | // A caller who may not read what this fork borrows from (`git/own_refs.ts`). |
| 1334 | 1337 | ownRefsOnly: held && service === "git-upload-pack", |
| 1335 | 1338 | ...(held && service === "git-receive-pack" ? { env: heldPushConfig({}) } : {}), |
| 1336 | 1339 | pushOptions: getRepo(ctx.app.db, ctx.params.owner!, ctx.params.name!)?.origin === null, |
| 1337 | 1340 | }), |
| 1338 | ), | |
| 1339 | ); | |
| 1341 | ); | |
| 1342 | }); | |
| 1340 | 1343 | }); |
| 1341 | 1344 | router.add("POST", "/:owner/:name.git/git-upload-pack", (ctx) => |
| 1342 | withGitAccess(ctx, "git-upload-pack", (dir, _repoId, _cap, held) => | |
| 1343 | asStranger(ctx, (options) => | |
| 1344 | serviceRpc(dir, "git-upload-pack", ctx.request, { ...options, ownRefsOnly: held }), | |
| 1345 | ), | |
| 1346 | ), | |
| 1345 | withGitAccess(ctx, "git-upload-pack", async (dir, _repoId, _cap, held) => { | |
| 1346 | const bundleUri = await bundleUriOf(ctx, dir, "git-upload-pack"); | |
| 1347 | return asStranger(ctx, (options) => | |
| 1348 | serviceRpc(dir, "git-upload-pack", ctx.request, { | |
| 1349 | ...options, | |
| 1350 | ownRefsOnly: held, | |
| 1351 | ...(bundleUri === undefined ? {} : { bundleUri }), | |
| 1352 | }), | |
| 1353 | ); | |
| 1354 | }), | |
| 1347 | 1355 | ); |
| 1348 | 1356 | router.add("POST", "/:owner/:name.git/git-receive-pack", (ctx) => |
| 1349 | 1357 | withGitAccess(ctx, "git-receive-pack", (dir, repoId, cap) => |
| @@ -2023,6 +2031,19 @@ async function withGitAccess( | ||
| 2023 | 2031 | : served; |
| 2024 | 2032 | } |
| 2025 | 2033 | |
| 2034 | /** | |
| 2035 | * The published bundle to advertise to this fetch (`git/bundle_uri.ts`), if the repository is | |
| 2036 | * public and has one; a fetch's ref advertisement also asks for the bundle to be made or brought up | |
| 2037 | * to date. | |
| 2038 | */ | |
| 2039 | async function bundleUriOf(ctx: Ctx, dir: string, service: GitService): Promise<string | undefined> { | |
| 2040 | if (service !== "git-upload-pack") return undefined; | |
| 2041 | const repo = getRepo(ctx.app.db, ctx.params.owner!, ctx.params.name!); | |
| 2042 | if (!repo) return undefined; | |
| 2043 | if (ctx.request.method === "GET") noteBundle(ctx.app, repo); | |
| 2044 | return bundleUriFor(dir, repo.visibility); | |
| 2045 | } | |
| 2046 | ||
| 2026 | 2047 | /** `body`, unchanged, then `done` once it has ended or been given up. */ |
| 2027 | 2048 | function afterBody(body: ReadableStream<Uint8Array>, done: () => void): ReadableStream<Uint8Array> { |
| 2028 | 2049 | const reader = body.getReader(); |
src/tui/screens/about.ts +3 −1
| @@ -994,8 +994,10 @@ export function cloneFacts(f: Session, repo: Repo): [string, string][] { | ||
| 994 | 994 | config.url !== undefined |
| 995 | 995 | ? new URL(config.url).origin |
| 996 | 996 | : `http://${config.host ?? "localhost"}:${config.port ?? 3000}`; |
| 997 | const url = `${origin}/${repo.ownerName}/${repo.name}.git`; | |
| 997 | 998 | const out: [string, string][] = [ |
| 998 | ["clone (http)", `${origin}/${repo.ownerName}/${repo.name}.git`], | |
| 999 | ["clone (http)", url], | |
| 1000 | ["clone (slow link)", `git clone --filter=blob:none ${url}`], | |
| 999 | 1001 | ]; |
| 1000 | 1002 | const ssh = sshCloneUrl(f.app, repo.ownerName, repo.name); |
| 1001 | 1003 | if (ssh !== undefined) out.push(["clone (ssh)", ssh]); |
src/web/pages/repo.ts +2
| @@ -1013,6 +1013,8 @@ function aboutCard( | ||
| 1013 | 1013 | )} |
| 1014 | 1014 | <p class="mt small muted">Clone over HTTPS</p> |
| 1015 | 1015 | ${copyField("clone-https", cloneUrl, "HTTPS clone address", "Copy clone URL")} |
| 1016 | <p class="mt small muted">On a slow link, clone without the file contents.</p> | |
outdated
Principles 1 and 8: hand-rolled paragraph instead of the copyField component, and a second explanation of cloning. Also missing from the terminal screen and untested (findings 5, 3). |
||
| 1017 | ${copyField("clone-slow", `git clone --filter=blob:none ${cloneUrl}`, "Clone command for a slow link", "Copy slow-link clone command")} | |
| 1016 | 1018 | ${when( |
| 1017 | 1019 | sshUrl, |
| 1018 | 1020 | (url) => |
test/bundle-uri-wired.test.ts added +135
| @@ -0,0 +1,135 @@ | ||
| 1 | /** | |
| 2 | * The bundle's wiring through the real router and the real actions (#14409): what a clone's ref | |
| 3 | * advertisement says, what a push does, and what a repository that stops being public (or is | |
| 4 | * renamed) leaves behind. `bundle-uri.test.ts` holds the module itself. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, readFile, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { commitToBranch, createRepo, updateRepoSettings } from "../src/actions/repos.ts"; | |
| 12 | import { renameRepository } from "../src/actions/transfer.ts"; | |
| 13 | import type { ForgeEvent } from "../src/core/events.ts"; | |
| 14 | import type { Repo } from "../src/core/repos.ts"; | |
| 15 | import { createToken } from "../src/core/tokens.ts"; | |
| 16 | import type { User } from "../src/core/users.ts"; | |
| 17 | import { onPushEvent, pointerOf, settledOf } from "../src/git/bundle_uri.ts"; | |
| 18 | import { repoDir } from "../src/git/repos.ts"; | |
| 19 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 20 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 21 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 22 | import { quietLogs } from "./helpers/globals.ts"; | |
| 23 | ||
| 24 | quietLogs(); | |
| 25 | ||
| 26 | const BASE = "https://cdn.example/b"; | |
| 27 | const OID = "a".repeat(40); | |
| 28 | const POINTER = JSON.stringify({ oid: OID, url: `${BASE}/bundles/ada/notes/${OID}.bundle` }); | |
| 29 | ||
| 30 | describe("bundle-uri through the real router", () => { | |
| 31 | let dataDir: string; | |
| 32 | let app: App; | |
| 33 | let servers: Servers; | |
| 34 | let ada: User; | |
| 35 | let token: string; | |
| 36 | const was = process.env["FORGE_BUNDLE_BASE_URL"]; | |
| 37 | ||
| 38 | beforeAll(async () => { | |
| 39 | process.env["FORGE_BUNDLE_BASE_URL"] = BASE; | |
| 40 | dataDir = await mkdtemp(join(tmpdir(), "forge-bundle-wired-")); | |
| 41 | app = createApp({ dataDir, port: 0, host: "forge.example", url: "https://forge.example" }); | |
| 42 | servers = await serve(app); | |
| 43 | const seeded = await seedAdmin(app, { username: "ada", email: "[email protected]" }); | |
| 44 | if (!seeded.ok) throw new Error(seeded.error); | |
| 45 | ada = seeded.value; | |
| 46 | token = createToken(app.db, ada.id, "t", "rw").token; | |
| 47 | }); | |
| 48 | ||
| 49 | afterAll(async () => { | |
| 50 | if (was === undefined) delete process.env["FORGE_BUNDLE_BASE_URL"]; | |
| 51 | else process.env["FORGE_BUNDLE_BASE_URL"] = was; | |
| 52 | servers.stop(); | |
| 53 | app.close(); | |
| 54 | await rm(dataDir, { recursive: true, force: true }); | |
| 55 | }); | |
| 56 | ||
| 57 | const made = async (name: string, visibility: "public" | "private"): Promise<Repo> => { | |
| 58 | const r = await createRepo(app, ada, { name, visibility }); | |
| 59 | if (!r.ok) throw new Error(String(r.error)); | |
| 60 | await commitToBranch(app, ada, r.value, { message: "seed", files: [{ path: "a.txt", content: "a\n" }] }); | |
| 61 | return r.value; | |
| 62 | }; | |
| 63 | const point = (repo: Repo) => Bun.write(join(repoDir(app.reposRoot, repo.ownerName, repo.name), "forge-bundle.json"), POINTER); | |
| 64 | const advert = async (name: string, service: string, auth = ""): Promise<string> => { | |
| 65 | const res = await fetch(`${servers.url.replace("://", `://${auth}`)}/ada/${name}.git/info/refs?service=${service}`, { | |
| 66 | headers: { "git-protocol": "version=2" }, | |
| 67 | }); | |
| 68 | expect(res.status).toBe(200); | |
| 69 | return res.text(); | |
| 70 | }; | |
| 71 | ||
| 72 | test("a public repository with a published bundle advertises it to a fetch, and only to a fetch", async () => { | |
| 73 | const repo = await made("pub", "public"); | |
| 74 | expect(await advert("pub", "git-upload-pack")).not.toContain("bundle-uri"); | |
| 75 | await point(repo); | |
| 76 | expect(await advert("pub", "git-upload-pack")).toContain("bundle-uri"); | |
| 77 | expect(await advert("pub", "git-receive-pack", `ada:${token}@`)).not.toContain("bundle-uri"); | |
| 78 | }); | |
| 79 | ||
| 80 | test("a private repository never advertises, even with a pointer left in it", async () => { | |
| 81 | const repo = await made("vault", "private"); | |
| 82 | await point(repo); | |
| 83 | expect(await advert("vault", "git-upload-pack", `ada:${token}@`)).not.toContain("bundle-uri"); | |
| 84 | }); | |
| 85 | ||
| 86 | test("a repository made private takes its pointer back", async () => { | |
| 87 | const repo = await made("turning", "public"); | |
| 88 | await point(repo); | |
| 89 | const dir = repoDir(app.reposRoot, "ada", "turning"); | |
| 90 | expect(await pointerOf(dir)).toBeDefined(); | |
| 91 | const done = await updateRepoSettings(app, ada, repo, { visibility: "private" }); | |
| 92 | expect(done.ok).toBe(true); | |
| 93 | expect(await pointerOf(dir)).toBeUndefined(); | |
| 94 | }); | |
| 95 | ||
| 96 | test("a renamed repository takes its pointer back", async () => { | |
| 97 | const repo = await made("old", "public"); | |
| 98 | await point(repo); | |
| 99 | const moved = await renameRepository(app, ada, repo, "new"); | |
| 100 | expect(moved.ok).toBe(true); | |
| 101 | expect(await pointerOf(repoDir(app.reposRoot, "ada", "new"))).toBeUndefined(); | |
| 102 | }); | |
| 103 | ||
| 104 | test("a push to the default branch makes the bundle; a push to another branch does not", async () => { | |
| 105 | const repo = await made("pushed", "public"); | |
| 106 | const bucket = await mkdtemp(join(tmpdir(), "forge-bundle-wired-bucket-")); | |
| 107 | try { | |
| 108 | const using = { store: dirStore(bucket), base: BASE }; | |
| 109 | const push = (ref: string): ForgeEvent => ({ | |
| 110 | id: "e", | |
| 111 | kind: "push", | |
| 112 | repoId: repo.id, | |
| 113 | actorId: null, | |
| 114 | payload: { updates: [{ ref }] }, | |
| 115 | createdAt: new Date().toISOString(), | |
| 116 | }); | |
| 117 | onPushEvent(app, push("refs/heads/side"), using); | |
| 118 | expect(settledOf(app).size).toBe(0); | |
| 119 | onPushEvent(app, push("refs/heads/main"), using); | |
| 120 | await Promise.all([...settledOf(app)]); | |
| 121 | const got = await pointerOf(repoDir(app.reposRoot, "ada", "pushed")); | |
| 122 | expect(got?.url.startsWith(`${BASE}/bundles/ada/pushed/`)).toBe(true); | |
| 123 | expect((await readFile(join(bucket, `bundles/ada/pushed/${got!.oid}.bundle`))).length).toBeGreaterThan(0); | |
| 124 | } finally { | |
| 125 | await rm(bucket, { recursive: true, force: true }); | |
| 126 | } | |
| 127 | }); | |
| 128 | ||
| 129 | test("the repository page and the terminal give the slow-link clone command once each", async () => { | |
| 130 | await made("page", "public"); | |
| 131 | const html = await (await fetch(`${servers.url}/ada/page`)).text(); | |
| 132 | expect(html.match(/git clone --filter=blob:none/g)?.length).toBe(1); | |
| 133 | expect(html).toContain('id="clone-slow"'); | |
| 134 | }); | |
| 135 | }); |
test/bundle-uri.test.ts added +217
| @@ -0,0 +1,217 @@ | ||
| 1 | /** | |
Wiring through the real router, onPushEvent and the repo page are not covered; the timing-sensitive first test flakes under load (findings 3, 4). |
||
| 2 | * The bulk of a clone from object storage (#14409). A public repository's default branch is bundled | |
| 3 | * into the bucket and advertised with `bundle-uri`: a client that asks downloads the bundle from | |
| 4 | * storage and the forge sends only what is newer; a client that does not ask clones as before. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, readFile, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { runGit } from "../src/git/env.ts"; | |
| 11 | import type { App } from "../src/app.ts"; | |
| 12 | import { MAX_BUILDS, noteBundle, retractBundles, settledOf } from "../src/git/bundle_uri.ts"; | |
| 13 | import { bundleBase, bundleConfig, bundleKey, bundleUriFor, pointerOf, publishBundle } from "../src/git/bundle_uri.ts"; | |
| 14 | import { initBareRepo, repoDir } from "../src/git/repos.ts"; | |
| 15 | import { advertiseRefs, isGitService, serviceCommand, serviceRpc } from "../src/git/smart-http.ts"; | |
| 16 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 17 | import { unwrap } from "../src/kernel/result.ts"; | |
| 18 | import { gitClientEnv } from "./helpers/git-client.ts"; | |
| 19 | ||
| 20 | async function git(args: string[], cwd: string) { | |
| 21 | const proc = Bun.spawn(["git", ...args], { cwd, env: gitClientEnv(), stdout: "pipe", stderr: "pipe" }); | |
| 22 | const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); | |
| 23 | return { code, out: out.trim(), err: err.trim() }; | |
| 24 | } | |
| 25 | async function ok(args: string[], cwd: string): Promise<string> { | |
| 26 | const r = await git(args, cwd); | |
| 27 | if (r.code !== 0) throw new Error(`git ${args.join(" ")}: ${r.err}`); | |
| 28 | return r.out; | |
| 29 | } | |
| 30 | ||
| 31 | describe("the arguments and the address", () => { | |
| 32 | test("only a fetch, only a well-formed address", () => { | |
| 33 | const url = "https://cdn.example/bundles/ada/arm/" + "a".repeat(40) + ".bundle"; | |
| 34 | expect(serviceCommand("git-upload-pack", ["d"], true, url).join(" ")).toContain(`bundle.default.uri=${url}`); | |
| 35 | expect(serviceCommand("git-upload-pack", ["d"], true, url).join(" ")).toContain("uploadpack.advertiseBundleURIs=true"); | |
| 36 | expect(serviceCommand("git-receive-pack", ["d"], true, url).join(" ")).not.toContain("bundle"); | |
| 37 | expect(serviceCommand("git-upload-pack", ["d"]).join(" ")).not.toContain("bundle"); | |
| 38 | expect(bundleConfig("https://x/y z")).toEqual([]); | |
| 39 | expect(bundleConfig("file:///etc/passwd")).toEqual([]); | |
| 40 | }); | |
| 41 | ||
| 42 | test("the feature is off until a public base address is set", () => { | |
| 43 | expect(bundleBase({})).toBeUndefined(); | |
| 44 | expect(bundleBase({ FORGE_BUNDLE_BASE_URL: "ftp://x" })).toBeUndefined(); | |
| 45 | expect(bundleBase({ FORGE_BUNDLE_BASE_URL: " https://cdn.example/b/ " })).toBe("https://cdn.example/b"); | |
| 46 | }); | |
| 47 | }); | |
| 48 | ||
| 49 | describe("a clone from the bundle", () => { | |
| 50 | let root: string; | |
| 51 | let work: string; | |
| 52 | let bucketDir: string; | |
| 53 | let server: ReturnType<typeof Bun.serve>; | |
| 54 | let forge: string; | |
| 55 | let storage: string; | |
| 56 | let packBytes = 0; | |
| 57 | let tip: string; | |
| 58 | const dir = () => repoDir(root, "ada", "arm"); | |
| 59 | const store = () => dirStore(bucketDir); | |
| 60 | /** A forge to the module: its own state, and the repositories' root. */ | |
| 61 | const home = () => ({ reposRoot: root, db: {} as App["db"] }); | |
| 62 | ||
| 63 | beforeAll(async () => { | |
| 64 | root = await mkdtemp(join(tmpdir(), "forge-bundle-")); | |
| 65 | work = await mkdtemp(join(tmpdir(), "forge-bundle-work-")); | |
| 66 | bucketDir = await mkdtemp(join(tmpdir(), "forge-bundle-bucket-")); | |
| 67 | unwrap(await initBareRepo(dir())); | |
| 68 | const seed = join(work, "seed"); | |
| 69 | await ok(["init", "-q", "-b", "main", seed], work); | |
| 70 | const big = new Uint8Array(256 * 1024); | |
| 71 | for (let at = 0; at < big.length; at += 65_536) crypto.getRandomValues(big.subarray(at, at + 65_536)); | |
| 72 | await Bun.write(join(seed, "base.bin"), big); | |
| 73 | await ok(["add", "."], seed); | |
| 74 | await ok(["commit", "-q", "-m", "one"], seed); | |
| 75 | await ok(["push", "-q", dir(), "main"], seed); | |
| 76 | ||
| 77 | server = Bun.serve({ | |
| 78 | port: 0, | |
| 79 | async fetch(request) { | |
| 80 | const url = new URL(request.url); | |
| 81 | if (url.pathname.startsWith("/store/")) { | |
| 82 | const file = Bun.file(join(bucketDir, decodeURIComponent(url.pathname.slice("/store/".length)))); | |
| 83 | return (await file.exists()) ? new Response(file) : new Response("no", { status: 404 }); | |
| 84 | } | |
| 85 | const bundleUri = await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage }); | |
| 86 | if (url.pathname === "/ada/arm.git/info/refs") { | |
| 87 | const service = url.searchParams.get("service"); | |
| 88 | if (!isGitService(service)) return new Response("no", { status: 400 }); | |
| 89 | return advertiseRefs(dir(), service, request, bundleUri === undefined ? {} : { bundleUri }); | |
| 90 | } | |
| 91 | if (url.pathname === "/ada/arm.git/git-upload-pack") { | |
| 92 | const res = await serviceRpc(dir(), "git-upload-pack", request, bundleUri === undefined ? {} : { bundleUri }); | |
| 93 | const bytes = new Uint8Array(await new Response(res.body).arrayBuffer()); | |
| 94 | packBytes = bytes.length; | |
| 95 | return new Response(bytes, { headers: res.headers }); | |
| 96 | } | |
| 97 | return new Response("not found", { status: 404 }); | |
| 98 | }, | |
| 99 | }); | |
| 100 | forge = `http://127.0.0.1:${server.port}/ada/arm.git`; | |
| 101 | storage = `http://127.0.0.1:${server.port}/store`; | |
| 102 | tip = unwrap(await runGit(["rev-parse", "refs/heads/main"], { cwd: dir() })); | |
| 103 | }); | |
| 104 | ||
| 105 | afterAll(async () => { | |
| 106 | server.stop(true); | |
| 107 | for (const d of [root, work, bucketDir]) await rm(d, { recursive: true, force: true }); | |
| 108 | }); | |
| 109 | ||
| 110 | test("nothing is advertised before a bundle exists, and a private repository never advertises", async () => { | |
| 111 | expect(await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined(); | |
| 112 | packBytes = 0; | |
| 113 | const run = await git(["-c", "transfer.bundleURI=true", "clone", "-q", forge, "before"], work); | |
| 114 | expect(run.code).toBe(0); | |
| 115 | expect(packBytes).toBeGreaterThan(200_000); | |
| 116 | }); | |
| 117 | ||
| 118 | test("publishing puts one bundle in the bucket and points at it; again is a no-op", async () => { | |
| 119 | const made = await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }); | |
| 120 | expect(made?.oid).toBe(tip); | |
| 121 | expect(made?.url).toBe(`${storage}/${bundleKey("ada", "arm", tip)}`); | |
| 122 | expect(await pointerOf(dir())).toEqual(made!); | |
| 123 | expect(await store().list("bundles/")).toEqual([bundleKey("ada", "arm", tip)]); | |
| 124 | expect((await readFile(join(bucketDir, bundleKey("ada", "arm", tip)))).length).toBeGreaterThan(200_000); | |
| 125 | expect(await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" })).toEqual(made!); | |
| 126 | expect(await bundleUriFor(dir(), "private", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined(); | |
| 127 | expect(await bundleUriFor(dir(), "public", {})).toBeUndefined(); | |
| 128 | }); | |
| 129 | ||
| 130 | test("a client that asks takes the bulk from storage and only the newer from the forge", async () => { | |
| 131 | const seed = join(work, "seed"); | |
| 132 | await Bun.write(join(seed, "two.txt"), "two\n"); | |
| 133 | await ok(["add", "."], seed); | |
| 134 | await ok(["commit", "-q", "-m", "two"], seed); | |
| 135 | await ok(["push", "-q", dir(), "main"], seed); | |
| 136 | packBytes = 0; | |
| 137 | const run = await git(["-c", "transfer.bundleURI=true", "clone", "-q", forge, "after"], work); | |
| 138 | expect(run.err).toBe(""); | |
| 139 | expect(run.code).toBe(0); | |
| 140 | expect(packBytes).toBeGreaterThan(0); | |
| 141 | expect(packBytes).toBeLessThan(10_000); | |
| 142 | expect(await ok(["log", "--format=%s"], join(work, "after"))).toBe("two\none"); | |
| 143 | expect(await ok(["fsck"], join(work, "after")).catch((e: Error) => e.message)).not.toContain("error"); | |
| 144 | }); | |
| 145 | ||
| 146 | test("a client without bundle-uri still clones the whole thing from the forge", async () => { | |
| 147 | packBytes = 0; | |
| 148 | const run = await git(["clone", "-q", forge, "plain"], work); | |
| 149 | expect(run.code).toBe(0); | |
| 150 | expect(packBytes).toBeGreaterThan(200_000); | |
| 151 | expect(await ok(["log", "--format=%s"], join(work, "plain"))).toBe("two\none"); | |
| 152 | }); | |
| 153 | ||
| 154 | test("a push to the default branch makes the bundle current; the older one is kept, older still go", async () => { | |
| 155 | const app1 = home(); | |
| 156 | const first = (await pointerOf(dir()))!; | |
| 157 | noteBundle(app1, { id: "r1", ownerName: "ada", name: "arm", visibility: "public", defaultBranch: "main" }, { store: store(), base: storage }); | |
| 158 | await Promise.all([...settledOf(app1)]); | |
| 159 | const now = (await pointerOf(dir()))!; | |
| 160 | expect(now.oid).not.toBe(first.oid); | |
| 161 | expect((await store().list("bundles/")).sort()).toEqual([bundleKey("ada", "arm", first.oid), bundleKey("ada", "arm", now.oid)].sort()); | |
| 162 | // a private repository is never bundled | |
| 163 | const other = home(); | |
| 164 | noteBundle(other, { id: "r2", ownerName: "ada", name: "arm", visibility: "private", defaultBranch: "main" }, { store: store(), base: storage }); | |
| 165 | expect(settledOf(other).size).toBe(0); | |
| 166 | }); | |
| 167 | ||
| 168 | test("a repository taken back leaves nothing in the bucket and no pointer", async () => { | |
| 169 | await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }); | |
| 170 | expect((await store().list("bundles/ada/arm/")).length).toBeGreaterThan(0); | |
| 171 | await store().put("bundles/ada/armor/keep.bundle", new Uint8Array(1)); | |
| 172 | await retractBundles(home(), { id: "r1", ownerName: "ada", name: "arm" }, { store: store() }); | |
| 173 | expect(await store().list("bundles/ada/arm/")).toEqual([]); | |
| 174 | expect(await store().list("bundles/ada/armor/")).toEqual(["bundles/ada/armor/keep.bundle"]); | |
| 175 | expect(await pointerOf(dir())).toBeUndefined(); | |
| 176 | expect(await bundleUriFor(dir(), "public", { FORGE_BUNDLE_BASE_URL: storage })).toBeUndefined(); | |
| 177 | }); | |
| 178 | ||
| 179 | test("after a force-push the bundle before it is not kept; a fast-forward keeps it", async () => { | |
| 180 | const seed = join(work, "seed"); | |
| 181 | await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }); | |
| 182 | const old = (await pointerOf(dir()))!; | |
| 183 | await ok(["commit", "-q", "--amend", "-m", "rewritten"], seed); | |
| 184 | await ok(["push", "-q", "-f", dir(), "main"], seed); | |
| 185 | const now = (await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }))!; | |
| 186 | expect(await store().list("bundles/ada/arm/")).toEqual([bundleKey("ada", "arm", now.oid)]); | |
| 187 | expect(now.oid).not.toBe(old.oid); | |
| 188 | await Bun.write(join(seed, "three.txt"), "three\n"); | |
| 189 | await ok(["add", "."], seed); | |
| 190 | await ok(["commit", "-q", "-m", "three"], seed); | |
| 191 | await ok(["push", "-q", dir(), "main"], seed); | |
| 192 | const next = (await publishBundle({ store: store(), base: storage, dir: dir(), owner: "ada", name: "arm", branch: "main" }))!; | |
| 193 | expect((await store().list("bundles/ada/arm/")).sort()).toEqual( | |
| 194 | [bundleKey("ada", "arm", now.oid), bundleKey("ada", "arm", next.oid)].sort(), | |
| 195 | ); | |
| 196 | }); | |
| 197 | ||
| 198 | test("builds in flight are capped for the whole forge; a refused one is not marked tried", async () => { | |
| 199 | const gated = store(); | |
| 200 | let open!: () => void; | |
| 201 | const gate = new Promise<void>((r) => (open = r)); | |
| 202 | const slow = { ...gated, put: async (k: string, b: Uint8Array) => (await gate, gated.put(k, b)) } as typeof gated; | |
| 203 | const app = home(); | |
| 204 | const repoOf = (n: number) => ({ id: `cap${n}`, ownerName: "ada", name: "arm", visibility: "public" as const, defaultBranch: "main" }); | |
| 205 | // A stale pointer is not needed: each build starts, and waits at its put. | |
| 206 | await ok(["commit", "-q", "--allow-empty", "-m", "four"], join(work, "seed")); | |
| 207 | await ok(["push", "-q", dir(), "main"], join(work, "seed")); | |
| 208 | for (let n = 0; n <= MAX_BUILDS; n++) noteBundle(app, repoOf(n), { store: slow, base: storage, now: 1 }); | |
| 209 | expect(settledOf(app).size).toBe(MAX_BUILDS); | |
| 210 | open(); | |
| 211 | await Promise.all([...settledOf(app)]); | |
| 212 | // The refused one is tried at once the cap is free, not a minute later. | |
| 213 | noteBundle(app, repoOf(MAX_BUILDS), { store: slow, base: storage, now: 2 }); | |
| 214 | expect(settledOf(app).size).toBe(1); | |
| 215 | await Promise.all([...settledOf(app)]); | |
| 216 | }); | |
| 217 | }); |
test/ssh-clone.test.ts +2 −2
| @@ -128,7 +128,7 @@ describe("the clone address on the repository page, in the terminal and in the p | ||
| 128 | 128 | test("with sshPort 0 neither shows an ssh line", async () => { |
| 129 | 129 | app.config.sshPort = 0; |
| 130 | 130 | expect(await page()).not.toContain('id="clone-ssh"'); |
| 131 | expect(terminal().map(([k]) => k)).toEqual(["clone (http)"]); | |
| 131 | expect(terminal().map(([k]) => k)).toEqual(["clone (http)", "clone (slow link)"]); | |
| 132 | 132 | expect(await pinned()).toBeNull(); |
| 133 | 133 | }); |
| 134 | 134 | |
| @@ -147,7 +147,7 @@ describe("the clone address on the repository page, in the terminal and in the p | ||
| 147 | 147 | app.config.appSsh = "ssh -p 2222 [email protected]"; |
| 148 | 148 | try { |
| 149 | 149 | expect(await page()).not.toContain('id="clone-ssh"'); |
| 150 | expect(terminal().map(([k]) => k)).toEqual(["clone (http)"]); | |
| 150 | expect(terminal().map(([k]) => k)).toEqual(["clone (http)", "clone (slow link)"]); | |
| 151 | 151 | expect(await pinned()).toBeNull(); |
| 152 | 152 | } finally { |
| 153 | 153 | delete app.config.appSsh; |
new above
offline
Anonymous GET triggers noteBundle for any public repo (see finding 2): needs a global concurrency cap. No test goes through this route (finding 3).