Public: anyone can read this; the crew can change it.
Contained checks and gates install from the repository's shared bun cache, filled by the forge and bound read-only into the cell #7549
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/7543-contained-checks-and-gates-install-from → main 4 commits fork 5 threads · 5 unresolved
5 files
changed +425
−9
src/checks/install.ts +129 −1
| @@ -16,7 +16,8 @@ | ||
| 16 | 16 | * the check again and the merge queue reads it as cut off, never as red. |
| 17 | 17 | */ |
| 18 | 18 | |
| 19 | import { lstatSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 19 | import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 20 | import { tmpdir } from "node:os"; | |
| 20 | 21 | import { dirname, join } from "node:path"; |
| 21 | 22 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 22 | 23 | |
| @@ -48,6 +49,133 @@ export function checkInstallCache(dataDir: string, repoId: string): string { | ||
| 48 | 49 | return join(dataDir, "check-install", repoId.replace(/[^A-Za-z0-9_-]/g, "_"), "bun"); |
| 49 | 50 | } |
| 50 | 51 | |
| 52 | /** The registry a fill fetches from unless the forge's own environment names another. */ | |
| 53 | export const DEFAULT_REGISTRY = "https://registry.npmjs.org/"; | |
| 54 | ||
| 55 | /** | |
| 56 | * A contained `bun install` links nothing from the read-only cache: hardlinks | |
| 57 | * cannot cross a bind onto another device, so it copies instead. Applied to | |
| 58 | * every contained install, on purpose: whether cache and cell share a device | |
| 59 | * is the host's layout, not something to probe per check, and copying a warm | |
| 60 | * cache costs milliseconds against a network install. Every `bun install` in | |
| 61 | * the command is asked, so `cd x && bun install` copies too. | |
| 62 | */ | |
| 63 | export function copying(command: string): string { | |
| 64 | return command.replace(/\bbun install\b(?![^&;|]*--backend)/g, "bun install --backend=copyfile"); | |
| 65 | } | |
| 66 | ||
| 67 | /** What a contained install runs: the command, and the cache env it reads. */ | |
| 68 | export function cellInstall( | |
| 69 | command: string, | |
| 70 | env: Record<string, string>, | |
| 71 | warm: string | undefined, | |
| 72 | ): { readonly command: string; readonly env: Record<string, string> } { | |
| 73 | return warm === undefined | |
| 74 | ? { command, env } | |
| 75 | : { command: copying(command), env: { ...env, BUN_INSTALL_CACHE_DIR: warm } }; | |
| 76 | } | |
| 77 | ||
| 78 | /** | |
| 79 | * How long a fill may run: never past half the lock's staleness, or another | |
| 80 | * process would take the lock mid-fill and two fills would write the cache. | |
| 81 | */ | |
| 82 | export function fillTimeoutMs(requested: number | undefined): number { | |
| 83 | return Math.min(requested ?? Number.POSITIVE_INFINITY, LOCK_STALE_MS / 2); | |
| 84 | } | |
| 85 | ||
| 86 | /** | |
| 87 | * Why a lockfile may not fill the shared cache, or undefined when it may: the | |
| 88 | * fill runs outside a cell, so it fetches only from the pinned registry. A | |
| 89 | * binary lockfile cannot be read, and a URL elsewhere or a git dependency | |
| 90 | * would make the forge fetch from a host the pull chose. | |
| 91 | */ | |
| 92 | export function lockfileRefusal(dir: string, registry: string): string | undefined { | |
| 93 | if (existsSync(join(dir, "bun.lockb")) && !existsSync(join(dir, "bun.lock"))) | |
| 94 | return "only a binary bun.lockb, which cannot be checked against the registry"; | |
| 95 | if (!existsSync(join(dir, "bun.lock"))) return "no bun lockfile in the checkout"; | |
| 96 | const text = readFileSync(join(dir, "bun.lock"), "utf8"); | |
| 97 | for (const url of text.match(/[a-z+]+:\/\/[^"\s]+/gi) ?? []) | |
| 98 | if (!url.startsWith(registry)) return `the lockfile names ${url.slice(0, 80)}, not the registry`; | |
outdated
This runs bun on the host with the lockfile's URLs. |
||
| 99 | if (/[@"'/](?:github|gitlab|bitbucket|git|ssh|git\+\w+):/i.test(text)) | |
| 100 | return "the lockfile has a git dependency"; | |
| 101 | return undefined; | |
| 102 | } | |
| 103 | ||
| 104 | /** | |
| 105 | * Fill the shared cache on the host before a contained install (#7543): the | |
| 106 | * cell binds the cache read-only, so only the forge writes it. Installs the | |
| 107 | * commit's lockfile into a scratch copy of the checkout (no `node_modules`, | |
| 108 | * no `.git`, and no `bunfig.toml` or `.npmrc`: the pull does not choose the | |
| 109 | * registry) with `--frozen-lockfile --ignore-scripts`, so no lifecycle script | |
| 110 | * runs outside a cell and the only host fetched is the pinned registry | |
| 111 | * (`BUN_CONFIG_REGISTRY` of `env`, else {@link DEFAULT_REGISTRY}). Holds the | |
| 112 | * same lock uncontained installs hold. A package the forge cannot fetch fails | |
| 113 | * here, with the install's own output, and releases the lock. | |
| 114 | * | |
| 115 | * The cell reads the cache without the lock while a later fill may write | |
| 116 | * it. That is safe because bun writes each package to a temp name in the | |
| 117 | * cache and renames it into place, so a reader sees a whole entry or none, | |
| 118 | * and a package the lockfile names was filled before this cell started. | |
| 119 | * | |
| 120 | * Answers whether it filled. False (a lockfile it will not trust, or none) is | |
| 121 | * not a failure: the cell then installs into a cache of its own. | |
| 122 | */ | |
| 123 | export async function prefillInstallCache(options: { | |
| 124 | readonly cache: string; | |
| 125 | readonly dir: string; | |
| 126 | readonly env: Record<string, string>; | |
| 127 | readonly log: (line: string) => void; | |
| 128 | readonly signal: AbortSignal; | |
| 129 | readonly timeoutMs?: number | undefined; | |
| 130 | }): Promise<Result<boolean, string>> { | |
| 131 | const { cache, dir, log, signal } = options; | |
| 132 | const registry = (options.env["BUN_CONFIG_REGISTRY"] ?? DEFAULT_REGISTRY).replace(/\/?$/, "/"); | |
| 133 | const refusal = lockfileRefusal(dir, registry); | |
| 134 | if (refusal !== undefined) { | |
| 135 | log(`cache not filled: ${refusal}`); | |
| 136 | return ok(false); | |
| 137 | } | |
| 138 | mkdirSync(cache, { recursive: true }); | |
| 139 | const held = await withLock( | |
| 140 | `${cache}.lock`, | |
| 141 | async (): Promise<Result<boolean, string>> => { | |
| 142 | const scratch = mkdtempSync(join(tmpdir(), "prefill-")); | |
| 143 | try { | |
| 144 | cpSync(dir, scratch, { | |
| 145 | recursive: true, | |
| 146 | filter: (path) => !/(^|\/)(node_modules|\.git|bunfig\.toml|\.npmrc)$/.test(path), | |
| 147 | }); | |
| 148 | const proc = Bun.spawn(["bun", "install", "--frozen-lockfile", "--ignore-scripts"], { | |
| 149 | cwd: scratch, | |
| 150 | env: { ...options.env, BUN_CONFIG_REGISTRY: registry, BUN_INSTALL_CACHE_DIR: cache }, | |
| 151 | stdin: "ignore", | |
| 152 | stdout: "pipe", | |
| 153 | stderr: "pipe", | |
| 154 | }); | |
| 155 | const stop = (): void => proc.kill(); | |
| 156 | signal.addEventListener("abort", stop, { once: true }); | |
| 157 | const timer = setTimeout(stop, fillTimeoutMs(options.timeoutMs)); | |
| 158 | const lines = async (stream: ReadableStream<Uint8Array>): Promise<void> => { | |
| 159 | const text = await new Response(stream).text(); | |
| 160 | for (const line of text.split("\n")) if (line.trim() !== "") log(line); | |
| 161 | }; | |
| 162 | try { | |
| 163 | await Promise.all([lines(proc.stdout), lines(proc.stderr)]); | |
| 164 | const code = await proc.exited; | |
| 165 | return code === 0 ? ok(true) : err(`filling the install cache exited ${code}`); | |
| 166 | } finally { | |
| 167 | clearTimeout(timer); | |
| 168 | signal.removeEventListener("abort", stop); | |
| 169 | } | |
| 170 | } finally { | |
| 171 | rmSync(scratch, { recursive: true, force: true }); | |
| 172 | } | |
| 173 | }, | |
| 174 | { signal }, | |
| 175 | ); | |
| 176 | return held.ok ? held.value : held; | |
| 177 | } | |
| 178 | ||
| 51 | 179 | /** |
| 52 | 180 | * Run `attempt` (one install, its output lines handed to `log`), and once |
| 53 | 181 | * more after `retryMs` when it failed with a {@link TRANSIENT_INSTALL} line. |
src/checks/service.ts +32 −6
| @@ -95,7 +95,7 @@ import { summariseCheckFailure } from "./summary.ts"; | ||
| 95 | 95 | import type { Result } from "../kernel/result.ts"; |
| 96 | 96 | import { placementFromEnv, type Placement } from "./placement.ts"; |
| 97 | 97 | import { JOB_VANISHED, runJobStep } from "./job_step.ts"; |
| 98 | import { checkInstallCache, installWithRetry, withLock } from "./install.ts"; | |
| 98 | import { cellInstall, checkInstallCache, installWithRetry, prefillInstallCache, withLock } from "./install.ts"; | |
| 99 | 99 | import { checkStage, type GateStage } from "./gate_reason.ts"; |
| 100 | 100 | import { extraGates, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts"; |
| 101 | 101 | import type { JobOptions } from "../cluster/jobs.ts"; |
| @@ -995,18 +995,44 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 995 | 995 | // cache is shared, so installs into it take turns (`checks/install.ts`). |
| 996 | 996 | const shared = |
| 997 | 997 | cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined; |
| 998 | const once = (log: (line: string) => void): Promise<Result<void, string>> => | |
| 999 | runStep("install", install, { | |
| 998 | // A cell installs from that same cache, bound read-only (#7543): | |
| 999 | // the forge fills it first, on the host, so no pull writes what | |
| 1000 | // the gate on main installs. Cluster jobs (`runJobStep`) are | |
| 1001 | // unchanged: a Job has its own cache and never takes this path. | |
| 1002 | const warm = cell === undefined ? undefined : checkInstallCache(app.config.dataDir, repo.id); | |
| 1003 | // Set once the fill has run: a cell installs from the shared cache | |
| 1004 | // only when the forge filled it; otherwise it keeps a cache of its own. | |
| 1005 | let filled = false; | |
| 1006 | const once = (log: (line: string) => void): Promise<Result<void, string>> => { | |
| 1007 | const base = shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared }; | |
| 1008 | const from = cellInstall(install, base, filled ? warm : undefined); | |
| 1009 | const revealed = cell !== undefined && filled && warm !== undefined ? { ...cell, reveal: [warm] } : cell; | |
| 1010 | return runStep("install", from.command, { | |
| 1000 | 1011 | cwd: dir, |
| 1001 | env: shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared }, | |
| 1012 | env: from.env, | |
| 1002 | 1013 | log, |
| 1003 | 1014 | signal, |
| 1004 | 1015 | timeoutMs: manifest.timeout * 1000, |
| 1005 | cell, | |
| 1016 | cell: revealed, | |
| 1006 | 1017 | }); |
outdated
With no lockfile the fill is skipped, then |
||
| 1018 | }; | |
| 1007 | 1019 | const installed = await installWithRetry( |
| 1008 | 1020 | async (log) => { |
| 1009 | if (shared === undefined) return once(log); | |
| 1021 | if (shared === undefined) { | |
| 1022 | if (warm !== undefined) { | |
| 1023 | const fill = await prefillInstallCache({ | |
| 1024 | cache: warm, | |
| 1025 | dir, | |
| 1026 | env, | |
| 1027 | log, | |
| 1028 | signal, | |
| 1029 | timeoutMs: manifest.timeout * 1000, | |
| 1030 | }); | |
| 1031 | if (!fill.ok) return fill; | |
| 1032 | filled = fill.value; | |
| 1033 | } | |
| 1034 | return once(log); | |
| 1035 | } | |
| 1010 | 1036 | mkdirSync(shared, { recursive: true }); |
| 1011 | 1037 | const held = await withLock(`${shared}.lock`, () => once(log), { signal }); |
| 1012 | 1038 | return held.ok ? held.value : held; |
test/check-install.test.ts +224 −2
| @@ -3,8 +3,8 @@ | ||
| 3 | 3 | * install failure is retried once, and failing the same way twice is the |
| 4 | 4 | * machine's — queued again, never the pull's red. |
| 5 | 5 | */ |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { existsSync, mkdirSync, writeFileSync } from "node:fs"; | |
| 6 | import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test"; | |
| 7 | import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; | |
| 8 | 8 | import { mkdtemp, rm } from "node:fs/promises"; |
| 9 | 9 | import { tmpdir } from "node:os"; |
| 10 | 10 | import { join } from "node:path"; |
| @@ -12,6 +12,11 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 12 | 12 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 13 | 13 | import { |
| 14 | 14 | checkInstallCache, |
| 15 | cellInstall, | |
| 16 | copying, | |
| 17 | fillTimeoutMs, | |
| 18 | LOCK_STALE_MS, | |
| 19 | prefillInstallCache, | |
| 15 | 20 | INFRASTRUCTURE, |
| 16 | 21 | installWithRetry, |
| 17 | 22 | transientLine, |
| @@ -228,3 +233,220 @@ describe("a check whose install fails on the machine", () => { | ||
| 228 | 233 | expect(runs.length).toBe(2); |
| 229 | 234 | }); |
| 230 | 235 | }); |
| 236 | ||
| 237 | describe("prefillInstallCache (#7543)", () => { | |
| 238 | setDefaultTimeout(60_000); | |
| 239 | const scratch = async (): Promise<string> => mkdtemp(join(tmpdir(), "prefill-test-")); | |
| 240 | ||
| 241 | /** A registry on loopback serving one package, `[email protected]`; counts its requests. */ | |
| 242 | const registry = async (): Promise<{ url: string; hits: string[]; stop: () => void; dir: string }> => { | |
| 243 | const dir = await scratch(); | |
| 244 | mkdirSync(join(dir, "pkg", "package"), { recursive: true }); | |
| 245 | writeFileSync(join(dir, "pkg", "package", "package.json"), JSON.stringify({ name: "fixture-pkg", version: "1.0.0" })); | |
| 246 | const made = Bun.spawnSync(["tar", "-czf", join(dir, "fixture.tgz"), "-C", join(dir, "pkg"), "package"]); | |
| 247 | if (made.exitCode !== 0) throw new Error("tar is needed for the fixture registry"); | |
| 248 | const tarball = new Uint8Array(await Bun.file(join(dir, "fixture.tgz")).arrayBuffer()); | |
| 249 | const hits: string[] = []; | |
| 250 | const server: ReturnType<typeof Bun.serve> = Bun.serve({ | |
| 251 | port: 0, | |
| 252 | hostname: "127.0.0.1", | |
| 253 | fetch(req): Response { | |
| 254 | const path = new URL(req.url).pathname; | |
| 255 | hits.push(path); | |
| 256 | if (path === "/fixture-pkg/-/fixture-pkg-1.0.0.tgz") return new Response(tarball); | |
| 257 | if (path === "/fixture-pkg") | |
| 258 | return Response.json({ | |
| 259 | name: "fixture-pkg", | |
| 260 | "dist-tags": { latest: "1.0.0" }, | |
| 261 | versions: { | |
| 262 | "1.0.0": { | |
outdated
This returns early when the registry is unreachable, so the test can pass while proving nothing. Use a local fixture registry. |
||
| 263 | name: "fixture-pkg", | |
| 264 | version: "1.0.0", | |
| 265 | dist: { tarball: `http://127.0.0.1:${server.port}/fixture-pkg/-/fixture-pkg-1.0.0.tgz` }, | |
| 266 | }, | |
| 267 | }, | |
| 268 | }); | |
| 269 | return new Response("not found", { status: 404 }); | |
| 270 | }, | |
| 271 | }); | |
| 272 | return { url: `http://127.0.0.1:${server.port}/`, hits, stop: () => void server.stop(true), dir }; | |
| 273 | }; | |
| 274 | ||
| 275 | /** A checkout depending on the fixture, with the lockfile bun writes against it. */ | |
| 276 | const checkout = async (reg: string, dir: string): Promise<string> => { | |
| 277 | const co = join(dir, "co"); | |
| 278 | mkdirSync(co); | |
| 279 | writeFileSync(join(co, "package.json"), JSON.stringify({ name: "x", version: "1.0.0", dependencies: { "fixture-pkg": "1.0.0" } })); | |
| 280 | const env = { PATH: process.env["PATH"] ?? "", HOME: dir, BUN_CONFIG_REGISTRY: reg }; | |
| 281 | // Async: the fixture registry answers from this process's event loop. | |
| 282 | const made = Bun.spawn(["bun", "install", "--lockfile-only"], { | |
| 283 | cwd: co, | |
| 284 | env: { ...env, BUN_INSTALL_CACHE_DIR: join(dir, "lock-cache") }, | |
| 285 | stdout: "pipe", | |
| 286 | stderr: "pipe", | |
outdated
A bun.lock of |
||
| 287 | }); | |
| 288 | if ((await made.exited) !== 0) throw new Error(`fixture lockfile: ${await new Response(made.stderr).text()}`); | |
| 289 | return co; | |
| 290 | }; | |
| 291 | ||
| 292 | const envFor = (dir: string, reg: string): Record<string, string> => ({ | |
| 293 | PATH: process.env["PATH"] ?? "", | |
| 294 | HOME: dir, | |
| 295 | BUN_CONFIG_REGISTRY: reg, | |
| 296 | }); | |
| 297 | ||
| 298 | test("copying asks every bun install in a contained command for the copyfile backend, once", () => { | |
| 299 | expect(copying("bun install --frozen-lockfile")).toBe("bun install --backend=copyfile --frozen-lockfile"); | |
| 300 | expect(copying("cd x && bun install")).toBe("cd x && bun install --backend=copyfile"); | |
| 301 | expect(copying("bun install --backend=hardlink")).toBe("bun install --backend=hardlink"); | |
| 302 | expect(copying("npm ci")).toBe("npm ci"); | |
| 303 | }); | |
| 304 | ||
| 305 | test("a contained install reads the shared cache; an unfilled one keeps its own", () => { | |
| 306 | const filled = cellInstall("bun install", { A: "1" }, "/c/bun"); | |
| 307 | expect(filled.command).toBe("bun install --backend=copyfile"); | |
| 308 | expect(filled.env).toEqual({ A: "1", BUN_INSTALL_CACHE_DIR: "/c/bun" }); | |
| 309 | expect(cellInstall("bun install", { A: "1" }, undefined)).toEqual({ command: "bun install", env: { A: "1" } }); | |
| 310 | }); | |
| 311 | ||
| 312 | test("a checkout with no lockfile fills nothing, says so, and is not a failure", async () => { | |
| 313 | const dir = await scratch(); | |
| 314 | const lines: string[] = []; | |
| 315 | const filled = await prefillInstallCache({ cache: join(dir, "cache"), dir, env: {}, log: (l) => lines.push(l), signal: never }); | |
| 316 | expect(filled).toEqual({ ok: true, value: false }); | |
| 317 | expect(lines.join("\n")).toContain("no bun lockfile"); | |
| 318 | await rm(dir, { recursive: true, force: true }); | |
| 319 | }); | |
| 320 | ||
| 321 | test("a fill needs only the pinned registry, and a warm cache needs none", async () => { | |
| 322 | const reg = await registry(); | |
| 323 | try { | |
| 324 | const co = await checkout(reg.url, reg.dir); | |
| 325 | const cache = join(reg.dir, "cache"); | |
| 326 | const first = await prefillInstallCache({ cache, dir: co, env: envFor(reg.dir, reg.url), log: () => {}, signal: never }); | |
| 327 | expect(first).toEqual({ ok: true, value: true }); | |
| 328 | expect(reg.hits).toContain("/fixture-pkg/-/fixture-pkg-1.0.0.tgz"); | |
| 329 | reg.stop(); | |
| 330 | const again = await prefillInstallCache({ | |
| 331 | cache, | |
| 332 | dir: co, | |
| 333 | env: envFor(reg.dir, reg.url), | |
| 334 | log: () => {}, | |
| 335 | signal: never, | |
| 336 | }); | |
| 337 | expect(again).toEqual({ ok: true, value: true }); | |
| 338 | expect(existsSync(`${cache}.lock`)).toBe(false); | |
| 339 | } finally { | |
| 340 | reg.stop(); | |
| 341 | await rm(reg.dir, { recursive: true, force: true }); | |
| 342 | } | |
| 343 | }); | |
| 344 | ||
| 345 | test("a fill's timeout is clamped under the lock's staleness", () => { | |
| 346 | expect(fillTimeoutMs(undefined)).toBe(LOCK_STALE_MS / 2); | |
| 347 | expect(fillTimeoutMs(LOCK_STALE_MS * 4)).toBe(LOCK_STALE_MS / 2); | |
| 348 | expect(fillTimeoutMs(1000)).toBe(1000); | |
| 349 | }); | |
| 350 | ||
| 351 | test("bun installs from a filled cache that is read-only, copying, with no registry", async () => { | |
| 352 | const reg = await registry(); | |
| 353 | try { | |
| 354 | const co = await checkout(reg.url, reg.dir); | |
| 355 | const cache = join(reg.dir, "cache"); | |
| 356 | const filled = await prefillInstallCache({ cache, dir: co, env: envFor(reg.dir, reg.url), log: () => {}, signal: never }); | |
| 357 | expect(filled).toEqual({ ok: true, value: true }); | |
| 358 | reg.stop(); | |
| 359 | Bun.spawnSync(["chmod", "-R", "a-w", cache]); | |
| 360 | const probe = join(cache, "probe"); | |
| 361 | let writable = true; | |
| 362 | try { | |
| 363 | writeFileSync(probe, "x"); | |
| 364 | } catch { | |
| 365 | writable = false; | |
| 366 | } | |
| 367 | // (root ignores modes; the install is then merely run, not proven read-only) | |
| 368 | if (process.getuid?.() !== 0) expect(writable).toBe(false); | |
| 369 | const proc = Bun.spawn(["bun", "install", "--frozen-lockfile", "--backend=copyfile"], { | |
| 370 | cwd: co, | |
| 371 | env: { ...envFor(reg.dir, "http://127.0.0.1:1/"), BUN_INSTALL_CACHE_DIR: cache }, | |
| 372 | stdout: "pipe", | |
| 373 | stderr: "pipe", | |
| 374 | }); | |
| 375 | const out = (await new Response(proc.stdout).text()) + (await new Response(proc.stderr).text()); | |
| 376 | const code = await proc.exited; | |
| 377 | Bun.spawnSync(["chmod", "-R", "u+w", cache]); | |
| 378 | expect({ code, out }).toMatchObject({ code: 0 }); | |
| 379 | expect(existsSync(join(co, "node_modules", "fixture-pkg", "package.json"))).toBe(true); | |
| 380 | } finally { | |
| 381 | reg.stop(); | |
| 382 | await rm(reg.dir, { recursive: true, force: true }); | |
| 383 | } | |
| 384 | }); | |
| 385 | ||
| 386 | test("a package the registry will not serve fails on the fetch, not the lockfile, and frees the lock", async () => { | |
| 387 | const reg = await registry(); | |
| 388 | try { | |
| 389 | const co = await checkout(reg.url, reg.dir); | |
| 390 | reg.stop(); | |
| 391 | const lines: string[] = []; | |
| 392 | const cache = join(reg.dir, "empty-cache"); | |
| 393 | const filled = await prefillInstallCache({ cache, dir: co, env: envFor(reg.dir, reg.url), log: (l) => lines.push(l), signal: never }); | |
| 394 | expect(filled.ok).toBe(false); | |
| 395 | expect(lines.join("\n")).not.toMatch(/lockfile had changes|frozen/i); | |
| 396 | expect(reg.hits.filter((h) => h.endsWith(".tgz"))).toEqual([]); // the tarball was asked for and never served | |
| 397 | expect(existsSync(`${cache}.lock`)).toBe(false); | |
| 398 | } finally { | |
| 399 | await rm(reg.dir, { recursive: true, force: true }); | |
| 400 | } | |
| 401 | }); | |
| 402 | ||
| 403 | test("the pull's own bunfig.toml and .npmrc do not choose where the forge fetches", async () => { | |
| 404 | const reg = await registry(); | |
| 405 | const decoy = await registry(); | |
| 406 | try { | |
| 407 | const co = await checkout(reg.url, reg.dir); | |
| 408 | writeFileSync(join(co, "bunfig.toml"), `[install]\nregistry = "${decoy.url}"\n`); | |
| 409 | writeFileSync(join(co, ".npmrc"), `registry=${decoy.url}\n`); | |
| 410 | const filled = await prefillInstallCache({ | |
| 411 | cache: join(reg.dir, "cache"), | |
| 412 | dir: co, | |
| 413 | env: envFor(reg.dir, reg.url), | |
| 414 | log: () => {}, | |
| 415 | signal: never, | |
| 416 | }); | |
| 417 | expect(filled).toEqual({ ok: true, value: true }); | |
| 418 | expect(decoy.hits).toEqual([]); | |
| 419 | expect(reg.hits.length).toBeGreaterThan(0); | |
| 420 | } finally { | |
| 421 | reg.stop(); | |
| 422 | decoy.stop(); | |
| 423 | await rm(reg.dir, { recursive: true, force: true }); | |
| 424 | await rm(decoy.dir, { recursive: true, force: true }); | |
| 425 | } | |
| 426 | }); | |
| 427 | ||
| 428 | test("a lockfile naming another host, or a git dependency, is not filled from", async () => { | |
| 429 | const reg = await registry(); | |
| 430 | try { | |
| 431 | const co = await checkout(reg.url, reg.dir); | |
| 432 | const cache = join(reg.dir, "cache"); | |
| 433 | const lock = join(co, "bun.lock"); | |
| 434 | const good = readFileSync(lock, "utf8"); | |
| 435 | for (const bad of [ | |
| 436 | `${good}\n// ["x", "http://169.254.169.254/x.tgz"]`, | |
| 437 | `${good}\n// ["y", "y@github:someone/repo#abc"]`, | |
| 438 | ]) { | |
| 439 | writeFileSync(lock, bad); | |
| 440 | const lines: string[] = []; | |
| 441 | const filled = await prefillInstallCache({ cache, dir: co, env: envFor(reg.dir, reg.url), log: (l) => lines.push(l), signal: never }); | |
| 442 | expect(filled).toEqual({ ok: true, value: false }); | |
| 443 | expect(lines.join("\n")).toContain("cache not filled"); | |
| 444 | } | |
| 445 | expect(reg.hits.filter((h) => h.endsWith(".tgz"))).toEqual([]); | |
| 446 | expect(existsSync(cache)).toBe(false); | |
| 447 | } finally { | |
| 448 | reg.stop(); | |
| 449 | await rm(reg.dir, { recursive: true, force: true }); | |
| 450 | } | |
| 451 | }); | |
| 452 | }); |
test/contain.test.ts +39
| @@ -1077,3 +1077,42 @@ describe.skipIf(noCells)( | ||
| 1077 | 1077 | }, 240_000); |
| 1078 | 1078 | }, |
| 1079 | 1079 | ); |
| 1080 | ||
| 1081 | describe.skipIf(noCells)("cells: the shared install cache is read-only (#7543)", () => { | |
| 1082 | test("a revealed cache under a hidden data dir is readable and a write into it fails", async () => { | |
| 1083 | const data = await mkdtemp(join(tmpdir(), "forge-cell-cache-")); | |
| 1084 | const cache = join(data, "check-install", "r", "bun"); | |
| 1085 | mkdirSync(cache, { recursive: true }); | |
| 1086 | writeFileSync(join(cache, "pkg"), "x"); | |
| 1087 | const dir = join(data, "..", `co-${Date.now()}`); | |
| 1088 | mkdirSync(dir); | |
| 1089 | const lines: string[] = []; | |
| 1090 | const result = await runStep( | |
| 1091 | "check", | |
| 1092 | `cat ${cache}/pkg && (echo y > ${cache}/new && echo WROTE || echo REFUSED)`, | |
| 1093 | { | |
| 1094 | cwd: dir, | |
| 1095 | env: childEnv({}), | |
| 1096 | log: (line) => lines.push(line), | |
| 1097 | signal: new AbortController().signal, | |
| 1098 | timeoutMs: 60_000, | |
| 1099 | cell: { | |
| 1100 | dir, | |
| 1101 | writable: [], | |
| 1102 | home: join(dir, ".home"), | |
| 1103 | hide: [data], | |
| 1104 | reveal: [cache], | |
| 1105 | egress: [], | |
| 1106 | reach: [], | |
| 1107 | limits: { memoryMiB: 1024, cpus: 2, diskMiB: 1024 }, | |
| 1108 | }, | |
| 1109 | }, | |
| 1110 | ); | |
| 1111 | expect(result.ok).toBe(true); | |
| 1112 | expect(lines).toContain("x"); | |
| 1113 | expect(lines).toContain("REFUSED"); | |
| 1114 | expect(lines).not.toContain("WROTE"); | |
| 1115 | await rm(data, { recursive: true, force: true }); | |
| 1116 | await rm(dir, { recursive: true, force: true }); | |
| 1117 | }, 60_000); | |
| 1118 | }); |
test/spawn-pipes.test.ts +1
| @@ -33,6 +33,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 33 | 33 | "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" }, |
| 34 | 34 | "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" }, |
| 35 | 35 | "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" }, |
| 36 | "src/checks/install.ts": { sites: ["stdout+stderr"], reason: "prefillInstallCache reads stdout and stderr to the end via Response.text() while the exit is awaited" }, | |
| 36 | 37 | "src/cli/config.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" }, |
| 37 | 38 | }; |
| 38 | 39 |
new above
offline
This copies bunfig.toml and .npmrc from the untrusted pull into the scratch dir, and the install then runs on the host with the pull's registry and URLs. Strip those files, pin the registry, and keep pull-written entries out of the cache main uses.