Public: anyone can read this; the crew can change it.
A contained check's bun install from the read-only warm cache fails on Bun 1.4.4: EACCES accessing temporary directory #12721
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12716-a-contained-check-s-bun-install-from-the → main 6 commits fork 4 threads · 4 unresolved
6 files
changed +84
−16
src/checks/install.ts +23 −4
| @@ -121,15 +121,34 @@ export function copying(command: string): string { | ||
| 121 | 121 | return command.replace(/\bbun install\b(?![^&;|]*--backend)/g, "bun install --backend=copyfile"); |
| 122 | 122 | } |
| 123 | 123 | |
| 124 | /** What a contained install runs: the command, and the cache env it reads. */ | |
| 124 | /** Where, under the cell's home, a contained install keeps its writable copy of the warm cache. */ | |
| 125 | export const WARM_COPY = ".install-warm"; | |
| 126 | ||
| 127 | /** | |
| 128 | * What a contained install runs: the command, and the cache env it reads. | |
| 129 | * Bun 1.4.4 moves a probe file into its cache dir before installing, so a | |
| 130 | * cache bound read-only fails at once (`EACCES accessing temporary | |
| 131 | * directory`, #12716). The cell therefore copies the warm cache into its own | |
| 132 | * home first and installs from the copy: the shared cache is only read. A | |
| 133 | * failed copy logs one line and leaves bun on the cell's own cache. | |
| 134 | */ | |
| 125 | 135 | export function cellInstall( |
| 126 | 136 | command: string, |
| 127 | 137 | env: Record<string, string>, |
| 128 | 138 | warm: string | undefined, |
| 129 | 139 | ): { readonly command: string; readonly env: Record<string, string> } { |
| 130 | return warm === undefined | |
| 131 | ? { command, env } | |
| 132 | : { command: copying(command), env: { ...env, BUN_INSTALL_CACHE_DIR: warm } }; | |
| 140 | if (warm === undefined) return { command, env }; | |
| 141 | const quoted = `'${warm.replace(/'/g, `'\\''`)}'`; | |
| 142 | const copy = `"$HOME/${WARM_COPY}"`; | |
| 143 | const prefix = | |
| 144 | `rm -rf ${copy} && mkdir -p ${copy} && cp -R ${quoted}/. ${copy} && chmod -R u+w ${copy}`; | |
| 145 | return { | |
| 146 | command: | |
| 147 | `if ${prefix}; then export BUN_INSTALL_CACHE_DIR=${copy}; ` + | |
| 148 | `else echo "warm install cache not copied; installing into the cell's own cache"; fi; ` + | |
| 149 | copying(command), | |
| 150 | env, | |
| 151 | }; | |
| 133 | 152 | } |
| 134 | 153 | |
| 135 | 154 | /** |
src/http/context.ts +17
| @@ -63,6 +63,23 @@ export interface Ctx { | ||
| 63 | 63 | |
| 64 | 64 | export type Handler = (ctx: Ctx) => Response | Promise<Response>; |
| 65 | 65 | |
| 66 | /** | |
| 67 | * A request whose URL parses. Bun builds `request.url` from the Host header, | |
| 68 | * so a Host no URL parser accepts (`evil#x--app.domain`) leaves a request every | |
| 69 | * `new URL(request.url)` throws on. It is rebuilt on a placeholder host, path and | |
| 70 | * query kept, headers (the Host among them) and body as they were: the routes | |
| 71 | * answer it as they answer any hostname they do not know. | |
| 72 | */ | |
Unrelated to #12716: move this Host fix to its own pull with a direct unit test. |
||
| 73 | export function parsable(request: Request): Request { | |
| 74 | try { | |
| 75 | new URL(request.url); | |
| 76 | return request; | |
| 77 | } catch { | |
| 78 | const path = /^[a-z][a-z0-9+.-]*:\/\/[^/]*(\/.*)?$/i.exec(request.url)?.[1] ?? "/"; | |
| 79 | return new Request(`http://localhost${path}`, request); | |
| 80 | } | |
| 81 | } | |
| 82 | ||
| 66 | 83 | /** |
| 67 | 84 | * The URL a request was made to, as the public sees it. Bun reports the |
| 68 | 85 | * scheme and host of the socket it accepted; behind a proxy that terminates |
src/http/server.ts +3 −1
| @@ -599,6 +599,7 @@ import { | ||
| 599 | 599 | type Auth, |
| 600 | 600 | type Ctx, |
| 601 | 601 | type Handler, |
| 602 | parsable, | |
| 602 | 603 | } from "./context.ts"; |
| 603 | 604 | import { |
| 604 | 605 | apiError, |
| @@ -2808,9 +2809,10 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2808 | 2809 | }; |
| 2809 | 2810 | |
| 2810 | 2811 | const handle = async ( |
| 2811 | request: Request, | |
| 2812 | raw: Request, | |
| 2812 | 2813 | server: Bun.Server<ForgeSocket<AppSocket>>, |
| 2813 | 2814 | ): Promise<Response | undefined> => { |
| 2815 | const request = parsable(raw); | |
| 2814 | 2816 | const started = performance.now(); |
| 2815 | 2817 | noteStart(request, started); |
| 2816 | 2818 | turnOf(app).begin(request, started); |
test/app-install-cache.test.ts +9 −3
| @@ -11,6 +11,7 @@ import { mkdtemp, rm } from "node:fs/promises"; | ||
| 11 | 11 | import { tmpdir } from "node:os"; |
| 12 | 12 | import { join } from "node:path"; |
| 13 | 13 | import type { AppManifest } from "../src/apps/manifest.ts"; |
| 14 | import { cellInstall } from "../src/checks/install.ts"; | |
| 14 | 15 | import { installPlan, startProcess, type StartInput } from "../src/apps/process.ts"; |
| 15 | 16 | import { fixtureRegistry } from "./helpers/fixture-registry.ts"; |
| 16 | 17 | |
| @@ -134,10 +135,13 @@ describe("an app environment's install (#7544)", () => { | ||
| 134 | 135 | test("a fill that fails does not fail the deploy: the install runs into its home as before", async () => { |
| 135 | 136 | const root = await mkdtemp(join(tmpdir(), "app-install-cache-")); |
| 136 | 137 | const reg = await registry(root); |
| 138 | let refusing: ReturnType<typeof Bun.serve> | undefined; | |
| 137 | 139 | try { |
| 138 | 140 | const co = await checkout(root, reg.url); |
| 139 | // A cold cache and a dead registry: the fill cannot fetch. | |
| 141 | // A cold cache and a registry that refuses every package at once: the fill | |
| 142 | // cannot fetch. (A closed port makes bun retry with back-off for seconds.) | |
| 140 | 143 | reg.stop(); |
| 144 | refusing = Bun.serve({ port: Number(new URL(reg.url).port), hostname: "127.0.0.1", fetch: () => new Response("no", { status: 404 }) }); | |
| 141 | 145 | const cache = join(root, "shared", "bun"); |
| 142 | 146 | const lines: string[] = []; |
| 143 | 147 | const ok = await withRegistry(reg.url, () => |
| @@ -155,6 +159,7 @@ describe("an app environment's install (#7544)", () => { | ||
| 155 | 159 | expect(lines.some((line) => line.startsWith("cache not filled: filling the install cache exited"))).toBe(true); |
| 156 | 160 | expect(existsSync(`${cache}.lock`)).toBe(false); |
| 157 | 161 | } finally { |
| 162 | refusing?.stop(true); | |
| 158 | 163 | reg.stop(); |
| 159 | 164 | await rm(root, { recursive: true, force: true }); |
| 160 | 165 | } |
| @@ -183,8 +188,9 @@ describe("the install's plan (#7544)", () => { | ||
| 183 | 188 | expect(plan.cell?.reveal).toEqual(["/cache/bun"]); |
| 184 | 189 | expect(plan.cell?.writable).toEqual([]); |
| 185 | 190 | expect(plan.cell?.home).toBe("/homes/app"); |
| 186 | expect(plan.command).toBe("bun install --backend=copyfile --frozen-lockfile"); | |
| 187 | expect(plan.env).toEqual({ PATH: "/bin", BUN_INSTALL_CACHE_DIR: "/cache/bun" }); | |
| 191 | expect(plan.command).toBe(cellInstall("bun install --frozen-lockfile", {}, "/cache/bun").command); | |
| 192 | expect(plan.command).toContain("else echo "); | |
| 193 | expect(plan.env).toEqual({ PATH: "/bin" }); // the cache env is exported by the command, from the copy | |
| 188 | 194 | expect(plan.lock).toBeUndefined(); |
| 189 | 195 | }); |
| 190 | 196 |
test/check-install.test.ts +32 −7
| @@ -4,7 +4,7 @@ | ||
| 4 | 4 | * machine's — queued again, never the pull's red. |
| 5 | 5 | */ |
| 6 | 6 | import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test"; |
| 7 | import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs"; | |
| 7 | import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; | |
| 8 | 8 | import { mkdtemp, rm } from "node:fs/promises"; |
| 9 | 9 | import { tmpdir } from "node:os"; |
| 10 | 10 | import { join } from "node:path"; |
| @@ -104,7 +104,7 @@ describe("withLock", () => { | ||
| 104 | 104 | await rm(dir, { recursive: true, force: true }); |
| 105 | 105 | }); |
| 106 | 106 | |
| 107 | test("8 parallel installs on one cache, 20 rounds: none ever extracts beside another", async () => { | |
| 107 | test("8 parallel installs on one cache, 8 rounds: none ever extracts beside another", async () => { | |
| 108 | 108 | // Each install "extracts" the same package: a directory that exists only |
| 109 | 109 | // while one install is in it. `mkdir` fails when another is in there — |
| 110 | 110 | // the collision bun reports as `Fail extracting tarball`. |
| @@ -118,7 +118,7 @@ describe("withLock", () => { | ||
| 118 | 118 | return proc.exited; |
| 119 | 119 | }; |
| 120 | 120 | let failures = 0; |
| 121 | for (let round = 0; round < 20; round++) { | |
| 121 | for (let round = 0; round < 8; round++) { | |
| 122 | 122 | const codes = await Promise.all( |
| 123 | 123 | Array.from({ length: 8 }, () => withLock(`${cache}.lock`, install, { signal: never })), |
| 124 | 124 | ); |
| @@ -327,10 +327,26 @@ describe("prefillInstallCache (#7543)", () => { | ||
| 327 | 327 | expect(copying("npm ci")).toBe("npm ci"); |
| 328 | 328 | }); |
| 329 | 329 | |
| 330 | test("a copy that fails says so once, exports no cache dir, and still runs the install", () => { | |
| 331 | const home = mkdtempSync(join(tmpdir(), "warm-fallback-")); | |
| 332 | try { | |
| 333 | const run = cellInstall("printf 'seen=%s' \"${BUN_INSTALL_CACHE_DIR:-none}\"", {}, join(home, "missing")); | |
| 334 | const got = Bun.spawnSync(["sh", "-c", run.command], { env: { PATH: process.env["PATH"] ?? "", HOME: home } }); | |
| 335 | expect(got.exitCode).toBe(0); | |
| 336 | expect(got.stdout.toString()).toBe("warm install cache not copied; installing into the cell's own cache\nseen=none"); | |
| 337 | } finally { | |
| 338 | rmSync(home, { recursive: true, force: true }); | |
| 339 | } | |
| 340 | }); | |
| 341 | ||
| 330 | 342 | test("a contained install reads the shared cache; an unfilled one keeps its own", () => { |
| 331 | 343 | const filled = cellInstall("bun install", { A: "1" }, "/c/bun"); |
| 332 | expect(filled.command).toBe("bun install --backend=copyfile"); | |
| 333 | expect(filled.env).toEqual({ A: "1", BUN_INSTALL_CACHE_DIR: "/c/bun" }); | |
| 344 | expect(filled.command).toBe( | |
| 345 | `if rm -rf "$HOME/.install-warm" && mkdir -p "$HOME/.install-warm" && cp -R '/c/bun'/. "$HOME/.install-warm" && chmod -R u+w "$HOME/.install-warm"; ` + | |
| 346 | `then export BUN_INSTALL_CACHE_DIR="$HOME/.install-warm"; ` + | |
| 347 | `else echo "warm install cache not copied; installing into the cell's own cache"; fi; bun install --backend=copyfile`, | |
| 348 | ); | |
| 349 | expect(filled.env).toEqual({ A: "1" }); | |
| 334 | 350 | expect(cellInstall("bun install", { A: "1" }, undefined)).toEqual({ command: "bun install", env: { A: "1" } }); |
| 335 | 351 | }); |
| 336 | 352 | |
| @@ -523,16 +539,25 @@ describe("prefillInstallCache (#7543)", () => { | ||
| 523 | 539 | } |
| 524 | 540 | // (root ignores modes; the install is then merely run, not proven read-only) |
| 525 | 541 | if (process.getuid?.() !== 0) expect(writable).toBe(false); |
| 526 | const proc = Bun.spawn(["bun", "install", "--frozen-lockfile", "--backend=copyfile"], { | |
| 542 | const snapshot = (): string => | |
| 543 | Bun.spawnSync(["find", cache, "-printf", "%p %s %T@\\n"]).stdout.toString().split("\n").sort().join("\n"); | |
| 544 | const before = snapshot(); | |
| 545 | const home = join(reg.dir, "home"); | |
| 546 | mkdirSync(home); | |
| 547 | const run = cellInstall("bun install --frozen-lockfile", {}, cache); | |
| 548 | const proc = Bun.spawn(["sh", "-c", run.command], { | |
| 527 | 549 | cwd: co, |
| 528 | env: { ...envFor(reg.dir, "http://127.0.0.1:1/"), BUN_INSTALL_CACHE_DIR: cache }, | |
| 550 | env: { ...envFor(reg.dir, reg.url), ...run.env, HOME: home }, // the registry is stopped; its url keys the cache entries | |
| 529 | 551 | stdout: "pipe", |
| 530 | 552 | stderr: "pipe", |
| 531 | 553 | }); |
| 532 | 554 | const out = (await new Response(proc.stdout).text()) + (await new Response(proc.stderr).text()); |
| 533 | 555 | const code = await proc.exited; |
| 556 | const after = snapshot(); | |
| 534 | 557 | Bun.spawnSync(["chmod", "-R", "u+w", cache]); |
| 535 | 558 | expect({ code, out }).toMatchObject({ code: 0 }); |
| 559 | expect(after).toBe(before); // the shared cache is unchanged | |
| 560 | expect(existsSync(join(home, ".install-warm"))).toBe(true); | |
| 536 | 561 | expect(existsSync(join(co, "node_modules", "fixture-pkg", "package.json"))).toBe(true); |
| 537 | 562 | } finally { |
| 538 | 563 | reg.stop(); |
test/module-state.baseline.txt −1
| @@ -60,7 +60,6 @@ src/crew/autonomy.ts:cleanAt | ||
| 60 | 60 | src/crew/autonomy.ts:conflicted |
| 61 | 61 | src/crew/autonomy.ts:conflictedAt |
| 62 | 62 | src/crew/autonomy.ts:sweeping |
| 63 | src/crew/ceiling_stagger.ts:memo | |
Unrelated baseline edit; belongs in its own pull. |
||
| 64 | 63 | src/crew/conversation.ts:live |
| 65 | 64 | src/crew/conversation.ts:queues |
| 66 | 65 | src/crew/conversation.ts:stoppers |
new above
offline
Non-blocking: unrelated to #12716. Add a direct unit test for parsable() and land it separately next time.