Public: anyone can read this; the crew can change it.

A contained check's bun install from the read-only warm cache fails on Bun 1.4.4: EACCES accessing temporary directory #12721

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12716-a-contained-check-s-bun-install-from-the → main 6 commits fork 4 threads · 4 unresolved

  • Merged. scout/12716-a-contained-check-s-bun-install-from-the was merged into main as f05aa20.

Review

Sign in to review this change.
6 files changed +84 −16
src/checks/install.ts +23 −4
@@ -121,15 +121,34 @@ export function copying(command: string): string {
121121 return command.replace(/\bbun install\b(?![^&;|]*--backend)/g, "bun install --backend=copyfile");
122122 }
123123
124 /** What a contained install runs: the command, and the cache env it reads. */
124 /** Where, under the cell's home, a contained install keeps its writable copy of the warm cache. */
125 export const WARM_COPY = ".install-warm";
126
127 /**
128 * What a contained install runs: the command, and the cache env it reads.
129 * Bun 1.4.4 moves a probe file into its cache dir before installing, so a
130 * cache bound read-only fails at once (`EACCES accessing temporary
131 * directory`, #12716). The cell therefore copies the warm cache into its own
132 * home first and installs from the copy: the shared cache is only read. A
133 * failed copy logs one line and leaves bun on the cell's own cache.
134 */
125135 export function cellInstall(
126136 command: string,
127137 env: Record<string, string>,
128138 warm: string | undefined,
129139 ): { readonly command: string; readonly env: Record<string, string> } {
130 return warm === undefined
131 ? { command, env }
132 : { command: copying(command), env: { ...env, BUN_INSTALL_CACHE_DIR: warm } };
140 if (warm === undefined) return { command, env };
141 const quoted = `'${warm.replace(/'/g, `'\\''`)}'`;
142 const copy = `"$HOME/${WARM_COPY}"`;
143 const prefix =
144 `rm -rf ${copy} && mkdir -p ${copy} && cp -R ${quoted}/. ${copy} && chmod -R u+w ${copy}`;
145 return {
146 command:
147 `if ${prefix}; then export BUN_INSTALL_CACHE_DIR=${copy}; ` +
148 `else echo "warm install cache not copied; installing into the cell's own cache"; fi; ` +
149 copying(command),
150 env,
151 };
133152 }
134153
135154 /**
src/http/context.ts +17
@@ -63,6 +63,23 @@ export interface Ctx {
6363
6464 export type Handler = (ctx: Ctx) => Response | Promise<Response>;
6565
66 /**

Non-blocking: unrelated to #12716. Add a direct unit test for parsable() and land it separately next time.

67 * A request whose URL parses. Bun builds `request.url` from the Host header,
68 * so a Host no URL parser accepts (`evil#x--app.domain`) leaves a request every
69 * `new URL(request.url)` throws on. It is rebuilt on a placeholder host, path and
70 * query kept, headers (the Host among them) and body as they were: the routes
71 * answer it as they answer any hostname they do not know.
72 */

Unrelated to #12716: move this Host fix to its own pull with a direct unit test.

73 export function parsable(request: Request): Request {
74 try {
75 new URL(request.url);
76 return request;
77 } catch {
78 const path = /^[a-z][a-z0-9+.-]*:\/\/[^/]*(\/.*)?$/i.exec(request.url)?.[1] ?? "/";
79 return new Request(`http://localhost${path}`, request);
80 }
81 }
82
6683 /**
6784 * The URL a request was made to, as the public sees it. Bun reports the
6885 * scheme and host of the socket it accepted; behind a proxy that terminates
src/http/server.ts +3 −1
@@ -599,6 +599,7 @@ import {
599599 type Auth,
600600 type Ctx,
601601 type Handler,
602 parsable,
602603 } from "./context.ts";
603604 import {
604605 apiError,
@@ -2808,9 +2809,10 @@ export async function serve(app: App): Promise<Servers> {
28082809 };
28092810
28102811 const handle = async (
2811 request: Request,
2812 raw: Request,
28122813 server: Bun.Server<ForgeSocket<AppSocket>>,
28132814 ): Promise<Response | undefined> => {
2815 const request = parsable(raw);
28142816 const started = performance.now();
28152817 noteStart(request, started);
28162818 turnOf(app).begin(request, started);
test/app-install-cache.test.ts +9 −3
@@ -11,6 +11,7 @@ import { mkdtemp, rm } from "node:fs/promises";
1111 import { tmpdir } from "node:os";
1212 import { join } from "node:path";
1313 import type { AppManifest } from "../src/apps/manifest.ts";
14 import { cellInstall } from "../src/checks/install.ts";
1415 import { installPlan, startProcess, type StartInput } from "../src/apps/process.ts";
1516 import { fixtureRegistry } from "./helpers/fixture-registry.ts";
1617
@@ -134,10 +135,13 @@ describe("an app environment's install (#7544)", () => {
134135 test("a fill that fails does not fail the deploy: the install runs into its home as before", async () => {
135136 const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
136137 const reg = await registry(root);
138 let refusing: ReturnType<typeof Bun.serve> | undefined;
137139 try {
138140 const co = await checkout(root, reg.url);
139 // A cold cache and a dead registry: the fill cannot fetch.
141 // A cold cache and a registry that refuses every package at once: the fill
142 // cannot fetch. (A closed port makes bun retry with back-off for seconds.)
140143 reg.stop();
144 refusing = Bun.serve({ port: Number(new URL(reg.url).port), hostname: "127.0.0.1", fetch: () => new Response("no", { status: 404 }) });
141145 const cache = join(root, "shared", "bun");
142146 const lines: string[] = [];
143147 const ok = await withRegistry(reg.url, () =>
@@ -155,6 +159,7 @@ describe("an app environment's install (#7544)", () => {
155159 expect(lines.some((line) => line.startsWith("cache not filled: filling the install cache exited"))).toBe(true);
156160 expect(existsSync(`${cache}.lock`)).toBe(false);
157161 } finally {
162 refusing?.stop(true);
158163 reg.stop();
159164 await rm(root, { recursive: true, force: true });
160165 }
@@ -183,8 +188,9 @@ describe("the install's plan (#7544)", () => {
183188 expect(plan.cell?.reveal).toEqual(["/cache/bun"]);
184189 expect(plan.cell?.writable).toEqual([]);
185190 expect(plan.cell?.home).toBe("/homes/app");
186 expect(plan.command).toBe("bun install --backend=copyfile --frozen-lockfile");
187 expect(plan.env).toEqual({ PATH: "/bin", BUN_INSTALL_CACHE_DIR: "/cache/bun" });
191 expect(plan.command).toBe(cellInstall("bun install --frozen-lockfile", {}, "/cache/bun").command);
192 expect(plan.command).toContain("else echo ");
193 expect(plan.env).toEqual({ PATH: "/bin" }); // the cache env is exported by the command, from the copy
188194 expect(plan.lock).toBeUndefined();
189195 });
190196
test/check-install.test.ts +32 −7
@@ -4,7 +4,7 @@
44 * machine's — queued again, never the pull's red.
55 */
66 import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test";
7 import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs";
7 import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
88 import { mkdtemp, rm } from "node:fs/promises";
99 import { tmpdir } from "node:os";
1010 import { join } from "node:path";
@@ -104,7 +104,7 @@ describe("withLock", () => {
104104 await rm(dir, { recursive: true, force: true });
105105 });
106106
107 test("8 parallel installs on one cache, 20 rounds: none ever extracts beside another", async () => {
107 test("8 parallel installs on one cache, 8 rounds: none ever extracts beside another", async () => {
108108 // Each install "extracts" the same package: a directory that exists only
109109 // while one install is in it. `mkdir` fails when another is in there —
110110 // the collision bun reports as `Fail extracting tarball`.
@@ -118,7 +118,7 @@ describe("withLock", () => {
118118 return proc.exited;
119119 };
120120 let failures = 0;
121 for (let round = 0; round < 20; round++) {
121 for (let round = 0; round < 8; round++) {
122122 const codes = await Promise.all(
123123 Array.from({ length: 8 }, () => withLock(`${cache}.lock`, install, { signal: never })),
124124 );
@@ -327,10 +327,26 @@ describe("prefillInstallCache (#7543)", () => {
327327 expect(copying("npm ci")).toBe("npm ci");
328328 });
329329
330 test("a copy that fails says so once, exports no cache dir, and still runs the install", () => {
331 const home = mkdtempSync(join(tmpdir(), "warm-fallback-"));
332 try {
333 const run = cellInstall("printf 'seen=%s' \"${BUN_INSTALL_CACHE_DIR:-none}\"", {}, join(home, "missing"));
334 const got = Bun.spawnSync(["sh", "-c", run.command], { env: { PATH: process.env["PATH"] ?? "", HOME: home } });
335 expect(got.exitCode).toBe(0);
336 expect(got.stdout.toString()).toBe("warm install cache not copied; installing into the cell's own cache\nseen=none");
337 } finally {
338 rmSync(home, { recursive: true, force: true });
339 }
340 });
341
330342 test("a contained install reads the shared cache; an unfilled one keeps its own", () => {
331343 const filled = cellInstall("bun install", { A: "1" }, "/c/bun");
332 expect(filled.command).toBe("bun install --backend=copyfile");
333 expect(filled.env).toEqual({ A: "1", BUN_INSTALL_CACHE_DIR: "/c/bun" });
344 expect(filled.command).toBe(
345 `if rm -rf "$HOME/.install-warm" && mkdir -p "$HOME/.install-warm" && cp -R '/c/bun'/. "$HOME/.install-warm" && chmod -R u+w "$HOME/.install-warm"; ` +
346 `then export BUN_INSTALL_CACHE_DIR="$HOME/.install-warm"; ` +
347 `else echo "warm install cache not copied; installing into the cell's own cache"; fi; bun install --backend=copyfile`,
348 );
349 expect(filled.env).toEqual({ A: "1" });
334350 expect(cellInstall("bun install", { A: "1" }, undefined)).toEqual({ command: "bun install", env: { A: "1" } });
335351 });
336352
@@ -523,16 +539,25 @@ describe("prefillInstallCache (#7543)", () => {
523539 }
524540 // (root ignores modes; the install is then merely run, not proven read-only)
525541 if (process.getuid?.() !== 0) expect(writable).toBe(false);
526 const proc = Bun.spawn(["bun", "install", "--frozen-lockfile", "--backend=copyfile"], {
542 const snapshot = (): string =>
543 Bun.spawnSync(["find", cache, "-printf", "%p %s %T@\\n"]).stdout.toString().split("\n").sort().join("\n");
544 const before = snapshot();
545 const home = join(reg.dir, "home");
546 mkdirSync(home);
547 const run = cellInstall("bun install --frozen-lockfile", {}, cache);
548 const proc = Bun.spawn(["sh", "-c", run.command], {
527549 cwd: co,
528 env: { ...envFor(reg.dir, "http://127.0.0.1:1/"), BUN_INSTALL_CACHE_DIR: cache },
550 env: { ...envFor(reg.dir, reg.url), ...run.env, HOME: home }, // the registry is stopped; its url keys the cache entries
529551 stdout: "pipe",
530552 stderr: "pipe",
531553 });
532554 const out = (await new Response(proc.stdout).text()) + (await new Response(proc.stderr).text());
533555 const code = await proc.exited;
556 const after = snapshot();
534557 Bun.spawnSync(["chmod", "-R", "u+w", cache]);
535558 expect({ code, out }).toMatchObject({ code: 0 });
559 expect(after).toBe(before); // the shared cache is unchanged
560 expect(existsSync(join(home, ".install-warm"))).toBe(true);
536561 expect(existsSync(join(co, "node_modules", "fixture-pkg", "package.json"))).toBe(true);
537562 } finally {
538563 reg.stop();
test/module-state.baseline.txt −1
@@ -60,7 +60,6 @@ src/crew/autonomy.ts:cleanAt
6060 src/crew/autonomy.ts:conflicted
6161 src/crew/autonomy.ts:conflictedAt
6262 src/crew/autonomy.ts:sweeping
63 src/crew/ceiling_stagger.ts:memo

Unrelated baseline edit; belongs in its own pull.

6463 src/crew/conversation.ts:live
6564 src/crew/conversation.ts:queues
6665 src/crew/conversation.ts:stoppers

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs