7db9ed0 · 91f325e
@@ -698,6 +698,25 @@
into it take turns under a lock file, so two never extract one package
into the same directory. A `job` check installs in its own pod and is not
retried.
**An app environment installs from that same cache** (`src/apps/process.ts`,
#7544). Before a cold checkout's `install`, production or a preview, the
forge fills the repository's cache on the host from the checkout's
`bun.lock`, the way checks fill it. The install then reads the cache:
read-only and copying inside a cell (`BUN_INSTALL_CACHE_DIR`), or under the
lock file without one. The environment's home stays its own, and a new
preview fetches nothing the cache already holds. A checkout with no
`bun.lock`, or one the fill does not trust, installs into its home as
before; so does one whose fill fails (the registry unreachable, the lock
cancelled, a timeout): the cache is an optimisation, and its failure is a
`cache not filled:` line in the deploy's log, never a failed deploy. A warm
start still runs no install.
Uncontained, the install holds the cache's lock for its whole run, since
an install outside a cell (its own lifecycle scripts included) could write
the cache. The cost: two cold environments of one repository install one
after the other there. Inside a cell the cache is read-only, so only the
fill takes the lock and installs run side by side.
**Placement.** `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check
in a cell on the forge's machine. `job` runs install and check as one
Kubernetes Job (`src/cluster/jobs.ts`) instead: `FORGE_CHECKS_IMAGE` is its
@@ -19,6 +19,7 @@
import { isReservedAppEnvName } from "../core/app_env.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import { spellings, stripForgeDirs } from "../runs/paths.ts";
import { seconds } from "./timing.ts";
import { cellInstall, prefillInstallCache, withLock } from "../checks/install.ts";
import {
  commandPid,
  killTree,
@@ -71,6 +72,14 @@
export interface StartInput {
   * start — the forge coming back, or an environment waking — only runs it.
   */
  readonly warm?: boolean | undefined;
  /**
   * The repository's shared bun cache (`checks/install.ts`
   * `checkInstallCache`), the one its checks fill (#7544). The forge fills
   * it on the host from the checkout's lockfile, then the install reads it:
   * read-only inside a cell, under its lock without one. A checkout the
   * fill will not trust (no `bun.lock`) installs as before, into its home.
   */
  readonly installCache?: string | undefined;
}
/** What a caller knows about a cell; the rest comes from the manifest. */
@@ -380,6 +389,73 @@
export function stepCell(input: StartInput): CellSpec | undefined {
  };
}
/** The forge's own `BUN_CONFIG_REGISTRY`, when its environment names one. */
function forgeRegistry(): Record<string, string> {
  const registry = process.env["BUN_CONFIG_REGISTRY"];
  return registry === undefined || registry === "" ? {} : { BUN_CONFIG_REGISTRY: registry };
}
/** How an install step runs: plain data, so its composition is testable without a cell. */
export interface InstallPlan {
  readonly command: string;
  readonly env: Record<string, string>;
  readonly cell: CellSpec | undefined;
  /** The lock it holds while it runs, if any. */
  readonly lock: string | undefined;
}
/**
 * The install once the fill has answered (#7544): from the repository's
 * shared cache when it `filled`, else into the environment's own home as
 * before. Inside a cell only the cache is added to its view, read-only;
 * uncontained, the install could write the cache, so it takes its turn
 * under the cache's lock.
 */
      
        junipercrew
        12d ago
        outdated
      
      A failed fill (err) returns here and fails the deploy at the install stage. Log the error and fall through to the home install instead: if (!fill.ok) { input.log(cache not filled: ${fill.error}); return step({ command, env }, stepCell(input)); }, then the !fill.value case follows the same path. Test it.
    
export function installPlan(
  command: string,
  input: StartInput,
  env: Record<string, string>,
  filled: boolean,
): InstallPlan {
  const cache = input.installCache;
  const cell = stepCell(input);
  if (cache === undefined || !filled) return { command, env, cell, lock: undefined };
  if (cell !== undefined)
    return { ...cellInstall(command, env, cache), cell: { ...cell, reveal: [...(cell.reveal ?? []), cache] }, lock: undefined };
  return { command, env: { ...env, BUN_INSTALL_CACHE_DIR: cache }, cell: undefined, lock: `${cache}.lock` };
}
/**
 * The install step. The cache is an optimisation: a fill that fails (the
 * registry unreachable, the lock cancelled, a timeout) is logged and the
 * install runs into the environment's home, as it did before the cache.
 */
async function install(
  command: string,
  input: StartInput,
  env: Record<string, string>,
): Promise<Result<void, string>> {
  let filled = false;
  if (input.installCache !== undefined) {
    const fill = await prefillInstallCache({
      cache: input.installCache,
      dir: input.dir,
      env,
      log: input.log,
      signal: input.signal,
      timeoutMs: STEP_TIMEOUT_MS,
    });
    if (fill.ok) filled = fill.value;
    else input.log(`cache not filled: ${fill.error}`);
  }
  const plan = installPlan(command, input, env, filled);
  const step = (): Promise<Result<void, string>> =>
    runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell });
  if (plan.lock === undefined) return step();
  const held = await withLock(plan.lock, step, { signal: input.signal });
  return held.ok ? held.value : held;
}
/**
 * Install, then start the app and run its tests side by side, and wait for
 * the app to answer on its port. Returns as soon as it does — `tested`
@@ -394,16 +470,13 @@
export async function startProcess(
  const stepEnv = childEnv({
    CI: "1",
    FORGE_APP_ENV: input.env === "production" ? "production" : "preview",
    // A mirror the forge itself is pointed at is the one the fill and the
    // install fetch from (`checks/install.ts` DEFAULT_REGISTRY otherwise).
    ...forgeRegistry(),
  });
  if (manifest.install !== null && input.warm !== true) {
    const installed = await runStep("install", manifest.install, {
      cwd: input.dir,
      env: stepEnv,
      log,
      signal: input.signal,
      cell: stepCell(input),
    });
    const installed = await install(manifest.install, input, stepEnv);
    if (!installed.ok)
      return err({ stage: "install", message: installed.error });
  }
@@ -32,6 +32,7 @@
import { mintAppToken, renewAppToken, revokeAppToken } from "./token.ts";
import type { ForgeEvent } from "../core/events.ts";
import { announceRun } from "../actions/runs.ts";
import { requeueAbandonedChecks } from "../checks/service.ts";
import { checkInstallCache } from "../checks/install.ts";
import { getPull, listPulls, type Pull } from "../core/pulls.ts";
import { pullMayRun } from "../core/pull_trust.ts";
import { canEnterApp, getRepoById, type Repo } from "../core/repos.ts";
@@ -1403,6 +1404,9 @@
export class AppsRuntime {
      log: write,
      warm,
      cell: this.cellFor(state.app, state.env),
      // Every environment, production too, installs from the cache the
      // repository's checks fill, so a new preview fetches nothing it has.
      installCache: checkInstallCache(this.app.config.dataDir, repo.id),
    };
    this.noteCell(run.id, write);
    clock.lap("setup");
@@ -0,0 +1,239 @@
/**
 * An app environment installs from its repository's shared bun cache
 * (#7544): the forge fills it on the host from the lockfile, as checks do,
 * so a second environment of the same app at the same lockfile — a new
 * preview — installs with its registry dead. A checkout with no `bun.lock`
 * installs as it always did, into its own home.
 */
import { describe, expect, setDefaultTimeout, test } from "bun:test";
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AppManifest } from "../src/apps/manifest.ts";
import { installPlan, startProcess, type StartInput } from "../src/apps/process.ts";
setDefaultTimeout(60_000);
const manifest: AppManifest = {
  run: `${process.execPath} server.ts`,
  install: "bun install --frozen-lockfile",
  test: null,
  health: "/",
  ready: 10,
  tty: null,
  service: null,
  looks: ["/"],
};
const SERVER = `Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("ok") });\n`;
/** A registry on loopback serving one package, `[email protected]`; counts its requests. */
async function registry(dir: string): Promise<{ url: string; hits: string[]; stop: () => void }> {
  mkdirSync(join(dir, "pkg", "package"), { recursive: true });
  writeFileSync(join(dir, "pkg", "package", "package.json"), JSON.stringify({ name: "fixture-pkg", version: "1.0.0" }));
  const made = Bun.spawnSync(["tar", "-czf", join(dir, "fixture.tgz"), "-C", join(dir, "pkg"), "package"]);
  if (made.exitCode !== 0) throw new Error("tar is needed for the fixture registry");
  const tarball = new Uint8Array(await Bun.file(join(dir, "fixture.tgz")).arrayBuffer());
  const hits: string[] = [];
  const server: ReturnType<typeof Bun.serve> = Bun.serve({
    port: 0,
    hostname: "127.0.0.1",
    fetch(req): Response {
      const path = new URL(req.url).pathname;
      hits.push(path);
      if (path === "/fixture-pkg/-/fixture-pkg-1.0.0.tgz") return new Response(tarball);
      if (path === "/fixture-pkg")
        return Response.json({
          name: "fixture-pkg",
          "dist-tags": { latest: "1.0.0" },
          versions: {
            "1.0.0": {
              name: "fixture-pkg",
              version: "1.0.0",
              dist: { tarball: `http://127.0.0.1:${server.port}/fixture-pkg/-/fixture-pkg-1.0.0.tgz` },
            },
          },
        });
      return new Response("not found", { status: 404 });
    },
  });
  return { url: `http://127.0.0.1:${server.port}/`, hits, stop: () => void server.stop(true) };
}
/** An app's checkout depending on the fixture, with the lockfile bun writes against it. */
async function checkout(dir: string, reg: string): Promise<string> {
  const co = join(dir, "first");
  mkdirSync(co);
  writeFileSync(join(co, "package.json"), JSON.stringify({ name: "x", version: "1.0.0", dependencies: { "fixture-pkg": "1.0.0" } }));
  writeFileSync(join(co, "server.ts"), SERVER);
  // Async: the fixture registry answers from this process's event loop.
  const made = Bun.spawn(["bun", "install", "--lockfile-only"], {
    cwd: co,
    env: { PATH: process.env["PATH"] ?? "", HOME: dir, BUN_CONFIG_REGISTRY: reg, BUN_INSTALL_CACHE_DIR: join(dir, "lock-cache") },
    stdout: "pipe",
    stderr: "pipe",
  });
  if ((await made.exited) !== 0) throw new Error(`fixture lockfile: ${await new Response(made.stderr).text()}`);
  return co;
}
/** Run `body` with the forge's own environment pointing bun at `url`, as an operator's mirror would. */
async function withRegistry<T>(url: string, body: () => Promise<T>): Promise<T> {
  const before = process.env["BUN_CONFIG_REGISTRY"];
  process.env["BUN_CONFIG_REGISTRY"] = url;
  try {
    return await body();
  } finally {
    if (before === undefined) delete process.env["BUN_CONFIG_REGISTRY"];
    else process.env["BUN_CONFIG_REGISTRY"] = before;
  }
}
async function start(dir: string, extra: Partial<StartInput>, lines: string[]): Promise<boolean> {
  const started = await startProcess({
    dir,
    dataDir: dir,
    manifest,
    env: "production",
    url: "http://app.test",
    commitOid: "0".repeat(40),
    forgeUrl: "http://forge.test",
    identityKey: "key",
    log: (line) => lines.push(line),
    signal: new AbortController().signal,
    ...extra,
  });
  if (started.ok) await started.value.handle.stop();
  return started.ok;
}
describe("an app environment's install (#7544)", () => {
  test("a second environment at the same lockfile installs from the shared cache with the registry dead", async () => {
    const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
    const reg = await registry(root);
    try {
      const first = await checkout(root, reg.url);
      const second = join(root, "second");
      cpSync(first, second, { recursive: true });
      const cache = join(root, "shared", "bun");
      const lines: string[] = [];
      expect(await withRegistry(reg.url, () => start(first, { installCache: cache }, lines))).toBe(true);
      expect(reg.hits).toContain("/fixture-pkg/-/fixture-pkg-1.0.0.tgz");
      expect(existsSync(join(first, "node_modules", "fixture-pkg", "package.json"))).toBe(true);
      // The repository's cache holds it, not the environment's home.
      expect(readdirSync(cache).some((name) => name.startsWith("fixture-pkg@"))).toBe(true);
      reg.stop();
      reg.hits.length = 0;
      const again: string[] = [];
      expect(await withRegistry(reg.url, () => start(second, { installCache: cache }, again))).toBe(true);
      expect(existsSync(join(second, "node_modules", "fixture-pkg", "package.json"))).toBe(true);
      expect(reg.hits).toEqual([]);
      expect(existsSync(`${cache}.lock`)).toBe(false);
    } finally {
      reg.stop();
      await rm(root, { recursive: true, force: true });
    }
  });
  test("a checkout with no bun.lock installs as before, without the shared cache", async () => {
    const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
    try {
      writeFileSync(join(root, "server.ts"), SERVER);
      const cache = join(root, "shared", "bun");
      const lines: string[] = [];
      const ok = await start(
        root,
        {
          installCache: cache,
          manifest: { ...manifest, install: `printf '%s' "\${BUN_INSTALL_CACHE_DIR:-none}" > seen` },
        },
        lines,
      );
      expect(ok).toBe(true);
      expect(readFileSync(join(root, "seen"), "utf8")).toBe("none");
      expect(lines).toContain("cache not filled: no bun lockfile in the checkout");
    } finally {
      await rm(root, { recursive: true, force: true });
    }
  });
  test("a fill that fails does not fail the deploy: the install runs into its home as before", async () => {
    const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
    const reg = await registry(root);
    try {
      const co = await checkout(root, reg.url);
      // A cold cache and a dead registry: the fill cannot fetch.
      reg.stop();
      const cache = join(root, "shared", "bun");
      const lines: string[] = [];
      const ok = await withRegistry(reg.url, () =>
        start(
          co,
          {
            installCache: cache,
            manifest: { ...manifest, install: `printf '%s' "\${BUN_INSTALL_CACHE_DIR:-none}" > seen` },
          },
          lines,
        ),
      );
      expect(ok).toBe(true);
      expect(readFileSync(join(co, "seen"), "utf8")).toBe("none");
      expect(lines.some((line) => line.startsWith("cache not filled: filling the install cache exited"))).toBe(true);
      expect(existsSync(`${cache}.lock`)).toBe(false);
    } finally {
      reg.stop();
      await rm(root, { recursive: true, force: true });
    }
  });
});
describe("the install's plan (#7544)", () => {
  const input = (extra: Partial<StartInput>): StartInput => ({
    dir: "/co",
    dataDir: "/data",
    manifest,
    env: "pull-7",
    url: "http://app.test",
    commitOid: "0".repeat(40),
    forgeUrl: "http://forge.test",
    identityKey: "key",
    log: () => {},
    signal: new AbortController().signal,
    ...extra,
  });
  const cell = { hide: ["/data"], home: "/homes/app" };
  const env = { PATH: "/bin" };
  test("in a cell, a filled cache is revealed read-only and the install copies from it", () => {
    const plan = installPlan("bun install --frozen-lockfile", input({ cell, installCache: "/cache/bun" }), env, true);
    expect(plan.cell?.reveal).toEqual(["/cache/bun"]);
    expect(plan.cell?.writable).toEqual([]);
    expect(plan.cell?.home).toBe("/homes/app");
    expect(plan.command).toBe("bun install --backend=copyfile --frozen-lockfile");
    expect(plan.env).toEqual({ PATH: "/bin", BUN_INSTALL_CACHE_DIR: "/cache/bun" });
    expect(plan.lock).toBeUndefined();
  });
  test("uncontained, a filled cache is read under its lock", () => {
    const plan = installPlan("bun install", input({ installCache: "/cache/bun" }), env, true);
    expect(plan.cell).toBeUndefined();
    expect(plan.command).toBe("bun install");
    expect(plan.env["BUN_INSTALL_CACHE_DIR"]).toBe("/cache/bun");
    expect(plan.lock).toBe("/cache/bun.lock");
  });
  test("an unfilled cache, or none, leaves the install as it was", () => {
    for (const plan of [
      installPlan("bun install", input({ cell, installCache: "/cache/bun" }), env, false),
      installPlan("bun install", input({ cell }), env, true),
    ]) {
      expect(plan.command).toBe("bun install");
      expect(plan.env).toEqual(env);
      expect(plan.cell?.reveal).toBeUndefined();
      expect(plan.lock).toBeUndefined();
    }
  });
});
@@ -24,6 +24,8 @@
import type { Repo } from "../src/core/repos.ts";
import { listRuns, type Run } from "../src/core/runs.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { checkInstallCache } from "../src/checks/install.ts";
import { readRunLog } from "../src/runs/log.ts";
const DOMAIN = "apps.test";
@@ -85,10 +87,11 @@
describe("apps adversarial: the install and test steps", () => {
      "the deploy settling",
    );
  /** The pull branch carrying only this forge.json, opened against the default branch. */
  /** The pull branch carrying this forge.json (and any `extra` files), opened against the default branch. */
  async function pullWith(
    branch: string,
    manifest: Record<string, string>,
    extra: readonly { path: string; content: string }[] = [],
  ): Promise<{ number: number; commitOid: string }> {
    const committed = await commitToBranch(app, alice, repo, {
      branch,
@@ -99,6 +102,7 @@
describe("apps adversarial: the install and test steps", () => {
          path: "forge.json",
          content: `${JSON.stringify({ app: manifest }, null, 2)}\n`,
        },
        ...extra,
      ],
    });
    if (!committed.ok) throw new Error(JSON.stringify(committed.error));
@@ -176,6 +180,49 @@
describe("apps adversarial: the install and test steps", () => {
    );
  }, 20_000);
  test("a preview's install reads its repository's shared cache (#7544)", async () => {
    // A local dependency: a real bun.lock the fill trusts, and nothing fetched.
    const pull = await pullWith(
      "shared-cache",
      {
        install: `bun install --frozen-lockfile && echo "install cache: \${BUN_INSTALL_CACHE_DIR:-none}"`,
        run: "bun run src/server.ts",
      },
      [
        { path: "dep/package.json", content: `{"name":"dep","version":"1.0.0"}\n` },
        {
          path: "package.json",
          content: `${JSON.stringify({ name: "steps", version: "0.1.0", private: true, dependencies: { dep: "file:./dep" } })}\n`,
        },
        {
          path: "bun.lock",
          content: `{
  "lockfileVersion": 1,
  "configVersion": 1,
  "workspaces": {
    "": {
      "name": "steps",
      "dependencies": {
        "dep": "file:./dep",
      },
    },
  },
  "packages": {
    "dep": ["dep@file:dep", {}],
  }
}
`,
        },
      ],
    );
    const run = await settled(
      (r) => r.env === `pull-${pull.number}` && r.commitOid === pull.commitOid,
    );
    const log = readRunLog(dataDir, run.id).lines;
    expect(run.state).toBe("ok");
    expect(log).toContain(`install cache: ${checkInstallCache(dataDir, repo.id)}`);
  }, 20_000);
  test("closing a pull whose deploy already answered kills its still-running test step — no orphan `bun test`", async () => {
    // A merge lands this same forge.json on the default branch and
    // production redeploys it too; only the preview's test step sleeps, so
		@@ -698,6 +698,25 @@ into it take turns under a lock file, so two never extract one package
698	698	into the same directory. A `job` check installs in its own pod and is not
699	699	retried.
700	700
	701	An app environment installs from that same cache (`src/apps/process.ts`,
	702	#7544). Before a cold checkout's `install`, production or a preview, the
	703	forge fills the repository's cache on the host from the checkout's
	704	`bun.lock`, the way checks fill it. The install then reads the cache:
	705	read-only and copying inside a cell (`BUN_INSTALL_CACHE_DIR`), or under the
	706	lock file without one. The environment's home stays its own, and a new
	707	preview fetches nothing the cache already holds. A checkout with no
	708	`bun.lock`, or one the fill does not trust, installs into its home as
	709	before; so does one whose fill fails (the registry unreachable, the lock
	710	cancelled, a timeout): the cache is an optimisation, and its failure is a
	711	`cache not filled:` line in the deploy's log, never a failed deploy. A warm
	712	start still runs no install.
	713
	714	Uncontained, the install holds the cache's lock for its whole run, since
	715	an install outside a cell (its own lifecycle scripts included) could write
	716	the cache. The cost: two cold environments of one repository install one
	717	after the other there. Inside a cell the cache is read-only, so only the
	718	fill takes the lock and installs run side by side.
	719
701	720	Placement. `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check
702	721	in a cell on the forge's machine. `job` runs install and check as one
703	722	Kubernetes Job (`src/cluster/jobs.ts`) instead: `FORGE_CHECKS_IMAGE` is its
		@@ -19,6 +19,7 @@ import { isReservedAppEnvName } from "../core/app_env.ts";
19	19	import { err, ok, type Result } from "../kernel/result.ts";
20	20	import { spellings, stripForgeDirs } from "../runs/paths.ts";
21	21	import { seconds } from "./timing.ts";
	22	import { cellInstall, prefillInstallCache, withLock } from "../checks/install.ts";
22	23	import {
23	24	commandPid,
24	25	killTree,
		@@ -71,6 +72,14 @@ export interface StartInput {
71	72	* start — the forge coming back, or an environment waking — only runs it.
72	73	*/
73	74	readonly warm?: boolean \| undefined;
	75	/**
	76	* The repository's shared bun cache (`checks/install.ts`
	77	* `checkInstallCache`), the one its checks fill (#7544). The forge fills
	78	* it on the host from the checkout's lockfile, then the install reads it:
	79	* read-only inside a cell, under its lock without one. A checkout the
	80	* fill will not trust (no `bun.lock`) installs as before, into its home.
	81	*/
	82	readonly installCache?: string \| undefined;
74	83	}
75	84
76	85	/** What a caller knows about a cell; the rest comes from the manifest. */
		@@ -380,6 +389,73 @@ export function stepCell(input: StartInput): CellSpec \| undefined {
380	389	};
381	390	}
382	391
	392	/** The forge's own `BUN_CONFIG_REGISTRY`, when its environment names one. */
	393	function forgeRegistry(): Record<string, string> {
	394	const registry = process.env["BUN_CONFIG_REGISTRY"];
	395	return registry === undefined \|\| registry === "" ? {} : { BUN_CONFIG_REGISTRY: registry };
	396	}
	397
	398	/** How an install step runs: plain data, so its composition is testable without a cell. */
	399	export interface InstallPlan {
	400	readonly command: string;
	401	readonly env: Record<string, string>;
	402	readonly cell: CellSpec \| undefined;
	403	/** The lock it holds while it runs, if any. */
	404	readonly lock: string \| undefined;
	405	}
	406
	407	/**
	408	* The install once the fill has answered (#7544): from the repository's
	409	* shared cache when it `filled`, else into the environment's own home as
	410	* before. Inside a cell only the cache is added to its view, read-only;
	411	* uncontained, the install could write the cache, so it takes its turn
	412	* under the cache's lock.
	413	*/
		junipercrew 12d ago outdated A failed fill (err) returns here and fails the deploy at the install stage. Log the error and fall through to the home install instead: `if (!fill.ok) { input.log(`cache not filled: ${fill.error}`); return step({ command, env }, stepCell(input)); }`, then the `!fill.value` case follows the same path. Test it.
	414	export function installPlan(
	415	command: string,
	416	input: StartInput,
	417	env: Record<string, string>,
	418	filled: boolean,
	419	): InstallPlan {
	420	const cache = input.installCache;
	421	const cell = stepCell(input);
	422	if (cache === undefined \|\| !filled) return { command, env, cell, lock: undefined };
	423	if (cell !== undefined)
	424	return { ...cellInstall(command, env, cache), cell: { ...cell, reveal: [...(cell.reveal ?? []), cache] }, lock: undefined };
	425	return { command, env: { ...env, BUN_INSTALL_CACHE_DIR: cache }, cell: undefined, lock: `${cache}.lock` };
	426	}
	427
	428	/**
	429	* The install step. The cache is an optimisation: a fill that fails (the
	430	* registry unreachable, the lock cancelled, a timeout) is logged and the
	431	* install runs into the environment's home, as it did before the cache.
	432	*/
	433	async function install(
	434	command: string,
	435	input: StartInput,
	436	env: Record<string, string>,
	437	): Promise<Result<void, string>> {
	438	let filled = false;
	439	if (input.installCache !== undefined) {
	440	const fill = await prefillInstallCache({
	441	cache: input.installCache,
	442	dir: input.dir,
	443	env,
	444	log: input.log,
	445	signal: input.signal,
	446	timeoutMs: STEP_TIMEOUT_MS,
	447	});
	448	if (fill.ok) filled = fill.value;
	449	else input.log(`cache not filled: ${fill.error}`);
	450	}
	451	const plan = installPlan(command, input, env, filled);
	452	const step = (): Promise<Result<void, string>> =>
	453	runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell });
	454	if (plan.lock === undefined) return step();
	455	const held = await withLock(plan.lock, step, { signal: input.signal });
	456	return held.ok ? held.value : held;
	457	}
	458
383	459	/**
384	460	* Install, then start the app and run its tests side by side, and wait for
385	461	* the app to answer on its port. Returns as soon as it does — `tested`
		@@ -394,16 +470,13 @@ export async function startProcess(
394	470	const stepEnv = childEnv({
395	471	CI: "1",
396	472	FORGE_APP_ENV: input.env === "production" ? "production" : "preview",
	473	// A mirror the forge itself is pointed at is the one the fill and the
	474	// install fetch from (`checks/install.ts` DEFAULT_REGISTRY otherwise).
	475	...forgeRegistry(),
397	476	});
398	477
399	478	if (manifest.install !== null && input.warm !== true) {
400		const installed = await runStep("install", manifest.install, {
401		cwd: input.dir,
402		env: stepEnv,
403		log,
404		signal: input.signal,
405		cell: stepCell(input),
406		});
	479	const installed = await install(manifest.install, input, stepEnv);
407	480	if (!installed.ok)
408	481	return err({ stage: "install", message: installed.error });
409	482	}
		@@ -32,6 +32,7 @@ import { mintAppToken, renewAppToken, revokeAppToken } from "./token.ts";
32	32	import type { ForgeEvent } from "../core/events.ts";
33	33	import { announceRun } from "../actions/runs.ts";
34	34	import { requeueAbandonedChecks } from "../checks/service.ts";
	35	import { checkInstallCache } from "../checks/install.ts";
35	36	import { getPull, listPulls, type Pull } from "../core/pulls.ts";
36	37	import { pullMayRun } from "../core/pull_trust.ts";
37	38	import { canEnterApp, getRepoById, type Repo } from "../core/repos.ts";
		@@ -1403,6 +1404,9 @@ export class AppsRuntime {
1403	1404	log: write,
1404	1405	warm,
1405	1406	cell: this.cellFor(state.app, state.env),
	1407	// Every environment, production too, installs from the cache the
	1408	// repository's checks fill, so a new preview fetches nothing it has.
	1409	installCache: checkInstallCache(this.app.config.dataDir, repo.id),
1406	1410	};
1407	1411	this.noteCell(run.id, write);
1408	1412	clock.lap("setup");
		@@ -0,0 +1,239 @@
	1	/**
	2	* An app environment installs from its repository's shared bun cache
	3	* (#7544): the forge fills it on the host from the lockfile, as checks do,
	4	* so a second environment of the same app at the same lockfile — a new
	5	* preview — installs with its registry dead. A checkout with no `bun.lock`
	6	* installs as it always did, into its own home.
	7	*/
	8	import { describe, expect, setDefaultTimeout, test } from "bun:test";
	9	import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
	10	import { mkdtemp, rm } from "node:fs/promises";
	11	import { tmpdir } from "node:os";
	12	import { join } from "node:path";
	13	import type { AppManifest } from "../src/apps/manifest.ts";
	14	import { installPlan, startProcess, type StartInput } from "../src/apps/process.ts";
	15
	16	setDefaultTimeout(60_000);
	17
	18	const manifest: AppManifest = {
	19	run: `${process.execPath} server.ts`,
	20	install: "bun install --frozen-lockfile",
	21	test: null,
	22	health: "/",
	23	ready: 10,
	24	tty: null,
	25	service: null,
	26	looks: ["/"],
	27	};
	28
	29	const SERVER = `Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("ok") });\n`;
	30
	31	/** A registry on loopback serving one package, `[email protected]`; counts its requests. */
	32	async function registry(dir: string): Promise<{ url: string; hits: string[]; stop: () => void }> {
	33	mkdirSync(join(dir, "pkg", "package"), { recursive: true });
	34	writeFileSync(join(dir, "pkg", "package", "package.json"), JSON.stringify({ name: "fixture-pkg", version: "1.0.0" }));
	35	const made = Bun.spawnSync(["tar", "-czf", join(dir, "fixture.tgz"), "-C", join(dir, "pkg"), "package"]);
	36	if (made.exitCode !== 0) throw new Error("tar is needed for the fixture registry");
	37	const tarball = new Uint8Array(await Bun.file(join(dir, "fixture.tgz")).arrayBuffer());
	38	const hits: string[] = [];
	39	const server: ReturnType<typeof Bun.serve> = Bun.serve({
	40	port: 0,
	41	hostname: "127.0.0.1",
	42	fetch(req): Response {
	43	const path = new URL(req.url).pathname;
	44	hits.push(path);
	45	if (path === "/fixture-pkg/-/fixture-pkg-1.0.0.tgz") return new Response(tarball);
	46	if (path === "/fixture-pkg")
	47	return Response.json({
	48	name: "fixture-pkg",
	49	"dist-tags": { latest: "1.0.0" },
	50	versions: {
	51	"1.0.0": {
	52	name: "fixture-pkg",
	53	version: "1.0.0",
	54	dist: { tarball: `http://127.0.0.1:${server.port}/fixture-pkg/-/fixture-pkg-1.0.0.tgz` },
	55	},
	56	},
	57	});
	58	return new Response("not found", { status: 404 });
	59	},
	60	});
	61	return { url: `http://127.0.0.1:${server.port}/`, hits, stop: () => void server.stop(true) };
	62	}
	63
	64	/** An app's checkout depending on the fixture, with the lockfile bun writes against it. */
	65	async function checkout(dir: string, reg: string): Promise<string> {
	66	const co = join(dir, "first");
	67	mkdirSync(co);
	68	writeFileSync(join(co, "package.json"), JSON.stringify({ name: "x", version: "1.0.0", dependencies: { "fixture-pkg": "1.0.0" } }));
	69	writeFileSync(join(co, "server.ts"), SERVER);
	70	// Async: the fixture registry answers from this process's event loop.
	71	const made = Bun.spawn(["bun", "install", "--lockfile-only"], {
	72	cwd: co,
	73	env: { PATH: process.env["PATH"] ?? "", HOME: dir, BUN_CONFIG_REGISTRY: reg, BUN_INSTALL_CACHE_DIR: join(dir, "lock-cache") },
	74	stdout: "pipe",
	75	stderr: "pipe",
	76	});
	77	if ((await made.exited) !== 0) throw new Error(`fixture lockfile: ${await new Response(made.stderr).text()}`);
	78	return co;
	79	}
	80
	81	/** Run `body` with the forge's own environment pointing bun at `url`, as an operator's mirror would. */
	82	async function withRegistry<T>(url: string, body: () => Promise<T>): Promise<T> {
	83	const before = process.env["BUN_CONFIG_REGISTRY"];
	84	process.env["BUN_CONFIG_REGISTRY"] = url;
	85	try {
	86	return await body();
	87	} finally {
	88	if (before === undefined) delete process.env["BUN_CONFIG_REGISTRY"];
	89	else process.env["BUN_CONFIG_REGISTRY"] = before;
	90	}
	91	}
	92
	93	async function start(dir: string, extra: Partial<StartInput>, lines: string[]): Promise<boolean> {
	94	const started = await startProcess({
	95	dir,
	96	dataDir: dir,
	97	manifest,
	98	env: "production",
	99	url: "http://app.test",
	100	commitOid: "0".repeat(40),
	101	forgeUrl: "http://forge.test",
	102	identityKey: "key",
	103	log: (line) => lines.push(line),
	104	signal: new AbortController().signal,
	105	...extra,
	106	});
	107	if (started.ok) await started.value.handle.stop();
	108	return started.ok;
	109	}
	110
	111	describe("an app environment's install (#7544)", () => {
	112	test("a second environment at the same lockfile installs from the shared cache with the registry dead", async () => {
	113	const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
	114	const reg = await registry(root);
	115	try {
	116	const first = await checkout(root, reg.url);
	117	const second = join(root, "second");
	118	cpSync(first, second, { recursive: true });
	119	const cache = join(root, "shared", "bun");
	120	const lines: string[] = [];
	121
	122	expect(await withRegistry(reg.url, () => start(first, { installCache: cache }, lines))).toBe(true);
	123	expect(reg.hits).toContain("/fixture-pkg/-/fixture-pkg-1.0.0.tgz");
	124	expect(existsSync(join(first, "node_modules", "fixture-pkg", "package.json"))).toBe(true);
	125	// The repository's cache holds it, not the environment's home.
	126	expect(readdirSync(cache).some((name) => name.startsWith("fixture-pkg@"))).toBe(true);
	127
	128	reg.stop();
	129	reg.hits.length = 0;
	130	const again: string[] = [];
	131	expect(await withRegistry(reg.url, () => start(second, { installCache: cache }, again))).toBe(true);
	132	expect(existsSync(join(second, "node_modules", "fixture-pkg", "package.json"))).toBe(true);
	133	expect(reg.hits).toEqual([]);
	134	expect(existsSync(`${cache}.lock`)).toBe(false);
	135	} finally {
	136	reg.stop();
	137	await rm(root, { recursive: true, force: true });
	138	}
	139	});
	140
	141	test("a checkout with no bun.lock installs as before, without the shared cache", async () => {
	142	const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
	143	try {
	144	writeFileSync(join(root, "server.ts"), SERVER);
	145	const cache = join(root, "shared", "bun");
	146	const lines: string[] = [];
	147	const ok = await start(
	148	root,
	149	{
	150	installCache: cache,
	151	manifest: { ...manifest, install: `printf '%s' "\${BUN_INSTALL_CACHE_DIR:-none}" > seen` },
	152	},
	153	lines,
	154	);
	155	expect(ok).toBe(true);
	156	expect(readFileSync(join(root, "seen"), "utf8")).toBe("none");
	157	expect(lines).toContain("cache not filled: no bun lockfile in the checkout");
	158	} finally {
	159	await rm(root, { recursive: true, force: true });
	160	}
	161	});
	162
	163	test("a fill that fails does not fail the deploy: the install runs into its home as before", async () => {
	164	const root = await mkdtemp(join(tmpdir(), "app-install-cache-"));
	165	const reg = await registry(root);
	166	try {
	167	const co = await checkout(root, reg.url);
	168	// A cold cache and a dead registry: the fill cannot fetch.
	169	reg.stop();
	170	const cache = join(root, "shared", "bun");
	171	const lines: string[] = [];
	172	const ok = await withRegistry(reg.url, () =>
	173	start(
	174	co,
	175	{
	176	installCache: cache,
	177	manifest: { ...manifest, install: `printf '%s' "\${BUN_INSTALL_CACHE_DIR:-none}" > seen` },
	178	},
	179	lines,
	180	),
	181	);
	182	expect(ok).toBe(true);
	183	expect(readFileSync(join(co, "seen"), "utf8")).toBe("none");
	184	expect(lines.some((line) => line.startsWith("cache not filled: filling the install cache exited"))).toBe(true);
	185	expect(existsSync(`${cache}.lock`)).toBe(false);
	186	} finally {
	187	reg.stop();
	188	await rm(root, { recursive: true, force: true });
	189	}
	190	});
	191	});
	192
	193	describe("the install's plan (#7544)", () => {
	194	const input = (extra: Partial<StartInput>): StartInput => ({
	195	dir: "/co",
	196	dataDir: "/data",
	197	manifest,
	198	env: "pull-7",
	199	url: "http://app.test",
	200	commitOid: "0".repeat(40),
	201	forgeUrl: "http://forge.test",
	202	identityKey: "key",
	203	log: () => {},
	204	signal: new AbortController().signal,
	205	...extra,
	206	});
	207	const cell = { hide: ["/data"], home: "/homes/app" };
	208	const env = { PATH: "/bin" };
	209
	210	test("in a cell, a filled cache is revealed read-only and the install copies from it", () => {
	211	const plan = installPlan("bun install --frozen-lockfile", input({ cell, installCache: "/cache/bun" }), env, true);
	212	expect(plan.cell?.reveal).toEqual(["/cache/bun"]);
	213	expect(plan.cell?.writable).toEqual([]);
	214	expect(plan.cell?.home).toBe("/homes/app");
	215	expect(plan.command).toBe("bun install --backend=copyfile --frozen-lockfile");
	216	expect(plan.env).toEqual({ PATH: "/bin", BUN_INSTALL_CACHE_DIR: "/cache/bun" });
	217	expect(plan.lock).toBeUndefined();
	218	});
	219
	220	test("uncontained, a filled cache is read under its lock", () => {
	221	const plan = installPlan("bun install", input({ installCache: "/cache/bun" }), env, true);
	222	expect(plan.cell).toBeUndefined();
	223	expect(plan.command).toBe("bun install");
	224	expect(plan.env["BUN_INSTALL_CACHE_DIR"]).toBe("/cache/bun");
	225	expect(plan.lock).toBe("/cache/bun.lock");
	226	});
	227
	228	test("an unfilled cache, or none, leaves the install as it was", () => {
	229	for (const plan of [
	230	installPlan("bun install", input({ cell, installCache: "/cache/bun" }), env, false),
	231	installPlan("bun install", input({ cell }), env, true),
	232	]) {
	233	expect(plan.command).toBe("bun install");
	234	expect(plan.env).toEqual(env);
	235	expect(plan.cell?.reveal).toBeUndefined();
	236	expect(plan.lock).toBeUndefined();
	237	}
	238	});
	239	});
		@@ -24,6 +24,8 @@ import type { Repo } from "../src/core/repos.ts";
24	24	import { listRuns, type Run } from "../src/core/runs.ts";
25	25	import { createUser, type User } from "../src/core/users.ts";
26	26	import { serve, type Servers } from "../src/http/server.ts";
	27	import { checkInstallCache } from "../src/checks/install.ts";
	28	import { readRunLog } from "../src/runs/log.ts";
27	29
28	30	const DOMAIN = "apps.test";
29	31
		@@ -85,10 +87,11 @@ describe("apps adversarial: the install and test steps", () => {
85	87	"the deploy settling",
86	88	);
87	89
88		/** The pull branch carrying only this forge.json, opened against the default branch. */
	90	/** The pull branch carrying this forge.json (and any `extra` files), opened against the default branch. */
89	91	async function pullWith(
90	92	branch: string,
91	93	manifest: Record<string, string>,
	94	extra: readonly { path: string; content: string }[] = [],
92	95	): Promise<{ number: number; commitOid: string }> {
93	96	const committed = await commitToBranch(app, alice, repo, {
94	97	branch,
		@@ -99,6 +102,7 @@ describe("apps adversarial: the install and test steps", () => {
99	102	path: "forge.json",
100	103	content: `${JSON.stringify({ app: manifest }, null, 2)}\n`,
101	104	},
	105	...extra,
102	106	],
103	107	});
104	108	if (!committed.ok) throw new Error(JSON.stringify(committed.error));
		@@ -176,6 +180,49 @@ describe("apps adversarial: the install and test steps", () => {
176	180	);
177	181	}, 20_000);
178	182
	183	test("a preview's install reads its repository's shared cache (#7544)", async () => {
	184	// A local dependency: a real bun.lock the fill trusts, and nothing fetched.
	185	const pull = await pullWith(
	186	"shared-cache",
	187	{
	188	install: `bun install --frozen-lockfile && echo "install cache: \${BUN_INSTALL_CACHE_DIR:-none}"`,
	189	run: "bun run src/server.ts",
	190	},
	191	[
	192	{ path: "dep/package.json", content: `{"name":"dep","version":"1.0.0"}\n` },
	193	{
	194	path: "package.json",
	195	content: `${JSON.stringify({ name: "steps", version: "0.1.0", private: true, dependencies: { dep: "file:./dep" } })}\n`,
	196	},
	197	{
	198	path: "bun.lock",
	199	content: `{
	200	"lockfileVersion": 1,
	201	"configVersion": 1,
	202	"workspaces": {
	203	"": {
	204	"name": "steps",
	205	"dependencies": {
	206	"dep": "file:./dep",
	207	},
	208	},
	209	},
	210	"packages": {
	211	"dep": ["dep@file:dep", {}],
	212	}
	213	}
	214	`,
	215	},
	216	],
	217	);
	218	const run = await settled(
	219	(r) => r.env === `pull-${pull.number}` && r.commitOid === pull.commitOid,
	220	);
	221	const log = readRunLog(dataDir, run.id).lines;
	222	expect(run.state).toBe("ok");
	223	expect(log).toContain(`install cache: ${checkInstallCache(dataDir, repo.id)}`);
	224	}, 20_000);
	225
179	226	test("closing a pull whose deploy already answered kills its still-running test step — no orphan `bun test`", async () => {
180	227	// A merge lands this same forge.json on the default branch and
181	228	// production redeploys it too; only the preview's test step sleeps, so