c65394b · b11e149 · 0d8a5a2 · ffd6747 · deb87f9 · 84558b4
@@ -17,6 +17,7 @@
and user accounts. The child mints its own. A test in #420 and #422 proves it.
| #420  | spawn plan: one validated value, dry-run                | #383             |
| #427  | placement seam; hosted inside the parent today          | #420             |
| #2926 | `cell` placement: a child as its own `bun src/main.ts` in a cell held to its size (`spawn/cell.ts`); `hosted` where the host cannot hold a cell. The spawn answer and the children list say `placement` | #2925 |
| #6023 | a cell child boots inside a forge that is itself in a cell (nothing under the cell's own `/run` is remounted read-only); its cell starts behind the first request, which is answered 503 with `Retry-After` until it is up, and a cell that fails turns the child hosted; the terminal's family lines say where each child runs | #2926 |
| #421  | spawn action: org, instance, apps as repos, briefs      | #420, #427       |
`src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token).
@@ -390,6 +390,42 @@
function origin(url: string): { host: string; port: number } | undefined {
const isLoopback = (host: string): boolean =>
  host === "localhost" || host === "::1" || /^127\./.test(host);
/** A path at or under `root`. */
const within = (path: string, root: string): boolean => path === root || path.startsWith(`${root}/`);
/**
 * What a cell covers and what it shows again, from what the caller names
 * and what the machine mounts (`hiddenMounts`). Deepest last, so a hidden
 * directory never covers a writable one inside it. `/tmp` and `/run` are
 * fresh in every cell, so nothing under them is the machine's: a forge
 * that itself runs in a cell has its own `/run/forge` mounted there, and
 * remounting that read-only would leave the inner cell no socket to listen on.
 */
export function cellMounts(
  spec: Pick<CellSpec, "hide" | "reveal">,
  machine: readonly string[],
  kind: (path: string) => "dir" | "file" | undefined = pathKind,
): { hide: string[]; blank: string[]; reveal: string[] } {
  const hidden = [...new Set([...spec.hide, ...machine])]
    // A path that is not there has nothing to hide (and cannot be mounted
    // over on a read-only root).
    .filter(
      (path) =>
        !["", "/"].includes(path) && !within(path, "/tmp") && !within(path, "/run") && kind(path) !== undefined,
    )
    .sort((a, b) => a.length - b.length);
  // A directory is covered by an empty read-only tmpfs; a file (a secret
  // mounted by itself) by /dev/null.
  const hide = hidden.filter((path) => kind(path) === "dir");
  const blank = hidden.filter((path) => kind(path) === "file");
  // The runtime itself must stay visible when it lives in a hidden home
  // (`~/.bun/bin/bun` on a developer's machine).
  const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) =>
    hide.some((h) => within(path, h)),
  );
  return { hide, blank, reveal };
}
/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
export function openCell(
  spec: CellSpec,
@@ -424,26 +460,7 @@
export function openCell(
      relay({ tcp: spec.listen }, { unix: join(sockets, "in.sock") }),
    );
  // Deepest last, so a hidden directory never covers a writable one inside it.
  // What the caller names, plus whatever the machine mounts that may carry
  // a secret (`hiddenMounts`): a cell never needs either.
  const hidden = [...new Set([...spec.hide, ...hiddenMounts()])]
    // A path that is not there has nothing to hide (and cannot be mounted
    // over on a read-only root). `/tmp` and `/run` are already the cell's own.
    .filter(
      (path) =>
        !["", "/", "/tmp", "/run"].includes(path) && existsSync(path),
    )
    .sort((a, b) => a.length - b.length);
  // A directory is covered by an empty read-only tmpfs; a file (a secret
  // mounted by itself) by /dev/null.
  const hide = hidden.filter((path) => isDirectory(path));
  const blank = hidden.filter((path) => !isDirectory(path));
  // The runtime itself must stay visible when it lives in a hidden home
  // (`~/.bun/bin/bun` on a developer's machine).
  const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) =>
    hide.some((h) => path === h || path.startsWith(`${h}/`)),
  );
  const { hide, blank, reveal } = cellMounts(spec, hiddenMounts());
  const writable = [...new Set([spec.dir, spec.home, ...spec.writable])];
  for (const path of writable) mkdirSync(path, { recursive: true });
  // tmpfs is memory the watch cannot see from outside, so every one is
@@ -541,6 +558,12 @@
export function openCell(
  };
}
/** What is at `path`: a directory, a file, or nothing. */
function pathKind(path: string): "dir" | "file" | undefined {
  if (!existsSync(path)) return undefined;
  return isDirectory(path) ? "dir" : "file";
}
/** Whether `path` is a directory (following links, as bubblewrap will). */
function isDirectory(path: string): boolean {
  try {
@@ -97,6 +97,8 @@
export function familyLines(view: FamilyView): [string, string][] {
            c.severedAt !== null ? `follows nothing since ${c.severedAt.slice(0, 10)}` : c.forgeTip === null ? "" : `code at ${c.forgeTip.slice(0, 7)}`,
            c.openProposals === null ? "" : `${c.openProposals} open`,
          ].filter((b) => b !== "");
    // Where it runs, as `/forges` says it: "own cell", "hosted", or why it is hosted.
    if (c.placement !== null) bits.push(c.placement);
    out.push([`child ${c.name}`, bits.join(" · ")]);
    const follow = followLine(c);
    if (follow !== null) out.push(["", follow]);
@@ -22,7 +22,8 @@
import {
  type PlacementKind,
} from "./placement.ts";
import { DEFAULT_SIZE, isSize, type Size } from "./sizes.ts";
import { cellChildren, forward } from "./cell.ts";
import { cellChildren, forward, type CellChild } from "./cell.ts";
import { cellsAvailable } from "../apps/contain.ts";
import { readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
@@ -118,23 +119,49 @@
export async function resolveHostedChild(
  if (name === undefined) return undefined;
  const ledger = readLedgerSync(parent, name);
  if (ledger === undefined || !ledger.done) return "unknown";
  // A child in its own cell is not an App here: the request is carried to it.
  if (ledger.placement === "cell") return settleCell(parent, ledger);
  // A child in its own cell is not an App here: the request is carried to
  // it, and while its cell is not up `answerFromCell` says so at once.
  if (ledger.placement === "cell") {
    if (cellChildren(parent).get(ledger.bodyId) !== undefined) return "cell";
    // A host that holds no cell at all is known without waiting.
    const cells = await cellsAvailable();
    if (!cells.ok) return openOnce(parent, demoted(parent, ledger, cells.reason));
    void settleCell(parent, ledger);
    return "cell";
  }
  return openOnce(parent, ledger);
}
/** Starts in flight, by body id: one demotion however many requests asked. */
const settling = new WeakMap<App, Map<string, Promise<CellChild | undefined>>>();
/**
 * Start a child's cell. When the cell cannot come up the child is not left
 * dead: the ledger turns it hosted, says why (`placementNote`, which
 * `/forges` lists), and the child is opened in this process.
 * dead: the ledger turns it hosted and says why (`placementNote`, which
 * `/forges` lists), and the next request opens it in this process.
 */
async function settleCell(parent: App, ledger: ChildLedger): Promise<App | "cell"> {
  const cells = cellChildren(parent);
  if ((await cells.start(ledger)) !== undefined) return "cell";
  const why = cells.failure(ledger.bodyId) ?? "its cell did not start";
function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild | undefined> {
  let flights = settling.get(parent);
  if (!flights) settling.set(parent, (flights = new Map()));
  let flight = flights.get(ledger.bodyId);
  if (!flight) {
    const cells = cellChildren(parent);
    flight = cells.start(ledger).then((child) => {
      if (child === undefined) demoted(parent, ledger, cells.failure(ledger.bodyId) ?? "its cell did not start");
      return child;
    });
    const mine = flights;
    flight.finally(() => mine.delete(ledger.bodyId)).catch(() => {});
    flights.set(ledger.bodyId, flight);
  }
  return flight;
}
/** Record that a cell child is hosted instead, and why; the ledger as it now reads. */
function demoted(parent: App, ledger: ChildLedger, why: string): ChildLedger {
  demote(parent, ledger.name, why);
  parent.log.warn("child forge placed hosted instead", { name: ledger.name, why });
  return openOnce(parent, { ...ledger, placement: "hosted" });
  return { ...ledger, placement: "hosted" };
}
function demote(parent: App, name: string, why: string): void {
@@ -174,8 +201,7 @@
export async function bootHostedChild(parent: App, name: string): Promise<void>
  let ledger = readLedgerSync(parent, name);
  if (ledger === undefined || !ledger.done) return;
  if (ledger.placement === "cell") {
    const settled = await settleCell(parent, ledger);
    if (settled === "cell") return;
    if ((await settleCell(parent, ledger)) !== undefined) return;
    ledger = { ...ledger, placement: "hosted" };
  }
  try {
@@ -245,15 +271,18 @@
export async function resolveHostedChildApp(
export const hostedChildApps = childAppsSummary;
/**
 * Answer a request for a child that runs in its own cell: start it if it is
 * not up (a parent that restarted, a child that stopped), then forward.
 * Answer a request for a child that runs in its own cell: forward when it
 * is up, and when it is not (just spawned, a parent that restarted, a
 * child that stopped) start it and answer 503 at once.
 */
export async function answerFromCell(parent: App, request: Request): Promise<Response> {
  const name = parseHostedChildName(request.headers.get("host") ?? "", childHost(parent));
  const ledger = name === undefined ? undefined : readLedgerSync(parent, name);
  if (ledger === undefined || ledger.placement !== "cell") return new Response("not found", { status: 404 });
  const child = await cellChildren(parent).start(ledger);
  // Unreachable once a cell has failed (the ledger turned hosted), but never a hang.
  if (child === undefined) return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5" } });
  return forward(child, request, parent.config.clientIpHeader);
  const child = cellChildren(parent).get(ledger.bodyId);
  if (child !== undefined) return forward(child, request, parent.config.clientIpHeader);
  // Never a wait on a cell that may not come up: it starts behind this
  // answer, and once it has failed the ledger says hosted.
  void settleCell(parent, ledger);
  return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5", "cache-control": "no-store" } });
}
@@ -21,6 +21,7 @@
import { openPull, submitReview } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
import {
  cellCounters,
  cellMounts,
  cellsAvailable,
  hiddenMounts,
  mountMode,
@@ -474,6 +475,22 @@
describe("cells hide every mount that may carry a secret", () => {
    ])
      expect(hidden).not.toContain(kept);
  });
  // A forge that runs in a cell (a preview) has its own sockets at
  // /run/forge; a child forge's cell inside it remounted them read-only and
  // never had a socket to listen on (#6023).
  test("nothing under the cell's own /tmp and /run is hidden again, so a cell inside a cell keeps its sockets", () => {
    const dirs = new Set(["/data", "/run/forge", "/run/secrets", "/tmp/fc-1", "/etc/plat-ca", "/data/code"]);
    const kind = (path: string) => (dirs.has(path) ? "dir" : path === "/etc/token" ? "file" : undefined);
    const mounts = cellMounts(
      { hide: ["/data"], reveal: ["/data/code"] },
      ["/run/forge", "/run/secrets", "/tmp", "/tmp/fc-1", "/run", "/etc/plat-ca", "/etc/token", "/var/run/gone"],
      kind,
    );
    expect(mounts.hide).toEqual(["/data", "/etc/plat-ca"]);
    expect(mounts.blank).toEqual(["/etc/token"]);
    expect(mounts.reveal).toContain("/data/code");
  });
});
describe("cells are labelled for the machine sampler (boards/sampler.ts)", () => {
@@ -245,6 +245,17 @@
describe("family heartbeats", () => {
    expect((await familyView(parent, true, () => ({ total: 0, running: 0, notRunning: [] }))).children[0]?.apps).toBeNull();
  });
  test("the terminal's family lines say where each child runs, as /forges does", async () => {
    const view = await familyView(parent, true);
    const lineOf = (v: typeof view) => familyLines(v).find(([k]) => k === "child maria")?.[1] ?? "";
    expect(view.children[0]?.placement).toBe("hosted");
    expect(lineOf(view)).toEndWith(" · hosted");
    const placed = (placement: string) => ({ ...view, children: view.children.map((c) => ({ ...c, placement })) });
    expect(lineOf(placed("own cell"))).toEndWith(" · own cell");
    expect(lineOf(placed("hosted: its cell did not come up"))).toEndWith(" · hosted: its cell did not come up");
    expect(lineOf({ ...view, children: view.children.map((c) => ({ ...c, placement: null })) })).not.toContain("hosted");
  });
  test("the terminal's Family card shows the apps line", async () => {
    const view = await familyView(parent, true, two);
    const { app } = createForgeApp(
@@ -0,0 +1,57 @@
/**
 * A real child forge in its own cell, as `test/spawn-cell.test.ts` drives
 * it. Its own process, because a check sets `FORGE_CELLS=off` for the
 * forges its tests boot (`checks/service.ts`) and the cell probe is asked
 * once per process; the test starts this one without it.
 *
 * The parent holds a finished `cell` ledger for `kid`. The first request
 * is answered before the cell is up; then the cell is awaited and `/` and
 * `/healthz` are carried to it. Printed: one JSON line of what was seen.
 */
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../../src/app.ts";
import { cellCounters, cellsAvailable } from "../../src/apps/contain.ts";
import { cellChildren, stopCellChildren } from "../../src/spawn/cell.ts";
import { answerFromCell } from "../../src/spawn/host.ts";
const cells = await cellsAvailable();
if (!cells.ok) {
  console.log(JSON.stringify({ skipped: cells.reason }));
  process.exit(0);
}
const dataDir = mkdtempSync(join(tmpdir(), "real-child-"));
const parent = createApp({ dataDir, url: "https://f.test" });
const ledger = { name: "kid", bodyId: "cell-real", size: "small" } as const;
mkdirSync(join(dataDir, "spawn"), { recursive: true });
writeFileSync(join(dataDir, "spawn", "kid.json"), JSON.stringify({ ...ledger, done: true, placement: "cell" }));
const ask = (path: string) => answerFromCell(parent, new Request(`https://kid--f.test${path}`, { headers: { host: "kid--f.test" } }));
try {
  const started = performance.now();
  const first = await ask("/");
  const firstMs = Math.round(performance.now() - started);
  const children = cellChildren(parent);
  const child = await children.start(ledger);
  const root = child === undefined ? null : (await ask("/")).status;
  const health = child === undefined ? null : (await ask("/healthz")).status;
  // The watch reads the cell's tree once a second, by its label.
  await Bun.sleep(1_500);
  console.log(
    JSON.stringify({
      first: first.status,
      firstMs,
      failure: children.failure(ledger.bodyId) ?? null,
      limits: child?.limits ?? null,
      root,
      health,
      labelled: cellCounters().some((c) => c.label === "forge:kid"),
    }),
  );
} finally {
  await stopCellChildren(parent);
  parent.close();
  rmSync(dataDir, { recursive: true, force: true });
}
@@ -3,7 +3,7 @@
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
import { tmpdir } from "node:os";
import { join } from "node:path";
import { askUntilBelieved, cellsAvailable, openCell } from "../src/apps/contain.ts";
import { closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts";
import { answerFromCell, closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts";
import { createApp } from "../src/app.ts";
import { cellChildren, stopCellChildren, cellEnv, cellLimits, cellPlacement, cellSpecFor, forward, ownsBody } from "../src/spawn/cell.ts";
import { fallbackPlacement, hostedPlacement, memoryBodies } from "../src/spawn/placement.ts";
@@ -11,6 +11,9 @@
import type { SpawnPlan } from "../src/spawn/plan.ts";
import { SIZES } from "../src/spawn/sizes.ts";
import { notePeer } from "../src/http/peer.ts";
import type { App } from "../src/app.ts";
import { portInsteadOfSocket } from "../src/git/hook-address.ts";
import { missing } from "./helpers/tools.ts";
import { expectBudget } from "./helpers/budget.ts";
import { openBeatDoor, parentLocal } from "../src/spawn/beat-door.ts";
import { lastHeard, sendBeat } from "../src/spawn/heartbeat.ts";
@@ -251,23 +254,36 @@
describe("a child that will not start", () => {
  });
});
// Where bwrap and socat are here, whatever FORGE_CELLS says: a check turns
// cells off for the forges its tests boot, and this is the one test that
// must boot a child in a cell anyway (#6023). In its own process, without it.
const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !missing("bwrap", "socat");
describe("a real child forge", () => {
  test.skipIf(!cells.ok)("boots in its cell and answers /healthz through the forward", async () => {
    const parent = createApp({ dataDir: join(tmp, "real-parent"), url: "https://f.test" });
  test.skipIf(!cellTools)("boots in its own forge:<name> cell and answers / through the forward; its first request does not wait for it", async () => {
    const env: Record<string, string> = {};
    for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v;
    const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-child-forge.ts")], { env, stdout: "pipe", stderr: "pipe" });
    const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
    const line = out.trim().split("\n").pop() ?? "";
    let seen: Record<string, unknown>;
    try {
      const children = cellChildren(parent);
      const child = await children.start({ bodyId: "cell-real", name: "kid", size: "small" });
      expect(children.failure("cell-real")).toBeUndefined();
      if (child === undefined) throw new Error(`the child did not start: ${children.failure("cell-real")}`);
      expect(child.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
      const res = await forward(child, new Request("http://kid--f.test/healthz", { headers: { host: "kid--f.test" } }));
      expect(res.status).toBe(200);
      await children.stop("cell-real");
    } finally {
      await stopCellChildren(parent);
      parent.close();
      seen = JSON.parse(line) as Record<string, unknown>;
    } catch {
      throw new Error(`the child forge run printed no result: ${out.slice(-800)} ${errText.slice(-800)}`);
    }
  }, 60_000);
    if (typeof seen["skipped"] === "string") {
      console.log(`SKIPPING: this host refuses a cell inside the test (${seen["skipped"]}), so no child forge was booted in one`);
      return;
    }
    expect(seen["failure"]).toBeNull();
    expect(seen["first"]).toBe(503);
    expect(seen["firstMs"] as number).toBeLessThan(1_000);
    expect(seen["limits"]).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
    expect(seen["root"] as number).toBeLessThan(400);
    expect(seen["health"]).toBe(200);
    expect(seen["labelled"]).toBe(true);
  }, 90_000);
  test.skipIf(!cells.ok)("its heartbeat reaches the parent through the cell, with no public address", async () => {
    // The parent's public url is not reachable from here, as in a preview's cell.
@@ -313,4 +329,33 @@
describe("a child whose cell will not come up", () => {
      parent.close();
    }
  });
  test.skipIf(cells.ok)("its first request is answered 503 at once, and once the cell has failed the next is hosted", async () => {
    const dataDir = join(tmp, "fast-parent");
    const parent = createApp({ dataDir, url: "https://f.test" });
    const request = () => new Request("https://kid--f.test/", { headers: { host: "kid--f.test" } });
    try {
      mkdirSync(join(dataDir, "spawn"), { recursive: true });
      const file = join(dataDir, "spawn", "kid.json");
      writeFileSync(file, JSON.stringify({ name: "kid", bodyId: "cell-f", done: true, placement: "cell", size: "small" }));
      // The baseline: a child with no ledger, answered from the same read with no start.
      const unknown = performance.now();
      expect((await answerFromCell(parent, new Request("https://nobody--f.test/", { headers: { host: "nobody--f.test" } }))).status).toBe(404);
      const against = performance.now() - unknown;
      const started = performance.now();
      const first = await answerFromCell(parent, request());
      expectBudget(performance.now() - started, { ms: 50, against });
      expect(first.status).toBe(503);
      expect(first.headers.get("retry-after")).toBe("5");
      // The start it set off fails here (no cells), and the ledger turns hosted.
      for (let i = 0; i < 50 && JSON.parse(readFileSync(file, "utf8")).placement !== "hosted"; i++) await Bun.sleep(10);
      expect(JSON.parse(readFileSync(file, "utf8")).placement).toBe("hosted");
      const next = await resolveHostedChild(parent, request());
      expect(next).not.toBe("cell");
      expect(next).not.toBe("unknown");
    } finally {
      await closeHostedChildren(parent);
      parent.close();
    }
  });
});
		@@ -17,6 +17,7 @@ and user accounts. The child mints its own. A test in #420 and #422 proves it.
17	17	\| #420 \| spawn plan: one validated value, dry-run \| #383 \|
18	18	\| #427 \| placement seam; hosted inside the parent today \| #420 \|
19	19	\| #2926 \| `cell` placement: a child as its own `bun src/main.ts` in a cell held to its size (`spawn/cell.ts`); `hosted` where the host cannot hold a cell. The spawn answer and the children list say `placement` \| #2925 \|
	20	\| #6023 \| a cell child boots inside a forge that is itself in a cell (nothing under the cell's own `/run` is remounted read-only); its cell starts behind the first request, which is answered 503 with `Retry-After` until it is up, and a cell that fails turns the child hosted; the terminal's family lines say where each child runs \| #2926 \|
20	21	\| #421 \| spawn action: org, instance, apps as repos, briefs \| #420, #427 \|
21	22
22	23	`src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token).
		@@ -390,6 +390,42 @@ function origin(url: string): { host: string; port: number } \| undefined {
390	390	const isLoopback = (host: string): boolean =>
391	391	host === "localhost" \|\| host === "::1" \|\| /^127\./.test(host);
392	392
	393	/** A path at or under `root`. */
	394	const within = (path: string, root: string): boolean => path === root \|\| path.startsWith(`${root}/`);
	395
	396	/**
	397	* What a cell covers and what it shows again, from what the caller names
	398	* and what the machine mounts (`hiddenMounts`). Deepest last, so a hidden
	399	* directory never covers a writable one inside it. `/tmp` and `/run` are
	400	* fresh in every cell, so nothing under them is the machine's: a forge
	401	* that itself runs in a cell has its own `/run/forge` mounted there, and
	402	* remounting that read-only would leave the inner cell no socket to listen on.
	403	*/
	404	export function cellMounts(
	405	spec: Pick<CellSpec, "hide" \| "reveal">,
	406	machine: readonly string[],
	407	kind: (path: string) => "dir" \| "file" \| undefined = pathKind,
	408	): { hide: string[]; blank: string[]; reveal: string[] } {
	409	const hidden = [...new Set([...spec.hide, ...machine])]
	410	// A path that is not there has nothing to hide (and cannot be mounted
	411	// over on a read-only root).
	412	.filter(
	413	(path) =>
	414	!["", "/"].includes(path) && !within(path, "/tmp") && !within(path, "/run") && kind(path) !== undefined,
	415	)
	416	.sort((a, b) => a.length - b.length);
	417	// A directory is covered by an empty read-only tmpfs; a file (a secret
	418	// mounted by itself) by /dev/null.
	419	const hide = hidden.filter((path) => kind(path) === "dir");
	420	const blank = hidden.filter((path) => kind(path) === "file");
	421	// The runtime itself must stay visible when it lives in a hidden home
	422	// (`~/.bun/bin/bun` on a developer's machine).
	423	const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) =>
	424	hide.some((h) => within(path, h)),
	425	);
	426	return { hide, blank, reveal };
	427	}
	428
393	429	/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
394	430	export function openCell(
395	431	spec: CellSpec,
		@@ -424,26 +460,7 @@ export function openCell(
424	460	relay({ tcp: spec.listen }, { unix: join(sockets, "in.sock") }),
425	461	);
426	462
427		// Deepest last, so a hidden directory never covers a writable one inside it.
428		// What the caller names, plus whatever the machine mounts that may carry
429		// a secret (`hiddenMounts`): a cell never needs either.
430		const hidden = [...new Set([...spec.hide, ...hiddenMounts()])]
431		// A path that is not there has nothing to hide (and cannot be mounted
432		// over on a read-only root). `/tmp` and `/run` are already the cell's own.
433		.filter(
434		(path) =>
435		!["", "/", "/tmp", "/run"].includes(path) && existsSync(path),
436		)
437		.sort((a, b) => a.length - b.length);
438		// A directory is covered by an empty read-only tmpfs; a file (a secret
439		// mounted by itself) by /dev/null.
440		const hide = hidden.filter((path) => isDirectory(path));
441		const blank = hidden.filter((path) => !isDirectory(path));
442		// The runtime itself must stay visible when it lives in a hidden home
443		// (`~/.bun/bin/bun` on a developer's machine).
444		const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) =>
445		hide.some((h) => path === h \|\| path.startsWith(`${h}/`)),
446		);
	463	const { hide, blank, reveal } = cellMounts(spec, hiddenMounts());
447	464	const writable = [...new Set([spec.dir, spec.home, ...spec.writable])];
448	465	for (const path of writable) mkdirSync(path, { recursive: true });
449	466	// tmpfs is memory the watch cannot see from outside, so every one is
		@@ -541,6 +558,12 @@ export function openCell(
541	558	};
542	559	}
543	560
	561	/** What is at `path`: a directory, a file, or nothing. */
	562	function pathKind(path: string): "dir" \| "file" \| undefined {
	563	if (!existsSync(path)) return undefined;
	564	return isDirectory(path) ? "dir" : "file";
	565	}
	566
544	567	/** Whether `path` is a directory (following links, as bubblewrap will). */
545	568	function isDirectory(path: string): boolean {
546	569	try {
		@@ -97,6 +97,8 @@ export function familyLines(view: FamilyView): [string, string][] {
97	97	c.severedAt !== null ? `follows nothing since ${c.severedAt.slice(0, 10)}` : c.forgeTip === null ? "" : `code at ${c.forgeTip.slice(0, 7)}`,
98	98	c.openProposals === null ? "" : `${c.openProposals} open`,
99	99	].filter((b) => b !== "");
	100	// Where it runs, as `/forges` says it: "own cell", "hosted", or why it is hosted.
	101	if (c.placement !== null) bits.push(c.placement);
100	102	out.push([`child ${c.name}`, bits.join(" · ")]);
101	103	const follow = followLine(c);
102	104	if (follow !== null) out.push(["", follow]);
		@@ -22,7 +22,8 @@ import {
22	22	type PlacementKind,
23	23	} from "./placement.ts";
24	24	import { DEFAULT_SIZE, isSize, type Size } from "./sizes.ts";
25		import { cellChildren, forward } from "./cell.ts";
	25	import { cellChildren, forward, type CellChild } from "./cell.ts";
	26	import { cellsAvailable } from "../apps/contain.ts";
26	27	import { readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
27	28	import { join } from "node:path";
28	29
		@@ -118,23 +119,49 @@ export async function resolveHostedChild(
118	119	if (name === undefined) return undefined;
119	120	const ledger = readLedgerSync(parent, name);
120	121	if (ledger === undefined \|\| !ledger.done) return "unknown";
121		// A child in its own cell is not an App here: the request is carried to it.
122		if (ledger.placement === "cell") return settleCell(parent, ledger);
	122	// A child in its own cell is not an App here: the request is carried to
	123	// it, and while its cell is not up `answerFromCell` says so at once.
	124	if (ledger.placement === "cell") {
	125	if (cellChildren(parent).get(ledger.bodyId) !== undefined) return "cell";
	126	// A host that holds no cell at all is known without waiting.
	127	const cells = await cellsAvailable();
	128	if (!cells.ok) return openOnce(parent, demoted(parent, ledger, cells.reason));
	129	void settleCell(parent, ledger);
	130	return "cell";
	131	}
123	132	return openOnce(parent, ledger);
124	133	}
125	134
	135	/** Starts in flight, by body id: one demotion however many requests asked. */
	136	const settling = new WeakMap<App, Map<string, Promise<CellChild \| undefined>>>();
	137
126	138	/**
127	139	* Start a child's cell. When the cell cannot come up the child is not left
128		* dead: the ledger turns it hosted, says why (`placementNote`, which
129		* `/forges` lists), and the child is opened in this process.
	140	* dead: the ledger turns it hosted and says why (`placementNote`, which
	141	* `/forges` lists), and the next request opens it in this process.
130	142	*/
131		async function settleCell(parent: App, ledger: ChildLedger): Promise<App \| "cell"> {
132		const cells = cellChildren(parent);
133		if ((await cells.start(ledger)) !== undefined) return "cell";
134		const why = cells.failure(ledger.bodyId) ?? "its cell did not start";
	143	function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild \| undefined> {
	144	let flights = settling.get(parent);
	145	if (!flights) settling.set(parent, (flights = new Map()));
	146	let flight = flights.get(ledger.bodyId);
	147	if (!flight) {
	148	const cells = cellChildren(parent);
	149	flight = cells.start(ledger).then((child) => {
	150	if (child === undefined) demoted(parent, ledger, cells.failure(ledger.bodyId) ?? "its cell did not start");
	151	return child;
	152	});
	153	const mine = flights;
	154	flight.finally(() => mine.delete(ledger.bodyId)).catch(() => {});
	155	flights.set(ledger.bodyId, flight);
	156	}
	157	return flight;
	158	}
	159
	160	/** Record that a cell child is hosted instead, and why; the ledger as it now reads. */
	161	function demoted(parent: App, ledger: ChildLedger, why: string): ChildLedger {
135	162	demote(parent, ledger.name, why);
136	163	parent.log.warn("child forge placed hosted instead", { name: ledger.name, why });
137		return openOnce(parent, { ...ledger, placement: "hosted" });
	164	return { ...ledger, placement: "hosted" };
138	165	}
139	166
140	167	function demote(parent: App, name: string, why: string): void {
		@@ -174,8 +201,7 @@ export async function bootHostedChild(parent: App, name: string): Promise<void>
174	201	let ledger = readLedgerSync(parent, name);
175	202	if (ledger === undefined \|\| !ledger.done) return;
176	203	if (ledger.placement === "cell") {
177		const settled = await settleCell(parent, ledger);
178		if (settled === "cell") return;
	204	if ((await settleCell(parent, ledger)) !== undefined) return;
179	205	ledger = { ...ledger, placement: "hosted" };
180	206	}
181	207	try {
		@@ -245,15 +271,18 @@ export async function resolveHostedChildApp(
245	271	export const hostedChildApps = childAppsSummary;
246	272
247	273	/**
248		* Answer a request for a child that runs in its own cell: start it if it is
249		* not up (a parent that restarted, a child that stopped), then forward.
	274	* Answer a request for a child that runs in its own cell: forward when it
	275	* is up, and when it is not (just spawned, a parent that restarted, a
	276	* child that stopped) start it and answer 503 at once.
250	277	*/
251	278	export async function answerFromCell(parent: App, request: Request): Promise<Response> {
252	279	const name = parseHostedChildName(request.headers.get("host") ?? "", childHost(parent));
253	280	const ledger = name === undefined ? undefined : readLedgerSync(parent, name);
254	281	if (ledger === undefined \|\| ledger.placement !== "cell") return new Response("not found", { status: 404 });
255		const child = await cellChildren(parent).start(ledger);
256		// Unreachable once a cell has failed (the ledger turned hosted), but never a hang.
257		if (child === undefined) return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5" } });
258		return forward(child, request, parent.config.clientIpHeader);
	282	const child = cellChildren(parent).get(ledger.bodyId);
	283	if (child !== undefined) return forward(child, request, parent.config.clientIpHeader);
	284	// Never a wait on a cell that may not come up: it starts behind this
	285	// answer, and once it has failed the ledger says hosted.
	286	void settleCell(parent, ledger);
	287	return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5", "cache-control": "no-store" } });
259	288	}
		@@ -21,6 +21,7 @@ import { openPull, submitReview } from "../src/actions/pulls.ts";
21	21	import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
22	22	import {
23	23	cellCounters,
	24	cellMounts,
24	25	cellsAvailable,
25	26	hiddenMounts,
26	27	mountMode,
		@@ -474,6 +475,22 @@ describe("cells hide every mount that may carry a secret", () => {
474	475	])
475	476	expect(hidden).not.toContain(kept);
476	477	});
	478
	479	// A forge that runs in a cell (a preview) has its own sockets at
	480	// /run/forge; a child forge's cell inside it remounted them read-only and
	481	// never had a socket to listen on (#6023).
	482	test("nothing under the cell's own /tmp and /run is hidden again, so a cell inside a cell keeps its sockets", () => {
	483	const dirs = new Set(["/data", "/run/forge", "/run/secrets", "/tmp/fc-1", "/etc/plat-ca", "/data/code"]);
	484	const kind = (path: string) => (dirs.has(path) ? "dir" : path === "/etc/token" ? "file" : undefined);
	485	const mounts = cellMounts(
	486	{ hide: ["/data"], reveal: ["/data/code"] },
	487	["/run/forge", "/run/secrets", "/tmp", "/tmp/fc-1", "/run", "/etc/plat-ca", "/etc/token", "/var/run/gone"],
	488	kind,
	489	);
	490	expect(mounts.hide).toEqual(["/data", "/etc/plat-ca"]);
	491	expect(mounts.blank).toEqual(["/etc/token"]);
	492	expect(mounts.reveal).toContain("/data/code");
	493	});
477	494	});
478	495
479	496	describe("cells are labelled for the machine sampler (boards/sampler.ts)", () => {
		@@ -245,6 +245,17 @@ describe("family heartbeats", () => {
245	245	expect((await familyView(parent, true, () => ({ total: 0, running: 0, notRunning: [] }))).children[0]?.apps).toBeNull();
246	246	});
247	247
	248	test("the terminal's family lines say where each child runs, as /forges does", async () => {
	249	const view = await familyView(parent, true);
	250	const lineOf = (v: typeof view) => familyLines(v).find(([k]) => k === "child maria")?.[1] ?? "";
	251	expect(view.children[0]?.placement).toBe("hosted");
	252	expect(lineOf(view)).toEndWith(" · hosted");
	253	const placed = (placement: string) => ({ ...view, children: view.children.map((c) => ({ ...c, placement })) });
	254	expect(lineOf(placed("own cell"))).toEndWith(" · own cell");
	255	expect(lineOf(placed("hosted: its cell did not come up"))).toEndWith(" · hosted: its cell did not come up");
	256	expect(lineOf({ ...view, children: view.children.map((c) => ({ ...c, placement: null })) })).not.toContain("hosted");
	257	});
	258
248	259	test("the terminal's Family card shows the apps line", async () => {
249	260	const view = await familyView(parent, true, two);
250	261	const { app } = createForgeApp(
		@@ -0,0 +1,57 @@
	1	/**
	2	* A real child forge in its own cell, as `test/spawn-cell.test.ts` drives
	3	* it. Its own process, because a check sets `FORGE_CELLS=off` for the
	4	* forges its tests boot (`checks/service.ts`) and the cell probe is asked
	5	* once per process; the test starts this one without it.
	6	*
	7	* The parent holds a finished `cell` ledger for `kid`. The first request
	8	* is answered before the cell is up; then the cell is awaited and `/` and
	9	* `/healthz` are carried to it. Printed: one JSON line of what was seen.
	10	*/
	11	import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
	12	import { tmpdir } from "node:os";
	13	import { join } from "node:path";
	14	import { createApp } from "../../src/app.ts";
	15	import { cellCounters, cellsAvailable } from "../../src/apps/contain.ts";
	16	import { cellChildren, stopCellChildren } from "../../src/spawn/cell.ts";
	17	import { answerFromCell } from "../../src/spawn/host.ts";
	18
	19	const cells = await cellsAvailable();
	20	if (!cells.ok) {
	21	console.log(JSON.stringify({ skipped: cells.reason }));
	22	process.exit(0);
	23	}
	24
	25	const dataDir = mkdtempSync(join(tmpdir(), "real-child-"));
	26	const parent = createApp({ dataDir, url: "https://f.test" });
	27	const ledger = { name: "kid", bodyId: "cell-real", size: "small" } as const;
	28	mkdirSync(join(dataDir, "spawn"), { recursive: true });
	29	writeFileSync(join(dataDir, "spawn", "kid.json"), JSON.stringify({ ...ledger, done: true, placement: "cell" }));
	30	const ask = (path: string) => answerFromCell(parent, new Request(`https://kid--f.test${path}`, { headers: { host: "kid--f.test" } }));
	31
	32	try {
	33	const started = performance.now();
	34	const first = await ask("/");
	35	const firstMs = Math.round(performance.now() - started);
	36	const children = cellChildren(parent);
	37	const child = await children.start(ledger);
	38	const root = child === undefined ? null : (await ask("/")).status;
	39	const health = child === undefined ? null : (await ask("/healthz")).status;
	40	// The watch reads the cell's tree once a second, by its label.
	41	await Bun.sleep(1_500);
	42	console.log(
	43	JSON.stringify({
	44	first: first.status,
	45	firstMs,
	46	failure: children.failure(ledger.bodyId) ?? null,
	47	limits: child?.limits ?? null,
	48	root,
	49	health,
	50	labelled: cellCounters().some((c) => c.label === "forge:kid"),
	51	}),
	52	);
	53	} finally {
	54	await stopCellChildren(parent);
	55	parent.close();
	56	rmSync(dataDir, { recursive: true, force: true });
	57	}
		@@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
3	3	import { tmpdir } from "node:os";
4	4	import { join } from "node:path";
5	5	import { askUntilBelieved, cellsAvailable, openCell } from "../src/apps/contain.ts";
6		import { closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts";
	6	import { answerFromCell, closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts";
7	7	import { createApp } from "../src/app.ts";
8	8	import { cellChildren, stopCellChildren, cellEnv, cellLimits, cellPlacement, cellSpecFor, forward, ownsBody } from "../src/spawn/cell.ts";
9	9	import { fallbackPlacement, hostedPlacement, memoryBodies } from "../src/spawn/placement.ts";
		@@ -11,6 +11,9 @@ import type { SpawnPlan } from "../src/spawn/plan.ts";
11	11	import { SIZES } from "../src/spawn/sizes.ts";
12	12	import { notePeer } from "../src/http/peer.ts";
13	13	import type { App } from "../src/app.ts";
	14	import { portInsteadOfSocket } from "../src/git/hook-address.ts";
	15	import { missing } from "./helpers/tools.ts";
	16	import { expectBudget } from "./helpers/budget.ts";
14	17	import { openBeatDoor, parentLocal } from "../src/spawn/beat-door.ts";
15	18	import { lastHeard, sendBeat } from "../src/spawn/heartbeat.ts";
16	19
		@@ -251,23 +254,36 @@ describe("a child that will not start", () => {
251	254	});
252	255	});
253	256
	257	// Where bwrap and socat are here, whatever FORGE_CELLS says: a check turns
	258	// cells off for the forges its tests boot, and this is the one test that
	259	// must boot a child in a cell anyway (#6023). In its own process, without it.
	260	const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !missing("bwrap", "socat");
	261
254	262	describe("a real child forge", () => {
255		test.skipIf(!cells.ok)("boots in its cell and answers /healthz through the forward", async () => {
256		const parent = createApp({ dataDir: join(tmp, "real-parent"), url: "https://f.test" });
	263	test.skipIf(!cellTools)("boots in its own forge:<name> cell and answers / through the forward; its first request does not wait for it", async () => {
	264	const env: Record<string, string> = {};
	265	for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v;
	266	const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-child-forge.ts")], { env, stdout: "pipe", stderr: "pipe" });
	267	const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
	268	const line = out.trim().split("\n").pop() ?? "";
	269	let seen: Record<string, unknown>;
257	270	try {
258		const children = cellChildren(parent);
259		const child = await children.start({ bodyId: "cell-real", name: "kid", size: "small" });
260		expect(children.failure("cell-real")).toBeUndefined();
261		if (child === undefined) throw new Error(`the child did not start: ${children.failure("cell-real")}`);
262		expect(child.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
263		const res = await forward(child, new Request("http://kid--f.test/healthz", { headers: { host: "kid--f.test" } }));
264		expect(res.status).toBe(200);
265		await children.stop("cell-real");
266		} finally {
267		await stopCellChildren(parent);
268		parent.close();
	271	seen = JSON.parse(line) as Record<string, unknown>;
	272	} catch {
	273	throw new Error(`the child forge run printed no result: ${out.slice(-800)} ${errText.slice(-800)}`);
269	274	}
270		}, 60_000);
	275	if (typeof seen["skipped"] === "string") {
	276	console.log(`SKIPPING: this host refuses a cell inside the test (${seen["skipped"]}), so no child forge was booted in one`);
	277	return;
	278	}
	279	expect(seen["failure"]).toBeNull();
	280	expect(seen["first"]).toBe(503);
	281	expect(seen["firstMs"] as number).toBeLessThan(1_000);
	282	expect(seen["limits"]).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
	283	expect(seen["root"] as number).toBeLessThan(400);
	284	expect(seen["health"]).toBe(200);
	285	expect(seen["labelled"]).toBe(true);
	286	}, 90_000);
271	287
272	288	test.skipIf(!cells.ok)("its heartbeat reaches the parent through the cell, with no public address", async () => {
273	289	// The parent's public url is not reachable from here, as in a preview's cell.
		@@ -313,4 +329,33 @@ describe("a child whose cell will not come up", () => {
313	329	parent.close();
314	330	}
315	331	});
	332
	333	test.skipIf(cells.ok)("its first request is answered 503 at once, and once the cell has failed the next is hosted", async () => {
	334	const dataDir = join(tmp, "fast-parent");
	335	const parent = createApp({ dataDir, url: "https://f.test" });
	336	const request = () => new Request("https://kid--f.test/", { headers: { host: "kid--f.test" } });
	337	try {
	338	mkdirSync(join(dataDir, "spawn"), { recursive: true });
	339	const file = join(dataDir, "spawn", "kid.json");
	340	writeFileSync(file, JSON.stringify({ name: "kid", bodyId: "cell-f", done: true, placement: "cell", size: "small" }));
	341	// The baseline: a child with no ledger, answered from the same read with no start.
	342	const unknown = performance.now();
	343	expect((await answerFromCell(parent, new Request("https://nobody--f.test/", { headers: { host: "nobody--f.test" } }))).status).toBe(404);
	344	const against = performance.now() - unknown;
	345	const started = performance.now();
	346	const first = await answerFromCell(parent, request());
	347	expectBudget(performance.now() - started, { ms: 50, against });
	348	expect(first.status).toBe(503);
	349	expect(first.headers.get("retry-after")).toBe("5");
	350	// The start it set off fails here (no cells), and the ledger turns hosted.
	351	for (let i = 0; i < 50 && JSON.parse(readFileSync(file, "utf8")).placement !== "hosted"; i++) await Bun.sleep(10);
	352	expect(JSON.parse(readFileSync(file, "utf8")).placement).toBe("hosted");
	353	const next = await resolveHostedChild(parent, request());
	354	expect(next).not.toBe("cell");
	355	expect(next).not.toBe("unknown");
	356	} finally {
	357	await closeHostedChildren(parent);
	358	parent.close();
	359	}
	360	});
316	361	});