Public: anyone can read this; the crew can change it.

A child forge runs as its own process in its own cell, held to its size #6019

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2926-a-child-forge-runs-as-its-own-process-in → main 3 commits fork 9 threads · 9 unresolved

  • Merged. scout/2926-a-child-forge-runs-as-its-own-process-in was merged into main as e360500.

Review

Sign in to review this change.
14 files changed +652 −13
docs/SPAWN.md +1
@@ -16,6 +16,7 @@ and user accounts. The child mints its own. A test in #420 and #422 proves it.
1616 | ----- | ------------------------------------------------------- | ---------------- |
1717 | #420 | spawn plan: one validated value, dry-run | #383 |
1818 | #427 | placement seam; hosted inside the parent today | #420 |
19 | #2926 | `cell` placement: a child as its own `bun src/main.ts` in a cell held to its size (`spawn/cell.ts`); `hosted` where the host cannot hold a cell. The spawn answer and the children list say `placement` | #2925 |
1920 | #421 | spawn action: org, instance, apps as repos, briefs | #420, #427 |
2021
2122 `src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token).
src/actions/spawn.ts +15 −2
@@ -52,7 +52,7 @@ import { Database } from "bun:sqlite";
5252 import { loadKey, open } from "../kernel/secret.ts";
5353 import { loadOrCreateHostKey } from "../ssh/keys.ts";
5454 import { claimLink, hasAdmin, mintClaim, spawnedOrg } from "../spawn/claim.ts";
55 import { childPublicUrl, parentOrigin, type Placement } from "../spawn/placement.ts";
55 import { childPublicUrl, parentOrigin, type Placement, type PlacementKind } from "../spawn/placement.ts";
5656 import { quotaRefusal } from "../spawn/quota.ts";
5757 import { planSpawn, type SpawnInput, type SpawnPlan, type SpawnSource } from "../spawn/plan.ts";
5858 import { DEFAULT_SIZE, isSize, type Size } from "../spawn/sizes.ts";
@@ -64,6 +64,8 @@ export interface Spawned {
6464 readonly org: string;
6565 readonly url: string;
6666 readonly bodyId: string;
67 /** Where the child runs: in this process, or as a process in its own cell. */
68 readonly placement: PlacementKind;
6769 /** True when this call found the child already made and changed nothing. */
6870 readonly existed: boolean;
6971 /**
@@ -112,6 +114,10 @@ interface Ledger {
112114 readonly createdAt?: string;
113115 /** Absent on a ledger written before sizes; reads as small. */
114116 readonly size?: Size;
117 /** Absent on a ledger written before cells; reads as hosted. */
118 readonly placement?: PlacementKind;
119 /** Why a child that wanted a cell is hosted instead (`spawn/host.ts`). */
120 readonly placementNote?: string;
115121 /**
116122 * What the parent shows the child to learn its private facts (when anything
117123 * last happened); the child keeps only its hash. Never in an answer.
@@ -234,7 +240,7 @@ async function make(
234240 });
235241 }
236242 if (ledger.done)
237 return ok({ name: plan.name, org: plan.org, url: liveChildUrl(app, plan.name), bodyId: ledger.bodyId, existed: true });
243 return ok({ name: plan.name, org: plan.org, url: liveChildUrl(app, plan.name), bodyId: ledger.bodyId, placement: ledger.placement ?? "hosted", existed: true });
238244 if (ledger.statusToken === undefined || ledger.beatKey === undefined) {
239245 ledger.statusToken ??= secret();
240246 ledger.beatKey ??= secret();
@@ -261,6 +267,7 @@ async function make(
261267 apps: plan.apps.map((a) => a.ref),
262268 createdAt: new Date().toISOString(),
263269 size: plan.size ?? DEFAULT_SIZE,
270 placement: placed.value.kind ?? "hosted",
264271 statusToken: secret(),
265272 beatKey: secret(),
266273 };
@@ -284,6 +291,7 @@ async function make(
284291 org: plan.org,
285292 url: liveChildUrl(app, plan.name),
286293 bodyId: ledger.bodyId,
294 placement: ledger.placement ?? "hosted",
287295 existed: false,
288296 claimUrl,
289297 });
@@ -665,6 +673,9 @@ export interface Child {
665673 /** When it was begun; null on a ledger older than the client list. */
666674 readonly createdAt: string | null;
667675 readonly size: Size;
676 readonly placement: PlacementKind;
677 /** Why it is hosted when it asked for a cell; null otherwise. */
678 readonly placementNote: string | null;
668679 }
669680
670681 /** The children this forge has made or begun, from its ledger, by name. */
@@ -684,6 +695,8 @@ export async function listChildren(app: App): Promise<Child[]> {
684695 apps: ledger.apps ?? [],
685696 createdAt: ledger.createdAt ?? null,
686697 size: isSize(ledger.size) ? ledger.size : DEFAULT_SIZE,
698 placement: ledger.placement ?? "hosted",
699 placementNote: ledger.placementNote ?? null,
687700 });
688701 }
689702 return out;
src/apps/contain.ts +3 −1
@@ -189,6 +189,8 @@ export interface CellSpec {
189189 /** Origins always reachable: the forge's own URL. */
190190 readonly reach: readonly string[];
191191 readonly limits: Limits;
192 /** Read-only paths to keep visible when they sit under a hidden directory (a forge's own code). */
193 readonly reveal?: readonly string[] | undefined;
192194 /** The port an app listens on; the forge relays `127.0.0.1:<port>` in. */
193195 readonly listen?: number | undefined;
194196 /** Keep the controlling terminal (an app's `tty`). */
@@ -405,7 +407,7 @@ export function openCell(
405407 const blank = hidden.filter((path) => !isDirectory(path));
406408 // The runtime itself must stay visible when it lives in a hidden home
407409 // (`~/.bun/bin/bun` on a developer's machine).
408 const reveal = [dirname(process.execPath)].filter((path) =>
410 const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) =>
409411 hide.some((h) => path === h || path.startsWith(`${h}/`)),
410412 );
411413 const writable = [...new Set([spec.dir, spec.home, ...spec.writable])];
src/http/server.ts +15 −1
@@ -532,8 +532,9 @@ import {
532532 resolveAppHost,
533533 } from "../apps/proxy.ts";
534534
535 import { stopCellChildren } from "../spawn/cell.ts";
535536 import { portInsteadOfSocket, tcpAddress } from "../git/hook-address.ts";
536 import { closeHostedChildren, resolveHostedChild, resolveHostedChildApp } from "../spawn/host.ts";
537 import { answerFromCell, closeHostedChildren, resolveHostedChild, resolveHostedChildApp } from "../spawn/host.ts";
537538 import { getVerifiedDomain } from "../core/domains.ts";
538539
539540 // ---- auth ----
@@ -2277,6 +2278,18 @@ export async function serve(app: App): Promise<Servers> {
22772278 const child = await resolveHostedChild(app, request);
22782279 if (child === "unknown")
22792280 return securityFloor(new Response("not found", { status: 404 }));
2281 if (child === "cell") {
outdated

High (pairs with cell.ts:239): no notePeer call before this, so the parent has no resolved client to carry. Medium: this also bypasses armClose, answered, counted, logged and settleBody, which the hosted branch below runs.

2282 // The child sees only this forge's relay, so it is told who the client is.
2283 notePeer(request, server.requestIP(request)?.address, trust);
2284 armClose(request, () => server.timeout(request, 1));
2285 const response = await answered(app, request, () => answerFromCell(app, request));
2286 counted(app, request, response, started);
2287 logged(app.config.dataDir, request, response, started);
2288 await settleBody(request);
2289 if (!mustClose(request)) return response;
2290 server.timeout(request, 1);
2291 return withHeader(response, "connection", "close");
2292 }
22802293 if (child !== undefined) {
22812294 const key = child.config.dataDir;
22822295 let http = childHttps.get(key);
@@ -2395,6 +2408,7 @@ export async function serve(app: App): Promise<Servers> {
23952408 for (const http of childHttps.values()) http.close();
23962409 childHttps.clear();
23972410 closeHostedChildren(app);
2411 void stopCellChildren(app);
23982412 flushAccess();
23992413 web.stop(true);
24002414 internal.stop(true);
src/spawn/cell.ts added +299
@@ -0,0 +1,299 @@
1 /**
2 * The `cell` placement: a child forge as a process of its own. It is
3 * `bun src/main.ts` with its data directory, started from `childEnv`
4 * (invariant 11) inside a cell (`apps/contain.ts`) whose memory, CPU and
5 * disk are its size's (`spawn/sizes.ts`). The parent keeps answering
6 * `<name>--<host>` and forwards to the loopback port the cell relays in on
7 * (`forward`). Where the host cannot hold a cell the child is hosted
8 * (`spawn/placement.ts` `fallbackPlacement`), and the ledger says which.
9 */
10
11 import { dirname, join } from "node:path";
12 import { clientAddress } from "../http/peer.ts";
13 import type { App } from "../app.ts";
14 import { cellsAvailable, killTree, openCell, type Cell, type CellSpec, type Limits } from "../apps/contain.ts";
15 import { childEnv, freePort } from "../apps/process.ts";
16 import { childDirOf } from "../actions/spawn.ts";
17 import { hostedPlacement, parentOrigin, childPublicUrl, type BodyStore, type Placement } from "./placement.ts";
18 import { SIZES, type Size } from "./sizes.ts";
19
20 export const CELL_PREFIX = "cell-";
21 export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX);
22
23 /** After a child fails to start, requests are answered 503 at once for this long. */
24 export const START_BACKOFF_MS = 30_000;
25
26 const MAIN = join(import.meta.dir, "..", "main.ts");
27 const CODE = join(import.meta.dir, "..", "..");
28
29 /** A size's limits as a cell holds them. */
30 export function cellLimits(size: Size): Limits {
31 const s = SIZES[size];
32 return { memoryMiB: s.memoryMiB, cpus: s.cpu, diskMiB: s.diskMiB };
33 }
34
35 /** The cell a child of `size` runs in; plain data, so a test reads the limits off it. */
36 export function cellSpecFor(input: {
37 dir: string;
38 parentDataDir: string;
39 parentUrl: string;
40 size: Size;
41 port: number;
42 label: string;
43 }): CellSpec {
44 return {
45 dir: input.dir,
46 writable: [],
47 home: join(input.dir, ".home"),
48 // The parent's data is the one thing a child must not see; its own
49 // directory sits under it and is bound back, deepest last.
50 hide: [input.parentDataDir],
51 // The forge's own code may live under the parent's data (a release checkout).
52 reveal: [CODE],
53 egress: [],
54 reach: [input.parentUrl],
55 limits: cellLimits(input.size),
56 listen: input.port,
57 label: input.label,
58 };
59 }
60
61 /** The child forge's environment: short, explicit, and its own. */
62 export function cellEnv(input: { port: number; dataDir: string; url: string }): Record<string, string> {
63 const env = childEnv({
64 PORT: String(input.port),
65 FORGE_HOST: "127.0.0.1",
66 FORGE_APP_URL: input.url,
67 FORGE_APP_DATA: input.dataDir,
68 NODE_ENV: "production",
69 });
70 delete env["USER"];
71 delete env["LOGNAME"];
72 return env;
73 }
74
75 export interface CellChild {
76 readonly bodyId: string;
77 readonly name: string;
78 readonly size: Size;
79 readonly port: number;
80 readonly limits: Limits;
81 readonly pid: number;
82 readonly exited: Promise<number>;
83 }
84
85 interface Held extends CellChild {
86 readonly cell: Cell;
87 readonly proc: ReturnType<typeof Bun.spawn>;
88 }
89
90 export interface CellChildren {
91 /** Start the child (idempotent); resolves once it answers, or says why not. */
92 start(input: { bodyId: string; name: string; size: Size }): Promise<CellChild | undefined>;
93 stop(bodyId: string): Promise<void>;
94 get(bodyId: string): CellChild | undefined;
95 /** Why the last start failed, in a sentence a person can read; undefined when it did not. */
96 failure(bodyId: string): string | undefined;
97 list(): readonly CellChild[];
98 }
99
100 const registries = new WeakMap<App, CellChildren>();
101
102 export function cellChildren(parent: App): CellChildren {
103 let r = registries.get(parent);
104 if (r === undefined) registries.set(parent, (r = makeRegistry(parent)));
105 return r;
106 }
107
108 async function answers(port: number, gone: Promise<unknown>): Promise<boolean> {
109 let over = false;
110 void gone.then(() => (over = true));
111 for (let i = 0; i < 300 && !over; i++) {
112 try {
113 await fetch(`http://127.0.0.1:${port}/`, { redirect: "manual", signal: AbortSignal.timeout(500) });
114 return true;
115 } catch {
116 await Bun.sleep(100);
117 }
118 }
119 return false;
120 }
121
122 function makeRegistry(parent: App): CellChildren {
123 const held = new Map<string, Held>();
124 const starting = new Map<string, Promise<CellChild | undefined>>();
125 /** When a start last failed: a broken child is not relaunched on every request. */
126 const failed = new Map<string, { at: number; why: string }>();
127 /** Held once it answers, never before: a start in flight is in `starting`. */
128
129 const launch = async (input: { bodyId: string; name: string; size: Size }): Promise<CellChild | undefined> => {
130 const cells = await cellsAvailable();
131 if (!cells.ok) {
132 parent.log.warn("child cell not started", { name: input.name, reason: cells.reason });
133 failed.set(input.bodyId, { at: Date.now(), why: cells.reason });
134 return undefined;
135 }
136 const recent: string[] = [];
137 const port = freePort();
138 const dir = childDirOf(parent, input.bodyId);
139 const url = childPublicUrl(input.name, parentOrigin(parent.config));
140 const spec = cellSpecFor({
141 dir,
142 parentDataDir: parent.config.dataDir,
143 parentUrl: parentOrigin(parent.config),
144 size: input.size,
outdated

On the preview this process exits within about 1 s of every launch, and the child never answers (503 on every request). Before trusting the paths here, prove that bun src/main.ts can read its code and node_modules, and write what it needs, inside the cell: hide plus hiddenMounts() may cover them.

145 port,
146 label: `forge:${input.name}`,
147 });
148 const log = (line: string): void => {
149 recent.push(line);
150 if (recent.length > 5) recent.shift();
151 parent.log.info("child forge", { name: input.name, line });
152 };
153 const cell = openCell(spec, log);
154 const command = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(MAIN)}`;
155 const proc = Bun.spawn(cell.argv(command), {
156 cwd: dirname(MAIN),
157 env: cell.env(cellEnv({ port, dataDir: dir, url })),
158 stdin: "ignore",
159 stdout: "pipe",
160 stderr: "pipe",
161 });
outdated

This is held before it answers. A concurrent start() returns it from held at line 173 and is forwarded to a port nobody listens on yet (502). Set held after answers() succeeds, and let starting cover the boot.

162 const exited = proc.exited.then((code) => {
163 const why = cell.breach();
164 if (why !== undefined) parent.log.warn("child forge stopped", { name: input.name, why });
165 cell.close();
166 if (held.get(input.bodyId)?.proc === proc) held.delete(input.bodyId);
167 if (why !== undefined) recent.push(why);
168 return code;
169 });
170 cell.watch(proc.pid, () => killTree(proc.pid));
171 void drain(proc.stdout, log);
172 void drain(proc.stderr, log);
173 const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc };
174 if (!(await answers(port, exited))) {
175 const why = `its cell did not come up: ${recent.slice(-3).join(" | ") || "no output"}`.slice(0, 400);
176 parent.log.warn("child forge did not answer", { name: input.name, why });
177 failed.set(input.bodyId, { at: Date.now(), why });
178 killTree(proc.pid);
outdated

If launch rejects (openCell/Bun.spawn throws), this .then is skipped: failed is never set, so the backoff never applies and every request relaunches. Catch to undefined first.

179 return undefined;
180 }
181 held.set(input.bodyId, child);
182 return child;
183 };
184
185 return {
186 start(input) {
187 const there = held.get(input.bodyId);
188 if (there !== undefined) return Promise.resolve(there);
189 const lastFailure = failed.get(input.bodyId);
190 if (lastFailure !== undefined && Date.now() - lastFailure.at < START_BACKOFF_MS) return Promise.resolve(undefined);
191 let flight = starting.get(input.bodyId);
192 if (flight === undefined) {
193 flight = launch(input)
194 .catch((error: unknown) => {
195 const why = `its cell could not be made: ${String(error)}`.slice(0, 400);
196 parent.log.warn("child forge could not start", { name: input.name, why });
197 failed.set(input.bodyId, { at: Date.now(), why });
198 return undefined;
199 })
200 .then((child) => {
201 if (child !== undefined) failed.delete(input.bodyId);
202 return child;
203 });
204 starting.set(input.bodyId, flight);
205 void flight.finally(() => starting.delete(input.bodyId)).catch(() => {});
206 }
207 return flight;
208 },
209 async stop(bodyId) {
210 const child = held.get(bodyId);
211 if (child === undefined) return;
212 held.delete(bodyId);
213 killTree(child.pid);
214 await child.exited;
215 },
216 get: (bodyId) => held.get(bodyId),
217 failure: (bodyId) => failed.get(bodyId)?.why,
218 list: () => [...held.values()],
219 };
220 }
221
222 async function drain(stream: ReadableStream<Uint8Array> | number | null | undefined, log: (line: string) => void): Promise<void> {
223 if (stream === null || stream === undefined || typeof stream === "number") return;
224 const text = new TextDecoder();
225 let rest = "";
226 try {
227 for await (const chunk of stream) {
228 rest += text.decode(chunk, { stream: true });
229 const lines = rest.split("\n");
230 rest = lines.pop() ?? "";
231 for (const line of lines) if (line !== "") log(line.slice(0, 500));
232 }
233 } catch {
234 // the process is gone
235 }
236 }
237
238 /** Stop every child cell this parent holds: when the parent stops, they go with it. */
239 export async function stopCellChildren(parent: App): Promise<void> {
outdated

High: this copies the client's x-forwarded-for, forwarded, x-real-ip and client-IP header to a child that trusts loopback, so the child's limits key on what the client wrote. Strip them, then set x-forwarded-for to the parent's resolved clientAddress(request). Add a test that a spoofed XFF does not reach the child.

240 const r = registries.get(parent);
241 if (r === undefined) return;
242 registries.delete(parent);
243 await Promise.all(r.list().map((c) => r.stop(c.bodyId)));
244 }
245
246 /** The cell placement: a body in the shared store, stopped when retired. */
247 export function cellPlacement(parent: App, store: BodyStore, maxChildren: () => number): Placement {
248 const base = hostedPlacement({
249 host: parentOrigin(parent.config),
250 store,
251 maxChildren,
252 kind: "cell",
253 newId: () => `${CELL_PREFIX}${crypto.randomUUID()}`,
254 });
255 return {
256 ...(base.room === undefined ? {} : { room: base.room }),
257 place: base.place,
258 async retire(bodyId) {
259 await cellChildren(parent).stop(bodyId);
260 return base.retire(bodyId);
261 },
262 };
263 }
264
265 // ---- forwarding ----
266
267 /** Headers that say who the client is: the child believes them from its relay, so only the forge writes them. */
268 const WHO = new Set(["x-forwarded-for", "forwarded", "x-real-ip", "cf-connecting-ip", "true-client-ip", "x-forwarded-host", "x-forwarded-proto"]);
269
270 const HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "content-length"]);
271
272 /** Carry a request to the child's loopback port and its answer back, streaming. */
273 export async function forward(
274 child: Pick<CellChild, "port">,
275 request: Request,
276 clientHeader?: string,
277 ): Promise<Response> {
278 const at = new URL(request.url);
279 const headers = new Headers();
280 const named = clientHeader?.toLowerCase();
281 for (const [k, v] of request.headers) if (!HOP.has(k) && !WHO.has(k) && k !== named) headers.set(k, v);
282 // The address the forge resolved (`http/peer.ts`), the only one the child hears.
283 headers.set("x-forwarded-for", clientAddress(request));
284 const hasBody = request.method !== "GET" && request.method !== "HEAD";
285 try {
286 const res = await fetch(`http://127.0.0.1:${child.port}${at.pathname}${at.search}`, {
287 method: request.method,
288 headers,
289 body: hasBody ? request.body : null,
290 redirect: "manual",
291 decompress: false,
292 } as RequestInit);
293 const out = new Headers();
294 for (const [k, v] of res.headers) if (!HOP.has(k)) out.append(k, v);
295 return new Response(res.body, { status: res.status, headers: out });
296 } catch {
297 return new Response("this forge is not answering", { status: 502 });
298 }
299 }
src/spawn/door.ts +11 −1
@@ -19,6 +19,7 @@ import type { Ctx } from "../http/context.ts";
1919 import { takeDoor } from "../http/strangers.ts";
2020 import {
2121 childPublicUrl,
22 fallbackPlacement,
2223 hostedPlacement,
2324 memoryBodies,
2425 parentOrigin,
@@ -27,6 +28,8 @@ import {
2728 } from "./placement.ts";
2829 import { dryRun, planSpawn, KEY_BUDGET_NOTE, MAX_APPS, type CatalogApp, type CrewBrief, type SpawnSource } from "./plan.ts";
2930 import { bootHostedChild } from "./host.ts";
31 import { cellPlacement, ownsBody } from "./cell.ts";
32 import { cellsAvailable } from "../apps/contain.ts";
3033 import { ANON_SPAWNS_PER_DAY, checkAnonSpawn, noteAnonSpawn } from "./rate.ts";
3134 import { readdirSync, readFileSync } from "node:fs";
3235 import { readdir, stat } from "node:fs/promises";
@@ -101,7 +104,14 @@ function placementFor(app: App): Placement {
101104 if (!p) {
102105 const store = memoryBodies();
103106 seedBodies(app, store);
104 p = hostedPlacement({ host: parentOrigin(app.config), store, maxChildren: () => maxChildren(app.db) });
107 const cap = () => maxChildren(app.db);
108 // A cell where the host can hold one, the parent's own process where it cannot.
109 p = fallbackPlacement(
110 cellPlacement(app, store, cap),
111 hostedPlacement({ host: parentOrigin(app.config), store, maxChildren: cap }),
112 async () => (await cellsAvailable()).ok,
113 ownsBody,
114 );
105115 placements.set(app, p);
106116 }
107117 return p;
src/spawn/family.ts +7 −1
@@ -9,12 +9,15 @@ import { parentForgeFor } from "../core/parent_releases.ts";
99 import { running } from "../kernel/version.ts";
1010 import type { ChildAppsSummary } from "./child-apps.ts";
1111 import { childStandings, parentUpstream, type ChildStanding } from "./heartbeat.ts";
12 import { listChildren } from "../actions/spawn.ts";
1213 import { hostedChildApps } from "./host.ts";
1314
1415 /** A child as the family shows it: its standing, and what its apps are doing. */
1516 export interface FamilyChild extends ChildStanding {
1617 /** "1 of 2 apps running — bakery asleep", or null when it was given no apps. */
1718 readonly apps: string | null;
19 /** Where it runs: "own cell", "hosted", or "hosted: <why its cell did not start>". */
20 readonly placement: string | null;
1821 }
1922
2023 const REASONS = new Set(["asleep", "failed", "stopped", "deploying"]);
@@ -57,6 +60,9 @@ export async function familyView(
5760 ): Promise<FamilyView> {
5861 const known = parentForgeFor(app.config);
5962 const url = app.config.parentUrl ?? (await parentUpstream(app));
63 const placed = new Map<string, string>();
64 if (isAdmin)
65 for (const c of await listChildren(app)) placed.set(c.name, c.placementNote ?? (c.placement === "cell" ? "own cell" : "hosted"));
6066 const kids = isAdmin ? await childStandings(app) : { items: [], more: 0 };
6167 return {
6268 parent:
@@ -67,7 +73,7 @@ export async function familyView(
6773 host: app.config.url === undefined ? null : new URL(app.config.url).host,
6874 version: running().version,
6975 },
70 children: kids.items.map((c) => ({ ...c, apps: appsLine(appsOf(app, c.name)) })),
76 children: kids.items.map((c) => ({ ...c, apps: appsLine(appsOf(app, c.name)), placement: placed.get(c.name) ?? null })),
7177 more: kids.more,
7278 };
7379 }
src/spawn/host.ts +64 −5
@@ -19,14 +19,19 @@ import {
1919 parentOrigin,
2020 parseHostedChildAppLabel,
2121 parseHostedChildName,
22 type PlacementKind,
2223 } from "./placement.ts";
23 import { readdirSync, readFileSync } from "node:fs";
24 import { DEFAULT_SIZE, isSize, type Size } from "./sizes.ts";
25 import { cellChildren, forward } from "./cell.ts";
26 import { readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
2427 import { join } from "node:path";
2528
2629 interface ChildLedger {
2730 readonly name: string;
2831 readonly bodyId: string;
2932 readonly done: boolean;
33 readonly placement: PlacementKind;
34 readonly size: Size;
3035 }
3136
3237 function readLedgerSync(app: App, name: string): ChildLedger | undefined {
@@ -36,9 +41,17 @@ function readLedgerSync(app: App, name: string): ChildLedger | undefined {
3641 name?: unknown;
3742 bodyId?: unknown;
3843 done?: unknown;
44 placement?: unknown;
45 size?: unknown;
3946 };
4047 if (typeof parsed.name !== "string" || typeof parsed.bodyId !== "string") return undefined;
41 return { name: parsed.name, bodyId: parsed.bodyId, done: parsed.done === true };
48 return {
49 name: parsed.name,
50 bodyId: parsed.bodyId,
51 done: parsed.done === true,
52 placement: parsed.placement === "cell" ? "cell" : "hosted",
53 size: isSize(parsed.size) ? parsed.size : DEFAULT_SIZE,
54 };
4255 } catch {
4356 return undefined;
4457 }
@@ -92,22 +105,49 @@ async function openChild(parent: App, ledger: ChildLedger): Promise<App> {
92105 * Resolve a request Host to a hosted child's App.
93106 * - `undefined` — not a child host (parent / app / other continues).
94107 * - `"unknown"` — shaped like a child host but no finished ledger.
108 * - `"cell"` — a process of its own: `answerFromCell` carries the request.
95109 * - `App` — answer with this child's routes.
96110 */
97111 export async function resolveHostedChild(
98112 parent: App,
99113 request: Request,
100 ): Promise<App | "unknown" | undefined> {
114 ): Promise<App | "cell" | "unknown" | undefined> {
101115 const host = request.headers.get("host");
102116 if (host === null) return undefined;
103117 const name = parseHostedChildName(host, childHost(parent));
104118 if (name === undefined) return undefined;
105119 const ledger = readLedgerSync(parent, name);
106120 if (ledger === undefined || !ledger.done) return "unknown";
107
121 // A child in its own cell is not an App here: the request is carried to it.
122 if (ledger.placement === "cell") return settleCell(parent, ledger);
108123 return openOnce(parent, ledger);
109124 }
110125
126 /**
127 * Start a child's cell. When the cell cannot come up the child is not left
128 * dead: the ledger turns it hosted, says why (`placementNote`, which
129 * `/forges` lists), and the child is opened in this process.
130 */
131 async function settleCell(parent: App, ledger: ChildLedger): Promise<App | "cell"> {
132 const cells = cellChildren(parent);
133 if ((await cells.start(ledger)) !== undefined) return "cell";

On the preview this start runs inside the visitor's request and waits the full answers() window (about 30 s) before demoting. Start the cell in the background at spawn, and while it is not up, answer from hosted or with a fast 503. Tracked in #6023.

134 const why = cells.failure(ledger.bodyId) ?? "its cell did not start";
135 demote(parent, ledger.name, why);
136 parent.log.warn("child forge placed hosted instead", { name: ledger.name, why });
137 return openOnce(parent, { ...ledger, placement: "hosted" });
138 }
139
140 function demote(parent: App, name: string, why: string): void {
141 const path = join(parent.config.dataDir, "spawn", `${name}.json`);
142 try {
143 const raw = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
144 writeFileSync(`${path}.tmp`, JSON.stringify({ ...raw, placement: "hosted", placementNote: `hosted: ${why}` }));
145 renameSync(`${path}.tmp`, path);
146 } catch (error) {
147 parent.log.error("could not record the placement", { name, error: String(error) });
148 }
149 }
150
111151 function openOnce(parent: App, ledger: ChildLedger): Promise<App> | App {
112152 const cache = cacheOf(parent);
113153 const hit = cache.get(ledger.bodyId);
@@ -131,8 +171,13 @@ function openOnce(parent: App, ledger: ChildLedger): Promise<App> | App {
131171 * the first visit after a claim finds them awake. Never throws.
132172 */
133173 export async function bootHostedChild(parent: App, name: string): Promise<void> {
134 const ledger = readLedgerSync(parent, name);
174 let ledger = readLedgerSync(parent, name);
135175 if (ledger === undefined || !ledger.done) return;
176 if (ledger.placement === "cell") {
177 const settled = await settleCell(parent, ledger);
178 if (settled === "cell") return;
179 ledger = { ...ledger, placement: "hosted" };
180 }
136181 try {
137182 await startChildApps(parent, await openOnce(parent, ledger), name);
138183 } catch (error) {
@@ -198,3 +243,17 @@ export async function resolveHostedChildApp(
198243
199244 /** What a child's apps are doing, for `/forges`; undefined when none run here. */
200245 export const hostedChildApps = childAppsSummary;
246
247 /**
248 * Answer a request for a child that runs in its own cell: start it if it is
249 * not up (a parent that restarted, a child that stopped), then forward.
250 */
251 export async function answerFromCell(parent: App, request: Request): Promise<Response> {
252 const name = parseHostedChildName(request.headers.get("host") ?? "", childHost(parent));
253 const ledger = name === undefined ? undefined : readLedgerSync(parent, name);
254 if (ledger === undefined || ledger.placement !== "cell") return new Response("not found", { status: 404 });
255 const child = await cellChildren(parent).start(ledger);
256 // Unreachable once a cell has failed (the ledger turned hosted), but never a hang.
257 if (child === undefined) return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5" } });
258 return forward(child, request, parent.config.clientIpHeader);
259 }
src/spawn/placement.ts +28 −1
@@ -12,9 +12,14 @@
1212 import { err, ok, type Result } from "../kernel/result.ts";
1313 import type { SpawnPlan } from "./plan.ts";
1414
15 /** Where a child runs: inside the parent's process, or as a process in its own cell. */
16 export type PlacementKind = "hosted" | "cell";
17
1518 export interface Placed {
1619 readonly url: string;
1720 readonly bodyId: string;
21 /** Absent reads as hosted. */
22 readonly kind?: PlacementKind;
1823 }
1924
2025 export type PlacementRefusal = { readonly reason: "children" | "depth" | "exists" | "quota" | "unknown"; readonly message: string };
@@ -143,6 +148,8 @@ export interface HostedOptions {
143148 readonly store?: BodyStore;
144149 /** Injectable so tests need no randomness. */
145150 readonly newId?: () => string;
151 /** What `place` says it did; hosted unless the caller is a cell placement. */
152 readonly kind?: PlacementKind;
146153 }
147154
148155 const refuse = (reason: PlacementRefusal["reason"], message: string) =>
@@ -182,7 +189,7 @@ export function hostedPlacement(opts: HostedOptions): Placement {
182189 const url = childPublicUrl(plan.name, opts.host);
183190 const bodyId = newId();
184191 store.add({ bodyId, name: plan.name, url, depth: depth + 1, agents: 1, ssh: false });
185 return ok({ url, bodyId });
192 return ok({ url, bodyId, ...(opts.kind === undefined ? {} : { kind: opts.kind }) });
186193 },
187194 async retire(bodyId) {
188195 store.remove(bodyId);
@@ -190,3 +197,23 @@ export function hostedPlacement(opts: HostedOptions): Placement {
190197 },
191198 };
192199 }
200
201 /**
202 * Two placements behind one: `preferred` where `holds()` says this host can,
203 * `fallback` where it cannot. A body is retired by whichever made it, told
204 * by its id's prefix. Capacity is the fallback's (they share one store).
205 */
206 export function fallbackPlacement(
207 preferred: Placement,
208 fallback: Placement,
209 holds: () => Promise<boolean>,
210 owns: (bodyId: string) => boolean,
211 ): Placement {
212 return {
213 ...(fallback.room === undefined ? {} : { room: fallback.room }),
214 async place(plan) {
215 return (await holds()) ? preferred.place(plan) : fallback.place(plan);
216 },
217 retire: (bodyId) => (owns(bodyId) ? preferred : fallback).retire(bodyId),
218 };
219 }
src/web/pages/family.ts +1
@@ -49,6 +49,7 @@ function standingNode(c: FamilyChild): Node {
4949 const follow = followLine(c);
5050 if (follow !== null) meta.push(html`<span data-follow>${follow}</span>`);
5151 if (c.apps !== null) meta.push(html`<span data-apps>${c.apps}</span>`);
52 if (c.placement !== null) meta.push(html`<span data-placement>${c.placement}</span>`);

The page gets the placement, but familyLines in src/spawn/family.ts (the terminal's family view) doesn't, so the two doors differ. Tracked in #6023.

5253 return {
5354 label: c.name,
5455 href: c.url,
test/spawn-api.test.ts +2
@@ -165,6 +165,8 @@ describe("spawn over the API and MCP", () => {
165165 apps: expect.any(Array),
166166 createdAt: expect.any(String),
167167 size: "small",
168 placement: "hosted",
169 placementNote: null,
168170 },
169171 ]);
170172 // The same request again is answered, not made twice.
test/spawn-cell.test.ts added +204
@@ -0,0 +1,204 @@
1 import { afterAll, describe, expect, test } from "bun:test";
2 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { cellsAvailable, openCell } from "../src/apps/contain.ts";
6 import { closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts";
7 import { createApp } from "../src/app.ts";
8 import { cellChildren, stopCellChildren, cellEnv, cellLimits, cellPlacement, cellSpecFor, forward, ownsBody } from "../src/spawn/cell.ts";
9 import { fallbackPlacement, hostedPlacement, memoryBodies } from "../src/spawn/placement.ts";
10 import type { SpawnPlan } from "../src/spawn/plan.ts";
11 import { SIZES } from "../src/spawn/sizes.ts";
12 import { notePeer } from "../src/http/peer.ts";
13 import type { App } from "../src/app.ts";
14
15 const plan = (name: string): SpawnPlan => ({ name, org: "o", apps: [], briefs: [], subscriptions: [], direction: "", keepAccess: false, budgetUsd: null });
16 const tmp = mkdtempSync(join(tmpdir(), "spawn-cell-"));
17 afterAll(() => rmSync(tmp, { recursive: true, force: true }));
18
19 describe("a child forge's cell", () => {
20 test("holds the size's memory, CPU and disk", () => {
21 for (const size of ["small", "medium", "large"] as const) {
22 expect(cellLimits(size)).toEqual({ memoryMiB: SIZES[size].memoryMiB, cpus: SIZES[size].cpu, diskMiB: SIZES[size].diskMiB });
23 }
24 const spec = cellSpecFor({ dir: "/d/children/c1", parentDataDir: "/d", parentUrl: "https://f.test", size: "small", port: 4100, label: "forge:kid" });
25 expect(spec.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
26 expect(spec.hide).toEqual(["/d"]);
27 expect(spec.listen).toBe(4100);
28 expect(spec.dir).toBe("/d/children/c1");
29 });
30
31 test("starts from childEnv: the parent's secrets are not handed on", () => {
32 process.env["FORGE_EDGE_TOKEN"] = "parent-secret";
33 try {
34 const env = cellEnv({ port: 4100, dataDir: "/d/children/c1", url: "https://kid--f.test" });
35 expect(env["FORGE_EDGE_TOKEN"]).toBeUndefined();
36 expect(env["FORGE_APP_URL"]).toBe("https://kid--f.test");
37 expect(env["FORGE_APP_DATA"]).toBe("/d/children/c1");
38 expect(env["PORT"]).toBe("4100");
39 } finally {
40 delete process.env["FORGE_EDGE_TOKEN"];
41 }
42 });
43 });
44
45 describe("the placement seam", () => {
46 const app = { config: { dataDir: tmp, url: "https://f.test" } } as unknown as App;
47 const build = (cells: boolean) => {
48 const store = memoryBodies();
49 const cap = () => 10;
50 return fallbackPlacement(
51 cellPlacement(app, store, cap),
52 hostedPlacement({ host: "https://f.test", store, maxChildren: cap, newId: () => "hosted-1" }),
53 async () => cells,
54 ownsBody,
55 );
56 };
57
58 test("a cell where the host can hold one, and it says so", async () => {
59 const r = await build(true).place(plan("kid"));
60 if (!r.ok) throw new Error("refused");
61 expect(r.value.kind).toBe("cell");
62 expect(ownsBody(r.value.bodyId)).toBe(true);
63 expect(r.value.url).toBe("https://kid--f.test");
64 });
65
66 test("hosted where it cannot", async () => {
67 const r = await build(false).place(plan("kid"));
68 if (!r.ok) throw new Error("refused");
69 expect(r.value.kind ?? "hosted").toBe("hosted");
70 expect(ownsBody(r.value.bodyId)).toBe(false);
71 });
72
73 test("a retired cell child frees its room", async () => {
74 const p = build(true);
75 const r = await p.place(plan("kid"));
76 if (!r.ok) throw new Error("refused");
77 expect((await p.retire(r.value.bodyId)).ok).toBe(true);
78 expect((await p.place(plan("kid"))).ok).toBe(true);
79 });
80 });
81
82 describe("forwarding to a child", () => {
83 test("carries method, path, query, body and the answer back", async () => {
84 const server = Bun.serve({
85 port: 0,
86 async fetch(req) {
87 const u = new URL(req.url);
88 return new Response(`${req.method} ${u.pathname}${u.search} ${await req.text()} ${req.headers.get("host")}`, {
89 status: 201,
90 headers: { "set-cookie": "a=b; HttpOnly" },
91 });
92 },
93 });
94 try {
95 const res = await forward({ port: server.port! }, new Request("http://kid--f.test/x?y=1", { method: "POST", body: "hi", headers: { host: "kid--f.test" } }));
96 expect(res.status).toBe(201);
97 expect(await res.text()).toBe("POST /x?y=1 hi kid--f.test");
98 expect(res.headers.get("set-cookie")).toBe("a=b; HttpOnly");
99 } finally {
100 server.stop(true);
101 }
102 });
103
104 test("a child that is not listening answers 502", async () => {
105 const res = await forward({ port: 1 }, new Request("http://kid--f.test/"));
106 expect(res.status).toBe(502);
107 });
108 });
109
110 const cells = await cellsAvailable();
outdated

This test runs ls, not the forge. Add one beside it that boots the child through cellChildren(app).start(...) and asserts that /healthz answers 200 through forward. That test would have caught the preview failure.

111 describe("a real cell", () => {
112 test.skipIf(!cells.ok)("runs the child with the small limits and cannot see the parent's data", async () => {
113 const parent = join(tmp, "parent");
114 const dir = join(parent, "children", "c1");
115 mkdirSync(dir, { recursive: true });
116 writeFileSync(join(parent, "forge.db"), "secret");
117 const spec = cellSpecFor({ dir, parentDataDir: parent, parentUrl: "https://f.test", size: "small", port: 4999, label: "forge:test" });
118 expect(spec.limits.memoryMiB).toBe(512);
119 const cell = openCell(spec);
120 const proc = Bun.spawn(cell.argv(`ls ${parent}; echo own > ${dir}/mine`), { env: cell.env({ PATH: process.env["PATH"] ?? "" }), stdout: "pipe", stderr: "pipe" });
121 const out = await new Response(proc.stdout).text();
122 await proc.exited;
123 cell.close();
124 expect(out).not.toContain("forge.db");
125 expect(out).toContain("children");
126 });
127 });
128
129 describe("who the child hears", () => {
130 test("a spoofed client header never reaches the child; it hears the address the forge resolved", async () => {
131 let seen: Headers | undefined;
132 const server = Bun.serve({ port: 0, fetch(req) { seen = req.headers; return new Response("ok"); } });
133 try {
134 const request = new Request("http://kid--f.test/", {
135 headers: {
136 host: "kid--f.test",
137 "x-forwarded-for": "6.6.6.6",
138 forwarded: "for=6.6.6.6",
139 "x-real-ip": "6.6.6.6",
140 "cf-connecting-ip": "6.6.6.6",
141 "x-edge-ip": "6.6.6.6",
142 },
143 });
144 notePeer(request, "203.0.113.9");
145 await forward({ port: server.port! }, request, "x-edge-ip");
146 expect(seen?.get("x-forwarded-for")).toBe("203.0.113.9");
147 for (const h of ["forwarded", "x-real-ip", "cf-connecting-ip", "x-edge-ip"]) expect(seen?.get(h)).toBeNull();
148 } finally {
149 server.stop(true);
150 }
151 });
152 });
153
154 describe("a child that will not start", () => {
155 test.skipIf(cells.ok)("is not relaunched on every request", async () => {
156 const warned: string[] = [];
157 const app = { config: { dataDir: tmp, url: "https://f.test" }, log: { warn: (m: string) => warned.push(m), info() {}, error() {} } } as unknown as App;
158 const children = cellChildren(app);
159 const input = { bodyId: "cell-x", name: "kid", size: "small" } as const;
160 expect(await children.start(input)).toBeUndefined();
161 expect(await children.start(input)).toBeUndefined();
162 expect(warned.length).toBe(1);
163 });
164 });
165
166 describe("a real child forge", () => {
167 test.skipIf(!cells.ok)("boots in its cell and answers /healthz through the forward", async () => {
168 const parent = createApp({ dataDir: join(tmp, "real-parent"), url: "https://f.test" });
169 try {
170 const children = cellChildren(parent);
171 const child = await children.start({ bodyId: "cell-real", name: "kid", size: "small" });
172 expect(children.failure("cell-real")).toBeUndefined();
173 if (child === undefined) throw new Error(`the child did not start: ${children.failure("cell-real")}`);
174 expect(child.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 });
175 const res = await forward(child, new Request("http://kid--f.test/healthz", { headers: { host: "kid--f.test" } }));
176 expect(res.status).toBe(200);
177 await children.stop("cell-real");
178 } finally {
179 await stopCellChildren(parent);
180 parent.close();
181 }
182 }, 60_000);
183 });
184
185 describe("a child whose cell will not come up", () => {
186 test.skipIf(cells.ok)("is hosted instead, and the ledger says why", async () => {
187 const dataDir = join(tmp, "demote-parent");
188 const parent = createApp({ dataDir, url: "https://f.test" });
189 try {
190 mkdirSync(join(dataDir, "spawn"), { recursive: true });
191 const file = join(dataDir, "spawn", "kid.json");
192 writeFileSync(file, JSON.stringify({ name: "kid", bodyId: "cell-d", done: true, placement: "cell", size: "small" }));
193 const child = await resolveHostedChild(parent, new Request("https://kid--f.test/", { headers: { host: "kid--f.test" } }));
194 expect(child).not.toBe("cell");
195 expect(child).not.toBe("unknown");
196 const ledger = JSON.parse(readFileSync(file, "utf8"));
197 expect(ledger.placement).toBe("hosted");
198 expect(ledger.placementNote).toStartWith("hosted: ");
199 } finally {
200 await closeHostedChildren(parent);
201 parent.close();
202 }
203 });
204 });
test/spawn-host.test.ts +1 −1
@@ -150,7 +150,7 @@ describe("a parent on plain http with a port", () => {
150150 parent,
151151 new Request("http://forge.lan:8080/healthz", { headers: { host: "kid--forge.lan:8080" } }),
152152 );
153 if (child === undefined || child === "unknown") throw new Error("child not found");
153 if (child === undefined || child === "unknown" || child === "cell") throw new Error("child not found");
154154 expect(child.config.url).toBe("http://kid--forge.lan:8080");
155155 });
156156 });
test/spawn-pipes.test.ts +1
@@ -16,6 +16,7 @@ import { join } from "node:path";
1616 const PIPING: Record<string, { sites: string[]; reason: string }> = {
1717 "src/apps/contain.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" },
1818 "src/apps/process.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "pump() reads stdout and stderr of the app until exit" },
19 "src/spawn/cell.ts": { sites: ["stdout+stderr"], reason: "drain() reads the child forge's stdout and stderr to its log until exit" },
1920 "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" },
2021 "src/crew/claude.ts": { sites: ["stdout+stderr", "stdout"], reason: "session pumps stdout and stderr; --version awaits stdout, stderr ignored" },
2122 "src/crew/plan.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs