Public: anyone can read this; the crew can change it.

Step 1 of a cell outliving its host: a child cell's egress proxy, loopback relays and inbound relay run in a supervisor of their own (FORGE_CELLS_OUTLIVE, off) #13761

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13755-a-cell-child-s-egress-proxy-inbound-rela → main 8 commits fork 1 thread · 1 unresolved

  • Merged. scout/13755-a-cell-child-s-egress-proxy-inbound-rela was merged into main as 49f61cb.

Review

Sign in to review this change.
10 files changed +893 −56
src/apps/cell_policy.ts added +44
@@ -0,0 +1,44 @@
1 /**
2 * A cell's way out and in, as plain data: what its egress proxy lets through and which relays it
3 * gets. One value is built per cell (`openCell`), and both the host's proxy and a supervisor's
4 * (`spawn/supervisor.ts`, #13755) turn it into proxy options here, so the two cannot drift.
5 */
6 import { cellProxy, type EgressPolicy } from "./egress.ts";
7
8 export interface CellPolicy {
9 readonly hosts: readonly string[];
10 readonly granted?: readonly string[];
11 readonly origins: readonly { host: string; port: number }[];
12 readonly label?: string;
13 /** Let undeclared public hosts through, and say so, instead of refusing them. */
14 readonly report: boolean;
15 /** Loopback ports of the forge the cell reaches. */
16 readonly loops: readonly number[];
17 /** The port the forge reaches an app by; relayed into the cell. */
18 readonly listen?: number;
19 }
20
21 /** Where what the proxy decides is told: whoever runs the proxy supplies these. */
22 export interface PolicyHooks {
23 readonly allowed?: (() => readonly string[]) | undefined;
24 readonly refused?: ((host: string, port: number, isPrivate: boolean) => void) | undefined;
25 readonly undeclared?: ((host: string, port: number) => void) | undefined;
26 }
27
28 /** The egress proxy's options for `policy`. */
29 export function proxyOptions(policy: CellPolicy, hooks: PolicyHooks): EgressPolicy {
30 return {
31 hosts: policy.hosts,
32 granted: policy.granted,
33 label: policy.label,
34 origins: policy.origins,
35 allowed: hooks.allowed,
36 refused: hooks.refused,
37 // A cell a child forge opens (its check, app or step) has no way out but the proxy of the cell
38 // the child runs in. A child in a cell is the forge told its parent's door; a check that boots
39 // a forge is not.
40 upstream: cellProxy(),
41 // Enforce unless the caller carries an admin's report (`egressSetting`, #10778).
42 report: policy.report ? (host, port) => hooks.undeclared?.(host, port) : undefined,
43 };
44 }
src/apps/contain.ts +56 −40
@@ -37,7 +37,9 @@ import type { Database } from "bun:sqlite";
3737 import { availableParallelism, tmpdir } from "node:os";
3838 import { dirname, join } from "node:path";
3939 import { portInsteadOfSocket } from "../git/hook-address.ts";
40 import { cellProxy, relay, startEgress, type Egress } from "./egress.ts";
40 import { relay, startEgress, type Egress } from "./egress.ts";
41 import { proxyOptions, type CellPolicy } from "./cell_policy.ts";
42 import { startSupervisor, type Supervised } from "../spawn/supervisor.ts";
4143 import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
4244 import { cellOom, oomFirst } from "./oom.ts";
4345 import {
@@ -386,6 +388,12 @@ export interface CellSpec {
386388 * forge's comes from its size (`spawn/sizes.ts`); unset, no cap.
387389 */
388390 readonly io?: IoMax | undefined;
391 /**
392 * The cell outlives the host process (#13755): its proxy, relays and way in run in a supervisor
393 * of their own, the cell is not tied to its parent, and its output goes to a file. Off unless
394 * asked.
395 */
396 readonly outlive?: boolean | undefined;
389397 }
390398
391399 /**
@@ -417,6 +425,8 @@ export interface Cell {
417425 * does.
418426 */
419427 readonly cgroupDir?: string;
428 /** Present when the cell outlives the host: its supervisor, and the file its output goes to. */
429 readonly supervised?: Supervised;
420430 }
421431
422432 // ---- can this host hold a cell? ----
@@ -497,7 +507,7 @@ async function tryOnce(extra: readonly string[] = []): Promise<Verdict> {
497507 const proc = Bun.spawn(
498508 [
499509 "bwrap",
500 ...NAMESPACES,
510 ...namespaces(false),
501511 ...extra,
502512 "--ro-bind",
503513 "/",
@@ -544,8 +554,9 @@ async function tryOnce(extra: readonly string[] = []): Promise<Verdict> {
544554 }
545555 }
546556
547 const NAMESPACES = [
548 "--die-with-parent",
557 /** The namespaces a cell is made with; one that outlives the host is not tied to its parent. */
558 const namespaces = (outlive: boolean): string[] => [
559 ...(outlive ? [] : ["--die-with-parent"]),
549560 "--unshare-user",
550561 "--unshare-pid",
551562 "--unshare-net",
@@ -967,6 +978,20 @@ export function cellArgv(
967978 return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned);
968979 }
969980
981 /** The one egress policy a cell is served by, host or supervisor (`cell_policy.ts`). */
982 export function cellPolicy(spec: CellSpec): CellPolicy {
983 const origins = spec.reach.map(origin).filter((o) => o !== undefined);
984 return {
985 hosts: spec.egress,
986 ...(spec.granted === undefined ? {} : { granted: spec.granted }),
987 origins,
988 ...(spec.label === undefined ? {} : { label: spec.label }),
989 report: spec.egressMode === "report",
990 loops: [...new Set(origins.filter((o) => isLoopback(o.host)).map((o) => o.port))],
991 ...(spec.listen === undefined ? {} : { listen: spec.listen }),
992 };
993 }
994
970995 /** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
971996 export function openCell(
972997 spec: CellSpec,
@@ -974,39 +999,25 @@ export function openCell(
974999 ): Cell {
9751000 const sockets = mkdtempSync(join(tmpdir(), "fc-"));
9761001 const opened: Egress[] = [];
977 const origins = spec.reach.map(origin).filter((o) => o !== undefined);
978 const loops = origins.filter((o) => isLoopback(o.host)).map((o) => o.port);
979 opened.push(
980 startEgress(
981 join(sockets, "out.sock"),
982 {
983 hosts: spec.egress,
984 granted: spec.granted,
985 label: spec.label,
986 allowed: spec.allowed,
987 refused: spec.onRefused,
988 origins,
989 // A cell a child forge opens (its check, app or step) has no way out
990 // but the proxy of the cell the child runs in. A child in a cell is
991 // the forge told its parent's door; a check that boots a forge is not.
992 upstream: cellProxy(),
993 report:
994 // Enforce unless the caller carries an admin's report (`egressSetting`, #10778).
995 spec.egressMode === "report"
996 ? (host, port) => spec.onUndeclared?.(host, port)
997 : undefined,
998 },
999 log,
1000 ),
1001 );
1002 for (const port of new Set(loops))
1003 opened.push(
1004 relay({ unix: join(sockets, `loop-${port}.sock`) }, { tcp: port }),
1005 );
1006 if (spec.listen !== undefined)
1007 opened.push(
1008 relay({ tcp: spec.listen }, { unix: join(sockets, "in.sock") }),
1009 );
1002 const policy = cellPolicy(spec);
1003 const hooks = { allowed: spec.allowed, refused: spec.onRefused, undeclared: spec.onUndeclared };
1004 // A cell that outlives the host has these run by a supervisor of its own; the rest, by the host.
1005 // Both serve the one `policy`.
1006 const supervised =
1007 spec.outlive === true
1008 ? startSupervisor(mkdtempSync(join(tmpdir(), "fc-ctl-")), { sockets, ...policy }, { ...hooks, log })
1009 : undefined;
1010 if (supervised === undefined) {
1011 opened.push(startEgress(join(sockets, "out.sock"), proxyOptions(policy, hooks), log));
1012 for (const port of policy.loops)
1013 opened.push(
1014 relay({ unix: join(sockets, `loop-${port}.sock`) }, { tcp: port }),
1015 );
1016 if (policy.listen !== undefined)
1017 opened.push(
1018 relay({ tcp: policy.listen }, { unix: join(sockets, "in.sock") }),
1019 );
1020 }
10101021
10111022 const { hide, blank, reveal } = cellMounts(spec, hiddenMounts());
10121023 const asked = [...new Set([spec.dir, spec.home, ...spec.writable])];
@@ -1030,11 +1041,14 @@ export function openCell(
10301041 const cgroup = cellCgroup(spec);
10311042 if (spec.cpuQuota === true) noteCpuHolder(cgroup?.cpuCapped === true, log);
10321043 noteServingCores(servingCoresNow(), Bun.which("taskset") !== null, log);
1033 const argv = (command: string): string[] => cellArgv(bwrapArgv(command), spec, cgroup);
1044 const argv = (command: string): string[] => {
1045 const inner = cellArgv(bwrapArgv(command), spec, cgroup);
1046 return supervised === undefined ? inner : ["setsid", ...inner];
1047 };
10341048 const rlimits = cellRlimits(spec.limits);
10351049 const bwrapArgv = (command: string): string[] => [
10361050 "bwrap",
1037 ...NAMESPACES,
1051 ...namespaces(spec.outlive === true),
10381052 ...hostnameArgs(spec.hostname),
10391053 ...(spec.terminal === true ? [] : ["--new-session"]),
10401054 "--ro-bind", "/", "/",
@@ -1059,7 +1073,7 @@ export function openCell(
10591073 WRAPPER,
10601074 "cell",
10611075 spec.listen === undefined ? "-" : String(spec.listen),
1062 [...new Set(loops)].join(" "),
1076 policy.loops.join(" "),
10631077 String(rlimits.fileBlocks),
10641078 String(rlimits.files),
10651079 String(rlimits.tasks),
@@ -1117,6 +1131,7 @@ export function openCell(
11171131 env,
11181132 watch,
11191133 ...(cgroup === undefined ? {} : { cgroupDir: cgroup.dir }),
1134 ...(supervised === undefined ? {} : { supervised }),
11201135 breach: () => breach ?? killed(),
11211136 close: () => {
11221137 const kernel = breach === undefined ? killed() : undefined;
@@ -1124,6 +1139,7 @@ export function openCell(
11241139 cgroup?.remove();
11251140 unwatch?.();
11261141 for (const o of opened) o.close();
1142 supervised?.close();
11271143 rmSync(sockets, { recursive: true, force: true });
11281144 },
11291145 };
src/core/capacity.ts +22 −6
@@ -225,7 +225,10 @@ function explicitDefault(key: CapacityKey): string | undefined {
225225
226226 /** Most shards a check takes by default: past four, a test run waits on git, not CPU. */
227227 const SHARDS_DEFAULT_MAX = 4;
228 /** CPUs a shard is given by default: its bun test, the git children it starts, and the forge it boots. */
228 /**
229 * CPUs a shard is given by default: its bun test, the git children it starts, and the forge it
230 * boots.
231 */
229232 const CPUS_PER_SHARD = 3;
230233
231234 /** Shards per check by default: `min(4, cpus)`, at least one. 80 CPUs give 4; 2 give 2. */
@@ -501,7 +504,10 @@ export function followsSpeed(key: CapacityKey): key is SpeedDerivedKey {
501504 return key === "pace.floor" || key === "queue.depth";
502505 }
503506
504 /** What Speed implies for a dial that follows it: half of Speed, at least 1 for the floor; half rounded up, 1 to 4, for the depth. */
507 /**
508 * What Speed implies for a dial that follows it: half of Speed, at least 1 for the floor; half
509 * rounded up, 1 to 4, for the depth.
510 */
505511 export function derivedFromSpeed(db: Database, key: SpeedDerivedKey): number {
506512 const speed = Number(capacityValue(db, "crew.concurrency"));
507513 if (key === "pace.floor") return Math.max(1, Math.floor(speed / 2));
@@ -562,12 +568,17 @@ const MODEL_HOSTS: Readonly<Record<string, readonly string[]>> = {
562568 both: ["openrouter.ai:443", "api.anthropic.com:443"],
563569 };
564570
565 /** Whether a child forge must have a filesystem of its own to start (`children.ownFilesystem`); off by default. */
571 /**
572 * Whether a child forge must have a filesystem of its own to start (`children.ownFilesystem`); off
573 * by default.
574 */
566575 export function childNeedsOwnFilesystem(db: Database): boolean {
567576 return capacityValue(db, "children.ownFilesystem") === "on";
568577 }
569578
570 /** The package registry a child forge cell may reach (`children.registryEgress`); none by default. */
579 /**
580 * The package registry a child forge cell may reach (`children.registryEgress`); none by default.
581 */
571582 export function childRegistryHosts(db: Database): readonly string[] {
572583 const value = capacityValue(db, "children.registryEgress");
573584 return value === "build" ? BUILD_HOSTS : value === "npm" ? ["registry.npmjs.org:443"] : [];
@@ -599,7 +610,10 @@ export function childOriginHosts(db: Database, parentUrl: string): readonly stri
599610
600611 const GIT_HOSTS: readonly string[] = ["github.com", "api.github.com", "codeload.github.com", "gitlab.com", "codeberg.org", "bitbucket.org"];
601612
602 /** The payment provider hosts a child forge cell may reach (`children.paymentEgress`); the cell's ceiling, each app still declares its own. */
613 /**
614 * The payment provider hosts a child forge cell may reach (`children.paymentEgress`); the cell's
615 * ceiling, each app still declares its own.
616 */
603617 export function childPaymentHosts(db: Database): readonly string[] {
604618 return capacityValue(db, "children.paymentEgress") === "stripe" ? PAYMENT_HOSTS : [];
605619 }
@@ -670,7 +684,9 @@ export function setCapacity(
670684 key: CapacityKey,
671685 value: string | null,
672686 actorId: string,
673 /** Why, in the admin's words, already cut to REASON_MAX by the action; kept beside who (#10778). */
687 /**
688 * Why, in the admin's words, already cut to REASON_MAX by the action; kept beside who (#10778).
689 */
674690 reason = "",
675691 ): { from: string; to: string } {
676692 const from = capacityValue(db, key);
src/kernel/destroy_reach.ts +2
@@ -82,6 +82,8 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
8282 { unit: "src/git/repos.ts:detachFork", why: "removes the alternates file of the fork just made" },
8383 { unit: "src/git/env.ts:trustedCaBundle", why: "removes the process's own temporary CA bundle at exit" },
8484 { unit: "src/apps/contain.ts:openCell", why: "clears the cell's socket directory before the cell opens" },
85 { unit: "src/spawn/supervisor.ts:startSupervisor", why: "removes the cell's own control directory (a fresh temp directory it made) when the host closes the cell" },
86 { unit: "src/spawn/supervisor.ts:runSupervisor", why: "removes the cell's own socket directory and, when no host is left, its control directory: both fresh temp directories made for that cell" },
8587 { unit: "src/apps/cgroup.ts:removeDir", why: "removes an empty cgroup directory" },
8688 { unit: "src/actions/spawn.ts:unmake", why: "removes the directory and ledger of a child whose first attempt failed before it was ever finished, and only that call's own (never a finished or resumed one): nothing of anyone's work is in it (#13135)" },
8789 { unit: "src/spawn/own_code.ts:checkout", why: "removes a half-made export of the forge's own code, keyed by commit" },
src/spawn/cell.ts +54 −10
@@ -28,6 +28,15 @@ import { liveChildDisk, type ChildDisk } from "./disk_mount.ts";
2828 import { oneAtATime, sourcesFor, startFailure, startFirstThatWorks, useTree, type CodeSource } from "./own_code.ts";
2929 import { childGate, type ChildGate } from "./child_reach.ts";
3030
31 /**
32 * Whether a child's cell outlives this process (`FORGE_CELLS_OUTLIVE=on`; #13755): its proxy and
33 * relays run in a supervisor of their own, the cell is not tied to its parent, and a stop of this
34 * process leaves it running. Off by default and undocumented: nothing yet re-adopts or reaps a cell
35 * on start (#13750 step 2), its beat door (`beat-door.ts`) still runs in this process, and its
36 * egress bytes are counted in the supervisor, out of this process's gauges (#13921).
37 */
38 export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on";
39
3140 export const CELL_PREFIX = "cell-";
3241 export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX);
3342
@@ -179,6 +188,7 @@ export function cellSpecFor(input: {
179188 // cannot starve the forge's database of the disk (#10854).
180189 io: SIZE_IO[input.size],
181190 listen: input.port,
191 ...(cellsOutlive() ? { outlive: true } : {}),
182192 label: input.label,
183193 // Its shell's prompt names the child, never the parent's pod (#10850).
184194 hostname: input.name,
@@ -272,12 +282,45 @@ export const SLEEP_AT_ONCE = 8;
272282
273283 const registries = new WeakMap<App, CellChildren>();
274284
275 export function cellChildren(parent: App): CellChildren {
285 /**
286 * The parent's one registry of child cells. `launcher` stands in for the cell launch the first time
287 * the registry is made (tests); later calls ignore it.
288 */
289 export function cellChildren(parent: App, launcher?: Launcher): CellChildren {
276290 let r = registries.get(parent);
277 if (r === undefined) registries.set(parent, (r = makeRegistry(parent)));
291 if (r === undefined) registries.set(parent, (r = makeRegistry(parent, launcher)));
278292 return r;
279293 }
280294
295 /**
296 * Run `command` in `cell` and wire its output to `log`. A supervised cell (one that outlives this
297 * process, #13755) writes to its supervisor's file, is adopted as the supervisor's root, and is
298 * followed there; any other is read from pipes this process holds. `unfollow` stops the following.
299 */
300 export function spawnInCell(
301 cell: Pick<Cell, "argv" | "env" | "supervised">,
302 command: string,
303 at: { cwd: string; env: Record<string, string> },
304 log: (line: string) => void,
305 ): { proc: ReturnType<typeof Bun.spawn>; unfollow: () => void } {
306 const sup = cell.supervised;
307 const proc = Bun.spawn(cell.argv(command), {
308 cwd: at.cwd,
309 env: cell.env(at.env),
310 stdin: "ignore",
311 // A cell that outlives this process writes to a file, not a pipe this process reads.
312 stdout: sup?.logFd ?? "pipe",
313 stderr: sup?.logFd ?? "pipe",
314 });
315 if (sup === undefined) {
316 void drain(proc.stdout, log);
317 void drain(proc.stderr, log);
318 return { proc, unfollow: () => {} };
319 }
320 sup.adopt(proc.pid);
321 return { proc, unfollow: sup.follow(log) };
322 }
323
281324 /**
282325 * Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not.
283326 */
@@ -403,14 +446,12 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
403446 throw error;
404447 }
405448 const command = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(source.main)}`;
406 const proc = Bun.spawn(cell.argv(command), {
449 const { proc, unfollow } = spawnInCell(cell, command, {
407450 cwd: dirname(source.main),
408 env: cell.env(childEnvFor({ port, dataDir: dir, url, parentPort, size: input.size, ...(cell.cgroupDir === undefined ? {} : { cgroupDir: cell.cgroupDir }) })),
409 stdin: "ignore",
410 stdout: "pipe",
411 stderr: "pipe",
412 });
451 env: childEnvFor({ port, dataDir: dir, url, parentPort, size: input.size, ...(cell.cgroupDir === undefined ? {} : { cgroupDir: cell.cgroupDir }) }),
452 }, log);
413453 const exited = proc.exited.then((code) => {
454 unfollow();
414455 const why = cell.breach();
415456 if (why !== undefined) {
416457 parent.log.warn("child forge stopped", { name: input.name, why });
@@ -424,8 +465,6 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
424465 return code;
425466 });
426467 cell.watch(proc.pid, () => killTree(proc.pid));
427 void drain(proc.stdout, log);
428 void drain(proc.stderr, log);
429468 const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc, door };
430469 let gone = false;
431470 void exited.then(() => (gone = true));
@@ -617,6 +656,11 @@ export async function stopCellChildren(parent: App): Promise<void> {
617656 const r = registries.get(parent);
618657 if (r === undefined) return;
619658 registries.delete(parent);
659 // A cell that outlives this process is left running; its supervisor holds its way in and out.
660 if (cellsOutlive()) {
661 closeChildDb();
662 return;
663 }
620664 await Promise.all(r.list().map((c) => r.stop(c.bodyId)));
621665 // The workers that ask the children's databases go with them; one is started again when next
622666 // asked.
src/spawn/supervisor.ts added +396
@@ -0,0 +1,396 @@
1 /**
2 * A cell's supervisor: the process that owns what the cell talks to (the egress proxy, the loopback
3 * relays, the way in) and the files the host reads it by, so the cell can outlive the host process
4 * (#13755, step 1 of #13750).
5 *
6 * It runs in its own session (`setsid`), not as a child that dies with the host. The host talks to
7 * it through plain files in one control directory, never to the cell:
8 *
9 * config.json what to serve (host writes, once)
10 * host.pid the host that started it (host writes)
11 * cell.pid the cell's root process (host writes once it is spawned)
12 * cell.log the cell's stdout and stderr, as a file the cell holds open
13 * allowed.json egress entries allowed since the cell started (host rewrites when they change)
14 * events.jsonl refusals and undeclared hosts (supervisor appends; the host tails)
15 *
16 * It ends when the cell's root process is gone, on SIGTERM, or when no cell appears in a minute.
17 * The host side of this file is `startSupervisor`; `bun supervisor.ts <ctl dir>` is the other.
18 */
19 import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, truncateSync, writeFileSync } from "node:fs";
20 import { join } from "node:path";
21 import { during } from "../perf/jobs.ts";
22 import { PROXY_VARS, relay, startEgress, type Egress } from "../apps/egress.ts";
23 import { proxyOptions, type CellPolicy } from "../apps/cell_policy.ts";
24
25 /** Plain data: what a supervisor serves. */
26 export interface SupervisorConfig extends CellPolicy {
27 /** The directory the cell sees at `/run/forge`; the sockets are made in it. */
28 readonly sockets: string;
29 /** Tests: the proxy listens on this loopback port, where a unix socket cannot be bound. */
30 readonly outPort?: number;
31 }
32
33 export type SupervisorEvent =
34 | { e: "refused"; host: string; port: number; private: boolean }
35 | { e: "undeclared"; host: string; port: number }
36 | { e: "log"; line: string };
37
38 const WAIT_FOR_CELL_MS = 60_000;
39 const TICK_MS = 1_000;
40 /**
41 * The most the cell's output file may hold; past it the file starts over (the cell keeps writing).
42 */
43 export const LOG_MAX_BYTES = 4_000_000;
44 /** The most the events file may hold. */
45 export const EVENTS_MAX_BYTES = 1_000_000;
46 /** Events the supervisor writes in a minute; past it they are dropped. */
47 export const EVENTS_PER_MINUTE = 300;
48
49 /**
50 * Start `path` over when it is past `max` bytes: a writer holding it open in append mode goes on.
51 */
52 export function capFile(path: string, max: number): boolean {
53 try {
54 if (statSync(path).size <= max) return false;
55 truncateSync(path, 0);
56 return true;
57 } catch {
58 return false;
59 }
60 }
61
62 /** A counter that says yes to `limit` calls in each minute, and no to the rest. */
63 export function perMinute(limit: number, now: () => number = Date.now): () => boolean {
64 let start = now();
65 let n = 0;
66 return () => {
67 const t = now();
68 if (t - start >= 60_000) {
69 start = t;
70 n = 0;
71 }
72 return ++n <= limit;
73 };
74 }
75
76 export const files = (ctl: string) => ({
77 config: join(ctl, "config.json"),
78 host: join(ctl, "host.pid"),
79 cell: join(ctl, "cell.pid"),
80 log: join(ctl, "cell.log"),
81 allowed: join(ctl, "allowed.json"),
82 events: join(ctl, "events.jsonl"),
83 });
84
85 export function alive(pid: number): boolean {
86 if (!Number.isInteger(pid) || pid <= 0) return false;
87 try {
88 process.kill(pid, 0);
89 return true;
90 } catch (error) {
91 return (error as NodeJS.ErrnoException).code === "EPERM";
92 }
93 }
94
95 /**
96 * When a process started, in clock ticks since boot (`/proc/<pid>/stat`); undefined where unknown.
97 */
98 export function startTime(pid: number): string | undefined {
99 try {
100 const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
101 return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19];
102 } catch {
103 return undefined;
104 }
105 }
106
107 /** A pid file holds `pid`, and `pid:start` when known: a pid whose start differs was reused. */
108 export const pidRecord = (pid: number): string => {
109 const start = startTime(pid);
110 return start === undefined ? String(pid) : `${pid}:${start}`;
111 };
112
113 /** The pid in a pid file that is still the same process; undefined when it is gone or reused. */
114 export function readPid(path: string): number | undefined {
115 try {
116 const [a, start] = readFileSync(path, "utf8").trim().split(":");
117 const n = Number(a);
118 if (!Number.isInteger(n) || n <= 0) return undefined;
119 if (start !== undefined && startTime(n) !== undefined && startTime(n) !== start) return undefined;
120 return n;
121 } catch {
122 return undefined;
123 }
124 }
125
126 /** The entries of `allowed.json`, only ever strings; what is not is ignored. */
127 export function readAllowed(path: string): string[] {
128 try {
129 const value: unknown = JSON.parse(readFileSync(path, "utf8"));
130 return Array.isArray(value) ? value.filter((v): v is string => typeof v === "string") : [];
131 } catch {
132 return [];
133 }
134 }
135
136 const isPort = (v: unknown): v is number => Number.isInteger(v) && (v as number) > 0 && (v as number) < 65_536;
137 const strings = (v: unknown): v is string[] => Array.isArray(v) && v.every((x) => typeof x === "string");
138
139 /** A `config.json` checked field by field; undefined when it is not one. */
140 export function parseConfig(text: string): SupervisorConfig | undefined {
141 try {
142 const v = JSON.parse(text) as Record<string, unknown>;
143 if (typeof v["sockets"] !== "string" || v["sockets"] === "" || !strings(v["hosts"])) return undefined;
144 if (v["granted"] !== undefined && !strings(v["granted"])) return undefined;
145 if (v["label"] !== undefined && typeof v["label"] !== "string") return undefined;
146 if (typeof v["report"] !== "boolean") return undefined;
147 if (!Array.isArray(v["loops"]) || !v["loops"].every(isPort)) return undefined;
148 if (v["listen"] !== undefined && !isPort(v["listen"])) return undefined;
149 if (v["outPort"] !== undefined && !isPort(v["outPort"])) return undefined;
150 const origins = v["origins"];
151 if (!Array.isArray(origins)) return undefined;
152 for (const o of origins as Record<string, unknown>[]) if (typeof o !== "object" || o === null || typeof o["host"] !== "string" || !isPort(o["port"])) return undefined;
153 return v as unknown as SupervisorConfig;
154 } catch {
155 return undefined;
156 }
157 }
158
159 /** Parse one line of `events.jsonl`; undefined for anything that is not an event. */
160 export function parseEvent(line: string): SupervisorEvent | undefined {
161 try {
162 const v = JSON.parse(line) as Record<string, unknown>;
163 if (v["e"] === "log" && typeof v["line"] === "string") return { e: "log", line: v["line"].slice(0, 500) };
164 if (typeof v["host"] !== "string" || typeof v["port"] !== "number") return undefined;
165 if (v["e"] === "undeclared") return { e: "undeclared", host: v["host"], port: v["port"] };
166 if (v["e"] === "refused") return { e: "refused", host: v["host"], port: v["port"], private: v["private"] === true };
167 } catch {
168 // not an event
169 }
170 return undefined;
171 }
172
173 /**
174 * Serve `config` until the cell is gone or `stop` is called. Resolves once everything is closed.
175 */
176 export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT_FOR_CELL_MS): { done: Promise<void>; stop: () => void } {
177 const f = files(ctl);
178 const eventAllowed = perMinute(EVENTS_PER_MINUTE);
179 const say = (event: SupervisorEvent): void => {
180 if (!eventAllowed()) return;
181 try {
182 appendFileSync(f.events, `${JSON.stringify(event)}\n`);
183 } catch {
184 // the control directory is gone: the host closed it
185 }
186 };
187 let cache: { at: number; list: string[] } = { at: -1, list: [] };
188 const allowed = (): readonly string[] => {
189 try {
190 const at = statSync(f.allowed).mtimeMs;
191 if (at !== cache.at) cache = { at, list: readAllowed(f.allowed) };
192 } catch {
193 // none yet
194 }
195 return cache.list;
196 };
197 const opened: Egress[] = [];
198 // TODO(#13921): `recordEgress` counts this proxy's bytes in this process, so the host's egress
199 // gauges miss an outliving cell's traffic until they are summed here and sent back as events.
200 const policy = proxyOptions(config, {
201 allowed,
202 refused: (host, port, isPrivate) => say({ e: "refused", host, port, private: isPrivate }),
203 undeclared: (host, port) => say({ e: "undeclared", host, port }),
204 });
205 const out = config.outPort === undefined ? join(config.sockets, "out.sock") : { tcp: config.outPort };
206 opened.push(startEgress(out, policy, (line) => say({ e: "log", line })));
207 for (const port of new Set(config.loops)) opened.push(relay({ unix: join(config.sockets, `loop-${port}.sock`) }, { tcp: port }));
208 if (config.listen !== undefined) opened.push(relay({ tcp: config.listen }, { unix: join(config.sockets, "in.sock") }));
209
210 const began = Date.now();
211 let finish: () => void = () => {};
212 const done = new Promise<void>((resolve) => (finish = resolve));
213 let closed = false;
214 const stop = (): void => {
215 if (closed) return;
216 closed = true;
217 clearInterval(timer);
218 for (const o of opened) o.close();
219 rmSync(config.sockets, { recursive: true, force: true });
220 // The host reads the last of the log and events when it is there; with no host, nobody will.
221 const host = readPid(f.host);
222 if (host === undefined || !alive(host)) rmSync(ctl, { recursive: true, force: true });
223 finish();
224 };
225 // Its own process has no job ring: this is the only timer it has.
226 const timer = setInterval(() => {
227 capFile(f.log, LOG_MAX_BYTES);
228 capFile(f.events, EVENTS_MAX_BYTES);
229 const cell = readPid(f.cell);
230 if (cell !== undefined ? !alive(cell) : Date.now() - began > wait) stop();
231 }, TICK_MS);
232 return { done, stop };
233 }
234
235 // ---- the host's side ----
236
237 export interface SupervisorHooks {
238 /**
239 * Egress entries allowed since the cell started; rewritten to the supervisor when they change.
240 */
241 readonly allowed?: (() => readonly string[]) | undefined;
242 readonly onRefused?: ((host: string, port: number, isPrivate: boolean) => void) | undefined;
243 readonly onUndeclared?: ((host: string, port: number) => void) | undefined;
244 readonly log: (line: string) => void;
245 }
246
247 export interface Supervised {
248 readonly ctl: string;
249 readonly pid: number;
250 /** An open file the cell writes its stdout and stderr to; hand it to the spawn, then `follow`. */
251 readonly logFd: number;
252 /** Tell the supervisor which process is the cell's root: it ends when that is gone. */
253 readonly adopt: (pid: number) => void;
254 /** Call `line` with each new line of the cell's output; returns what stops it. */
255 readonly follow: (line: (text: string) => void) => () => void;
256 /** Stop the supervisor and remove the control directory. */
257 readonly close: () => void;
258 }
259
260 /**
261 * Call `line` for each new line appended to `path`, polling; the returned function stops, after a
262 * last read.
263 */
264 export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES): () => void {
265 let offset = 0;
266 let rest = "";
267 const read = (): void => {
268 let fd: number;
269 try {
270 fd = openSync(path, "r");
271 } catch {
272 return;
273 }
274 try {
275 const size = statSync(path).size;
276 // The file was started over (`capFile`): read it again from its top.
277 if (size < offset) offset = 0;
278 while (offset < size) {
279 const buf = Buffer.alloc(Math.min(65_536, size - offset));
280 const n = readSync(fd, buf, 0, buf.length, offset);
281 if (n <= 0) break;
282 offset += n;
283 rest += buf.subarray(0, n).toString("utf8");
284 const cut = rest.split("\n");
285 rest = cut.pop() ?? "";
286 if (rest.length > 4_000) {
287 cut.push(rest);
288 rest = "";
289 }
290 for (const l of cut) if (l !== "") line(l);
291 }
292 } finally {
293 closeSync(fd);
294 }
295 };
296 // The host bounds the file too: the supervisor may be gone, and the cell not.
297 const timer = setInterval(() => void during("cell output tail", () => (read(), capFile(path, max))), everyMs);
298 timer.unref();
299 return () => {
300 clearInterval(timer);
301 read();
302 if (rest !== "") line(rest);
303 rest = "";
304 };
305 }
306
307 /**
308 * The environment a supervisor needs: where to find tools, and the way out a child forge's cell
309 * has.
310 */
311 function supervisorEnv(): Record<string, string> {
312 const env: Record<string, string> = {};
313 for (const key of ["PATH", "FORGE_PARENT_LOCAL", "FORGE_TEST_SOCKET", ...PROXY_VARS]) {
314 const v = process.env[key];
315 if (v !== undefined) env[key] = v;
316 }
317 return env;
318 }
319
320 /** Start a supervisor in its own session, and the host's side of talking to it. */
321 export function startSupervisor(ctl: string, config: SupervisorConfig, hooks: SupervisorHooks): Supervised {
322 const f = files(ctl);
323 mkdirSync(ctl, { recursive: true, mode: 0o700 });
324 writeFileSync(f.config, JSON.stringify(config));
325 writeFileSync(f.host, pidRecord(process.pid));
326 writeFileSync(f.events, "");
327 const logFd = openSync(f.log, "a");
328 const writeAllowed = (() => {
329 let last = "";
330 return (): void => {
331 const now = JSON.stringify([...(hooks.allowed?.() ?? [])]);
332 if (now === last) return;
333 last = now;
334 writeFileSync(f.allowed, now);
335 };
336 })();
337 writeAllowed();
338 const proc = Bun.spawn(["setsid", process.execPath, import.meta.path, ctl], {
339 env: supervisorEnv(),
340 stdin: "ignore",
341 stdout: "ignore",
342 stderr: "ignore",
343 });
344 proc.unref();
345 const sync = setInterval(() => void during("cell allowed sync", writeAllowed), TICK_MS);
346 sync.unref();
347 const stopEvents = tailLines(f.events, (text) => {
348 const event = parseEvent(text);
349 if (event === undefined) return;
350 if (event.e === "log") hooks.log(event.line);
351 else if (event.e === "refused") hooks.onRefused?.(event.host, event.port, event.private);
352 else hooks.onUndeclared?.(event.host, event.port);
353 });
354 let closed = false;
355 return {
356 ctl,
357 pid: proc.pid,
358 logFd,
359 adopt: (pid) => writeFileSync(f.cell, pidRecord(pid)),
360 follow: (line) => tailLines(f.log, line),
361 close: () => {
362 if (closed) return;
363 closed = true;
364 clearInterval(sync);
365 stopEvents();
366 try {
367 process.kill(proc.pid, "SIGTERM");
368 } catch {
369 // already gone
370 }
371 try {
372 closeSync(logFd);
373 } catch {
374 // closed
375 }
376 rmSync(ctl, { recursive: true, force: true });
377 },
378 };
379 }
380
381 if (import.meta.main) {
382 const ctl = process.argv[2];
383 if (ctl === undefined || !existsSync(files(ctl).config)) {
384 console.error("usage: supervisor.ts <control directory>");
385 process.exit(2);
386 }
387 const config = parseConfig(readFileSync(files(ctl).config, "utf8"));
388 if (config === undefined) {
389 console.error("supervisor: config.json is not a config");
390 process.exit(2);
391 }
392 const s = runSupervisor(ctl, config);
393 for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, s.stop);
394 await s.done;
395 process.exit(0);
396 }
test/cell-outlives-host.test.ts added +312
@@ -0,0 +1,312 @@
1 /**
2 * A cell's supervisor owns its proxy, relays and way in outside the host process (#13755): with the
3 * host killed, the cell and its listener stay up, and the supervisor goes when the cell does.
4 */
5 import { afterEach, expect, test } from "bun:test";
6 import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync, appendFileSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { cellPolicy, openCell } from "../src/apps/contain.ts";
10 import { proxyOptions } from "../src/apps/cell_policy.ts";
11 import { createApp } from "../src/app.ts";
12 import { freePort } from "../src/apps/process.ts";
13 import { cellChildren, cellLimits, cellSpecFor, cellsOutlive, spawnInCell, stopCellChildren } from "../src/spawn/cell.ts";
14 import { alive, files, parseConfig, parseEvent, readAllowed, tailLines, type Supervised } from "../src/spawn/supervisor.ts";
15
16 const killed: number[] = [];
17 afterEach(() => {
18 for (const pid of killed.splice(0))
19 try {
20 process.kill(pid, "SIGKILL");
21 } catch {
22 // gone
23 }
24 });
25
26 const until = async (what: () => boolean, ms = 8_000): Promise<boolean> => {
27 const end = Date.now() + ms;
28 while (Date.now() < end) {
29 if (what()) return true;
30 await Bun.sleep(50); // real-timer: polling a real child process
31 }
32 return what();
33 };
34
35 const accepts = (port: number): Promise<boolean> =>
36 new Promise((resolve) => {
37 Bun.connect({ hostname: "127.0.0.1", port, socket: { open: (s) => (s.end(), resolve(true)), data() {}, error: () => resolve(false), connectError: () => resolve(false) } }).catch(() => resolve(false));
38 });
39
40 const pgid = (pid: number): string => Bun.spawnSync(["ps", "-o", "pgid=", "-p", String(pid)]).stdout.toString().trim();
41
42 test("events and the allowed list are read as plain data, and anything else is ignored", () => {
43 expect(parseEvent('{"e":"refused","host":"a.com","port":443,"private":true}')).toEqual({ e: "refused", host: "a.com", port: 443, private: true });
44 expect(parseEvent('{"e":"undeclared","host":"a.com","port":80}')).toEqual({ e: "undeclared", host: "a.com", port: 80 });
45 expect(parseEvent('{"e":"log","line":"hi"}')).toEqual({ e: "log", line: "hi" });
46 expect(parseEvent('{"e":"refused","host":1}')).toBeUndefined();
47 expect(parseEvent("not json")).toBeUndefined();
48 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
49 const path = join(dir, "allowed.json");
50 writeFileSync(path, '["a.com:443", 7, "b.com"]');
51 expect(readAllowed(path)).toEqual(["a.com:443", "b.com"]);
52 expect(readAllowed(join(dir, "missing"))).toEqual([]);
53 });
54
55 test("a file is followed by line, and its last line is read when it stops", async () => {
56 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
57 const path = join(dir, "cell.log");
58 writeFileSync(path, "one\ntw");
59 const heard: string[] = [];
60 const stop = tailLines(path, (l) => heard.push(l), 20);
61 await until(() => heard.length === 1);
62 appendFileSync(path, "o\nthree");
63 await until(() => heard.length === 2);
64 stop();
65 expect(heard).toEqual(["one", "two", "three"]);
66 });
67
68 test("kill the host: the cell and its listener stay up, and the supervisor ends with the cell", async () => {
69 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
70 const ctl = join(dir, "ctl");
71 const sockets = join(dir, "sockets");
72 const listen = freePort();
73 const outPort = freePort();
74 const script = `
75 import { startSupervisor } from ${JSON.stringify(join(import.meta.dir, "../src/spawn/supervisor.ts"))};
76 const s = startSupervisor(${JSON.stringify(ctl)}, { sockets: ${JSON.stringify(sockets)}, hosts: [], origins: [], report: false, loops: [], listen: ${listen}, outPort: ${outPort} }, { log: () => {} });
77 const cell = Bun.spawn(["sleep", "300"], { stdout: s.logFd, stderr: s.logFd });
78 cell.unref();
79 s.adopt(cell.pid);
80 console.log(JSON.stringify({ supervisor: s.pid, cell: cell.pid }));
81 setInterval(() => {}, 1000);
82 `;
83 const host = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "inherit", env: { PATH: process.env["PATH"] ?? "", FORGE_TEST_SOCKET: "port" } });
84 killed.push(host.pid);
85 const text = await new Promise<string>(async (resolve) => {
86 let got = "";
87 for await (const chunk of host.stdout) {
88 got += new TextDecoder().decode(chunk);
89 if (got.includes("\n")) return resolve(got);
90 }
91 resolve(got);
92 });
93 const { supervisor, cell } = JSON.parse(text) as { supervisor: number; cell: number };
94 killed.push(supervisor, cell);
95 expect(await until(() => existsSync(files(ctl).cell))).toBe(true);
96 expect(await until(() => alive(supervisor))).toBe(true);
97 // Its own session: not the host's group, so a signal to the host's group never reaches it.
98 expect(pgid(supervisor)).not.toBe(pgid(host.pid));
99 expect(pgid(supervisor)).toBe(String(supervisor));
100
101 let up = false;
102 for (let i = 0; i < 100 && !up; i++) {
103 up = await accepts(listen);
104 if (!up) await Bun.sleep(50); // real-timer: polling a real listening socket
105 }
106 expect(up).toBe(true);
107
108 host.kill("SIGKILL");
109 await host.exited;
110 await Bun.sleep(500); // real-timer: grace period to prove a real supervisor survives its host being killed
111 expect(alive(supervisor)).toBe(true);
112 expect(alive(cell)).toBe(true);
113 expect(await accepts(listen)).toBe(true);
114 expect(await accepts(outPort)).toBe(true);
115
116 process.kill(cell, "SIGKILL");
117 expect(await until(() => !alive(supervisor), 8_000)).toBe(true);
118 // No host to read what is left: the control directory is gone with the supervisor.
119 expect(existsSync(ctl)).toBe(false);
120 expect(await accepts(listen)).toBe(false);
121 });
122
123 const spec = (dir: string) => cellSpecFor({ dir, parentDataDir: join(dir, "p"), parentUrl: "https://f.test", size: "small", port: freePort(), label: "forge:t" });
124
125 // Runs under FORGE_TEST_SOCKET too: an outliving cell binds no socket in this process (its
126 // supervisor, a process of its own, does), and nothing here waits on that.
127 test("FORGE_CELLS_OUTLIVE=on makes a child's cell its own session, not tied to its parent", () => {
128 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
129 expect(cellsOutlive({})).toBe(false);
130 expect(cellsOutlive({ FORGE_CELLS_OUTLIVE: "on" })).toBe(true);
131 expect(spec(dir).outlive).toBeUndefined();
132 const cell = openCell({ ...spec(dir), outlive: true });
133 try {
134 const argv = cell.argv("true");
135 expect(cell.supervised).toBeDefined();
136 expect(argv[0]).toBe("setsid");
137 expect(argv).not.toContain("--die-with-parent");
138 } finally {
139 cell.close();
140 }
141 expect(existsSync(cell.supervised?.ctl ?? "")).toBe(false);
142 });
143
144 // Unix sockets cannot be bound where FORGE_TEST_SOCKET is set; the forge's check runs this.
145 test.skipIf(process.env["FORGE_TEST_SOCKET"] !== undefined)("the default cell dies with its parent and is not made a session", () => {
146 const cell = openCell(spec(mkdtempSync(join(tmpdir(), "outlive-"))));
147 try {
148 expect(cell.supervised).toBeUndefined();
149 expect(cell.argv("true")).toContain("--die-with-parent");
150 expect(cell.argv("true")[0]).not.toBe("setsid");
151 } finally {
152 cell.close();
153 }
154 });
155
156 test("a cell's output file is bounded: past the cap it starts over and the tail reads the new top", async () => {
157 const { capFile, perMinute, parseConfig, pidRecord, readPid } = await import("../src/spawn/supervisor.ts");
158 const dir = mkdtempSync(join(tmpdir(), "fc-cap-"));
159 const log = join(dir, "cell.log");
160 writeFileSync(log, "x".repeat(2_000));
161 expect(capFile(log, 5_000)).toBe(false);
162 appendFileSync(log, "y".repeat(4_000));
163 expect(capFile(log, 5_000)).toBe(true);
164 expect(Bun.file(log).size).toBe(0);
165 const heard: string[] = [];
166 const stop = tailLines(log, (l) => heard.push(l), 20, 1_000);
167 appendFileSync(log, `${"z".repeat(900)}\n`);
168 expect(await until(() => heard.length === 1)).toBe(true);
169 appendFileSync(log, `${"w".repeat(900)}\n`); // past 1,000 bytes: the tail starts the file over
170 expect(await until(() => heard.length === 2)).toBe(true);
171 expect(await until(() => Bun.file(log).size === 0)).toBe(true);
172 appendFileSync(log, "after\n");
173 expect(await until(() => heard.at(-1) === "after")).toBe(true);
174 stop();
175
176 let t = 0;
177 const ok = perMinute(2, () => t);
178 expect([ok(), ok(), ok()]).toEqual([true, true, false]);
179 t = 60_000;
180 expect(ok()).toBe(true);
181
182 // a config is checked field by field; a pid file with another start time is another process
183 const good = { sockets: "/s", hosts: ["a"], origins: [{ host: "h", port: 80 }], report: false, loops: [8080] };
184 expect(parseConfig(JSON.stringify(good))).toBeDefined();
185 expect(parseConfig(JSON.stringify({ ...good, loops: ["8080"] }))).toBeUndefined();
186 expect(parseConfig(JSON.stringify({ ...good, hosts: [1] }))).toBeUndefined();
187 expect(parseConfig("nope")).toBeUndefined();
188 const pidFile = join(dir, "cell.pid");
189 writeFileSync(pidFile, pidRecord(process.pid));
190 expect(readPid(pidFile)).toBe(process.pid);
191 writeFileSync(pidFile, `${process.pid}:1`);
192 expect(readPid(pidFile)).toBeUndefined();
193 });
194
195 /** A cell stand-in that runs `command` under sh, with the supervision given. */
196 const shCell = (supervised: Supervised | undefined) => ({
197 argv: (command: string) => ["sh", "-c", command],
198 env: (extra: Record<string, string>) => ({ PATH: process.env["PATH"] ?? "", ...extra }),
199 ...(supervised === undefined ? {} : { supervised }),
200 });
201
202 test("a supervised cell writes to its supervisor's file, is adopted, and is followed there (fake supervisor)", async () => {
203 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
204 const path = join(dir, "cell.log");
205 writeFileSync(path, "");
206 const adopted: number[] = [];
207 let unfollowed = false;
208 const fake: Supervised = {
209 ctl: dir,
210 pid: process.pid,
211 logFd: openSync(path, "a"),
212 adopt: (pid) => void adopted.push(pid),
213 follow: (line) => {
214 const stop = tailLines(path, line, 20);
215 return () => ((unfollowed = true), stop());
216 },
217 close: () => {},
218 };
219 const heard: string[] = [];
220 const { proc, unfollow } = spawnInCell(shCell(fake), "echo out; echo err >&2", { cwd: dir, env: {} }, (l) => void heard.push(l));
221 await proc.exited;
222 expect(adopted).toEqual([proc.pid]);
223 // The output went to the file, not to a pipe this process holds.
224 expect(proc.stdout).toBe(fake.logFd);
225 expect(readFileSync(path, "utf8")).toBe("out\nerr\n");
226 expect(await until(() => heard.length === 2)).toBe(true);
227 unfollow();
228 expect(unfollowed).toBe(true);
229 expect(heard).toEqual(["out", "err"]);
230 });
231
232 test("an unsupervised cell is read from pipes, and nothing is adopted", async () => {
233 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
234 const heard: string[] = [];
235 const { proc, unfollow } = spawnInCell(shCell(undefined), "echo piped", { cwd: dir, env: {} }, (l) => void heard.push(l));
236 await proc.exited;
237 expect(await until(() => heard.includes("piped"))).toBe(true);
238 unfollow();
239 });
240
241 test("an outliving cell's real supervisor adopts the spawned root and its output reaches the host's log", async () => {
242 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
243 const cell = openCell({ ...spec(dir), outlive: true });
244 const sup = cell.supervised;
245 expect(sup).toBeDefined();
246 if (sup === undefined) return;
247 killed.push(sup.pid);
248 try {
249 const heard: string[] = [];
250 const { proc, unfollow } = spawnInCell(shCell(sup), "echo from-the-cell; sleep 30", { cwd: dir, env: {} }, (l) => void heard.push(l));
251 killed.push(proc.pid);
252 expect(await until(() => heard.includes("from-the-cell"))).toBe(true);
253 expect(await until(() => existsSync(files(sup.ctl).cell) && readFileSync(files(sup.ctl).cell, "utf8").startsWith(`${proc.pid}`))).toBe(true);
254 unfollow();
255 proc.kill("SIGKILL");
256 await proc.exited;
257 } finally {
258 cell.close();
259 }
260 });
261
262 test("the host and a supervisor serve one policy: the supervisor's config is the cell's policy", () => {
263 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
264 const s = { ...spec(dir), egress: ["api.example.com"], granted: ["10.0.0.5:5432"], egressMode: "report" as const, reach: ["http://127.0.0.1:4100", "https://f.test"], outlive: true };
265 const cell = openCell(s);
266 try {
267 const ctl = cell.supervised?.ctl ?? "";
268 const config = parseConfig(readFileSync(files(ctl).config, "utf8"));
269 expect(config).toBeDefined();
270 const { sockets: _sockets, outPort: _outPort, ...served } = config ?? { sockets: "" };
271 expect(served).toEqual(cellPolicy(s));
272 expect(cellPolicy(s).loops).toEqual([4100]);
273 } finally {
274 cell.close();
275 }
276 // The one mapping to proxy options: report is a hook only when the policy reports.
277 const told: string[] = [];
278 const opts = proxyOptions(cellPolicy(s), { undeclared: (h, p) => void told.push(`${h}:${p}`) });
279 opts.report?.("x.test", 443);
280 expect(told).toEqual(["x.test:443"]);
281 expect(opts.hosts).toEqual(["api.example.com"]);
282 expect(opts.granted).toEqual(["10.0.0.5:5432"]);
283 expect(proxyOptions({ ...cellPolicy(s), report: false }, {}).report).toBeUndefined();
284 });
285
286 const stopsWith = async (outlive: boolean): Promise<boolean> => {
287 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
288 const app = createApp({ dataDir: dir, port: 0 });
289 const before = process.env["FORGE_CELLS_OUTLIVE"];
290 try {
291 const registry = cellChildren(app, async (input) => {
292 const proc = Bun.spawn(["sleep", "30"], { stdout: "ignore", stderr: "ignore" });
293 killed.push(proc.pid);
294 return { ...input, port: 1, limits: cellLimits(input.size), pid: proc.pid, exited: proc.exited };
295 });
296 const child = await registry.start({ bodyId: "cell-kept", name: "kept", size: "small" });
297 expect(child).toBeDefined();
298 if (outlive) process.env["FORGE_CELLS_OUTLIVE"] = "on";
299 else delete process.env["FORGE_CELLS_OUTLIVE"];
300 await stopCellChildren(app);
301 return !alive(child?.pid ?? 0) || (await Promise.race([child?.exited.then(() => true), Bun.sleep(200).then(() => false)])) === true; // real-timer: a real child either exited or is still up
302 } finally {
303 if (before === undefined) delete process.env["FORGE_CELLS_OUTLIVE"];
304 else process.env["FORGE_CELLS_OUTLIVE"] = before;
305 app.close();
306 }
307 };
308
309 test("stopping the host leaves an outliving cell running, and stops a default one", async () => {
310 expect(await stopsWith(true)).toBe(false);
311 expect(await stopsWith(false)).toBe(true);
312 });
test/job-slices.test.ts +2
@@ -292,6 +292,8 @@ const UNTIMED: Record<string, string> = {
292292 "roll-probe": "cluster only (FORGE_RELEASE_KEEPER=on): one HTTP probe",
293293 "apps-sweep": "AppsRuntime's sweep over running app processes; the fixture has no apps",
294294 "profiler-drain": "drains the sampling profiler's stack traces on a timer (perf/sample.ts) so they do not pile up; runs only where the profiler is on, and the fixture samples nothing",
295 "cell output tail": "runs in a cell's supervisor process, not the host: reads a bounded tail of the child's output file and trims it; needs a supervised cell",
296 "cell allowed sync": "runs in a cell's supervisor process, not the host: one small file write of the allowed hosts; needs a supervised cell",
295297 "cell-watch": "watches running app cells' cgroups; needs cells, skipped in this sandbox mode",
296298 "cell-probe-deadline": "a cell's start probe running out of time: needs cells, skipped in this sandbox mode",
297299 "cell-kill-root": "kills a cell's root process after its grace: needs cells, skipped in this sandbox mode",
test/spawn-sites.test.ts +2
@@ -93,6 +93,8 @@ const SITES: Record<string, string> = {
9393 "bun (process.execPath) on the forge's own child_db_reader.ts, a fixed argv with an empty environment; the directory asked about and what a request presented travel over the IPC channel, never argv",
9494 "src/spawn/cell.ts":
9595 "a child forge launched from a launch spec the parent built, wrapped by its cell",
96 "src/spawn/supervisor.ts":
97 "setsid with the forge's own bun and its own supervisor.ts, one control directory the host made as argv; nothing from a request",
9698 "src/tui/screens/external.ts":
9799 "the operator's $EDITOR resolved with Bun.which, refused when it holds whitespace, and a file path the TUI made, as argv",
98100 "src/tui/term/terminal.ts":
test/timers-named.test.ts +3
@@ -64,6 +64,9 @@ const allow = (call: Call, why: string, at: readonly (readonly [string, string])
6464 * does work on the loop: a span around each would only fill the ring with nothing.
6565 */
6666 const ALLOWED: readonly Allowed[] = [
67 ...allow("setInterval", "the cell supervisor's own tick: a separate process with no job ring, its one timer", [
outdated

read (tailLines) and writeAllowed run in the host process, not the supervisor process, so 'no job ring' is false for them. Name those jobs; allow only the supervisor's own tick.

68 ["src/spawn/supervisor.ts", "() => { capFile(f.log"],
69 ]),
6770 ...allow("setInterval", `SSE heartbeat, ${PER_CONNECTION}`, [["src/http/sse.ts", "() => write("]]),
6871 ...allow("setInterval", `a talk stream's poll, ${PER_CONNECTION}`, [["src/web/pages/talk.ts", "tick"]]),
6972 ...allow("setInterval", `a shell session's idle check, ${PER_CONNECTION}`, [["src/machines/shell.ts", "() => { if (idle(session"]]),

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs