Public: anyone can read this; the crew can change it.
Step 1 of a cell outliving its host: a child cell's egress proxy, loopback relays and inbound relay run in a supervisor of their own (FORGE_CELLS_OUTLIVE, off) #13761
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13755-a-cell-child-s-egress-proxy-inbound-rela → main 8 commits fork 1 thread · 1 unresolved
10 files
changed +893
−56
src/apps/cell_policy.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | /** | |
| 2 | * A cell's way out and in, as plain data: what its egress proxy lets through and which relays it | |
| 3 | * gets. One value is built per cell (`openCell`), and both the host's proxy and a supervisor's | |
| 4 | * (`spawn/supervisor.ts`, #13755) turn it into proxy options here, so the two cannot drift. | |
| 5 | */ | |
| 6 | import { cellProxy, type EgressPolicy } from "./egress.ts"; | |
| 7 | ||
| 8 | export interface CellPolicy { | |
| 9 | readonly hosts: readonly string[]; | |
| 10 | readonly granted?: readonly string[]; | |
| 11 | readonly origins: readonly { host: string; port: number }[]; | |
| 12 | readonly label?: string; | |
| 13 | /** Let undeclared public hosts through, and say so, instead of refusing them. */ | |
| 14 | readonly report: boolean; | |
| 15 | /** Loopback ports of the forge the cell reaches. */ | |
| 16 | readonly loops: readonly number[]; | |
| 17 | /** The port the forge reaches an app by; relayed into the cell. */ | |
| 18 | readonly listen?: number; | |
| 19 | } | |
| 20 | ||
| 21 | /** Where what the proxy decides is told: whoever runs the proxy supplies these. */ | |
| 22 | export interface PolicyHooks { | |
| 23 | readonly allowed?: (() => readonly string[]) | undefined; | |
| 24 | readonly refused?: ((host: string, port: number, isPrivate: boolean) => void) | undefined; | |
| 25 | readonly undeclared?: ((host: string, port: number) => void) | undefined; | |
| 26 | } | |
| 27 | ||
| 28 | /** The egress proxy's options for `policy`. */ | |
| 29 | export function proxyOptions(policy: CellPolicy, hooks: PolicyHooks): EgressPolicy { | |
| 30 | return { | |
| 31 | hosts: policy.hosts, | |
| 32 | granted: policy.granted, | |
| 33 | label: policy.label, | |
| 34 | origins: policy.origins, | |
| 35 | allowed: hooks.allowed, | |
| 36 | refused: hooks.refused, | |
| 37 | // A cell a child forge opens (its check, app or step) has no way out but the proxy of the cell | |
| 38 | // the child runs in. A child in a cell is the forge told its parent's door; a check that boots | |
| 39 | // a forge is not. | |
| 40 | upstream: cellProxy(), | |
| 41 | // Enforce unless the caller carries an admin's report (`egressSetting`, #10778). | |
| 42 | report: policy.report ? (host, port) => hooks.undeclared?.(host, port) : undefined, | |
| 43 | }; | |
| 44 | } |
src/apps/contain.ts +56 −40
| @@ -37,7 +37,9 @@ import type { Database } from "bun:sqlite"; | ||
| 37 | 37 | import { availableParallelism, tmpdir } from "node:os"; |
| 38 | 38 | import { dirname, join } from "node:path"; |
| 39 | 39 | import { portInsteadOfSocket } from "../git/hook-address.ts"; |
| 40 | import { cellProxy, relay, startEgress, type Egress } from "./egress.ts"; | |
| 40 | import { relay, startEgress, type Egress } from "./egress.ts"; | |
| 41 | import { proxyOptions, type CellPolicy } from "./cell_policy.ts"; | |
| 42 | import { startSupervisor, type Supervised } from "../spawn/supervisor.ts"; | |
| 41 | 43 | import { BACKGROUND_CPU_WEIGHT } from "./background.ts"; |
| 42 | 44 | import { cellOom, oomFirst } from "./oom.ts"; |
| 43 | 45 | import { |
| @@ -386,6 +388,12 @@ export interface CellSpec { | ||
| 386 | 388 | * forge's comes from its size (`spawn/sizes.ts`); unset, no cap. |
| 387 | 389 | */ |
| 388 | 390 | readonly io?: IoMax | undefined; |
| 391 | /** | |
| 392 | * The cell outlives the host process (#13755): its proxy, relays and way in run in a supervisor | |
| 393 | * of their own, the cell is not tied to its parent, and its output goes to a file. Off unless | |
| 394 | * asked. | |
| 395 | */ | |
| 396 | readonly outlive?: boolean | undefined; | |
| 389 | 397 | } |
| 390 | 398 | |
| 391 | 399 | /** |
| @@ -417,6 +425,8 @@ export interface Cell { | ||
| 417 | 425 | * does. |
| 418 | 426 | */ |
| 419 | 427 | readonly cgroupDir?: string; |
| 428 | /** Present when the cell outlives the host: its supervisor, and the file its output goes to. */ | |
| 429 | readonly supervised?: Supervised; | |
| 420 | 430 | } |
| 421 | 431 | |
| 422 | 432 | // ---- can this host hold a cell? ---- |
| @@ -497,7 +507,7 @@ async function tryOnce(extra: readonly string[] = []): Promise<Verdict> { | ||
| 497 | 507 | const proc = Bun.spawn( |
| 498 | 508 | [ |
| 499 | 509 | "bwrap", |
| 500 | ...NAMESPACES, | |
| 510 | ...namespaces(false), | |
| 501 | 511 | ...extra, |
| 502 | 512 | "--ro-bind", |
| 503 | 513 | "/", |
| @@ -544,8 +554,9 @@ async function tryOnce(extra: readonly string[] = []): Promise<Verdict> { | ||
| 544 | 554 | } |
| 545 | 555 | } |
| 546 | 556 | |
| 547 | const NAMESPACES = [ | |
| 548 | "--die-with-parent", | |
| 557 | /** The namespaces a cell is made with; one that outlives the host is not tied to its parent. */ | |
| 558 | const namespaces = (outlive: boolean): string[] => [ | |
| 559 | ...(outlive ? [] : ["--die-with-parent"]), | |
| 549 | 560 | "--unshare-user", |
| 550 | 561 | "--unshare-pid", |
| 551 | 562 | "--unshare-net", |
| @@ -967,6 +978,20 @@ export function cellArgv( | ||
| 967 | 978 | return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned); |
| 968 | 979 | } |
| 969 | 980 | |
| 981 | /** The one egress policy a cell is served by, host or supervisor (`cell_policy.ts`). */ | |
| 982 | export function cellPolicy(spec: CellSpec): CellPolicy { | |
| 983 | const origins = spec.reach.map(origin).filter((o) => o !== undefined); | |
| 984 | return { | |
| 985 | hosts: spec.egress, | |
| 986 | ...(spec.granted === undefined ? {} : { granted: spec.granted }), | |
| 987 | origins, | |
| 988 | ...(spec.label === undefined ? {} : { label: spec.label }), | |
| 989 | report: spec.egressMode === "report", | |
| 990 | loops: [...new Set(origins.filter((o) => isLoopback(o.host)).map((o) => o.port))], | |
| 991 | ...(spec.listen === undefined ? {} : { listen: spec.listen }), | |
| 992 | }; | |
| 993 | } | |
| 994 | ||
| 970 | 995 | /** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */ |
| 971 | 996 | export function openCell( |
| 972 | 997 | spec: CellSpec, |
| @@ -974,39 +999,25 @@ export function openCell( | ||
| 974 | 999 | ): Cell { |
| 975 | 1000 | const sockets = mkdtempSync(join(tmpdir(), "fc-")); |
| 976 | 1001 | const opened: Egress[] = []; |
| 977 | const origins = spec.reach.map(origin).filter((o) => o !== undefined); | |
| 978 | const loops = origins.filter((o) => isLoopback(o.host)).map((o) => o.port); | |
| 979 | opened.push( | |
| 980 | startEgress( | |
| 981 | join(sockets, "out.sock"), | |
| 982 | { | |
| 983 | hosts: spec.egress, | |
| 984 | granted: spec.granted, | |
| 985 | label: spec.label, | |
| 986 | allowed: spec.allowed, | |
| 987 | refused: spec.onRefused, | |
| 988 | origins, | |
| 989 | // A cell a child forge opens (its check, app or step) has no way out | |
| 990 | // but the proxy of the cell the child runs in. A child in a cell is | |
| 991 | // the forge told its parent's door; a check that boots a forge is not. | |
| 992 | upstream: cellProxy(), | |
| 993 | report: | |
| 994 | // Enforce unless the caller carries an admin's report (`egressSetting`, #10778). | |
| 995 | spec.egressMode === "report" | |
| 996 | ? (host, port) => spec.onUndeclared?.(host, port) | |
| 997 | : undefined, | |
| 998 | }, | |
| 999 | log, | |
| 1000 | ), | |
| 1001 | ); | |
| 1002 | for (const port of new Set(loops)) | |
| 1003 | opened.push( | |
| 1004 | relay({ unix: join(sockets, `loop-${port}.sock`) }, { tcp: port }), | |
| 1005 | ); | |
| 1006 | if (spec.listen !== undefined) | |
| 1007 | opened.push( | |
| 1008 | relay({ tcp: spec.listen }, { unix: join(sockets, "in.sock") }), | |
| 1009 | ); | |
| 1002 | const policy = cellPolicy(spec); | |
| 1003 | const hooks = { allowed: spec.allowed, refused: spec.onRefused, undeclared: spec.onUndeclared }; | |
| 1004 | // A cell that outlives the host has these run by a supervisor of its own; the rest, by the host. | |
| 1005 | // Both serve the one `policy`. | |
| 1006 | const supervised = | |
| 1007 | spec.outlive === true | |
| 1008 | ? startSupervisor(mkdtempSync(join(tmpdir(), "fc-ctl-")), { sockets, ...policy }, { ...hooks, log }) | |
| 1009 | : undefined; | |
| 1010 | if (supervised === undefined) { | |
| 1011 | opened.push(startEgress(join(sockets, "out.sock"), proxyOptions(policy, hooks), log)); | |
| 1012 | for (const port of policy.loops) | |
| 1013 | opened.push( | |
| 1014 | relay({ unix: join(sockets, `loop-${port}.sock`) }, { tcp: port }), | |
| 1015 | ); | |
| 1016 | if (policy.listen !== undefined) | |
| 1017 | opened.push( | |
| 1018 | relay({ tcp: policy.listen }, { unix: join(sockets, "in.sock") }), | |
| 1019 | ); | |
| 1020 | } | |
| 1010 | 1021 | |
| 1011 | 1022 | const { hide, blank, reveal } = cellMounts(spec, hiddenMounts()); |
| 1012 | 1023 | const asked = [...new Set([spec.dir, spec.home, ...spec.writable])]; |
| @@ -1030,11 +1041,14 @@ export function openCell( | ||
| 1030 | 1041 | const cgroup = cellCgroup(spec); |
| 1031 | 1042 | if (spec.cpuQuota === true) noteCpuHolder(cgroup?.cpuCapped === true, log); |
| 1032 | 1043 | noteServingCores(servingCoresNow(), Bun.which("taskset") !== null, log); |
| 1033 | const argv = (command: string): string[] => cellArgv(bwrapArgv(command), spec, cgroup); | |
| 1044 | const argv = (command: string): string[] => { | |
| 1045 | const inner = cellArgv(bwrapArgv(command), spec, cgroup); | |
| 1046 | return supervised === undefined ? inner : ["setsid", ...inner]; | |
| 1047 | }; | |
| 1034 | 1048 | const rlimits = cellRlimits(spec.limits); |
| 1035 | 1049 | const bwrapArgv = (command: string): string[] => [ |
| 1036 | 1050 | "bwrap", |
| 1037 | ...NAMESPACES, | |
| 1051 | ...namespaces(spec.outlive === true), | |
| 1038 | 1052 | ...hostnameArgs(spec.hostname), |
| 1039 | 1053 | ...(spec.terminal === true ? [] : ["--new-session"]), |
| 1040 | 1054 | "--ro-bind", "/", "/", |
| @@ -1059,7 +1073,7 @@ export function openCell( | ||
| 1059 | 1073 | WRAPPER, |
| 1060 | 1074 | "cell", |
| 1061 | 1075 | spec.listen === undefined ? "-" : String(spec.listen), |
| 1062 | [...new Set(loops)].join(" "), | |
| 1076 | policy.loops.join(" "), | |
| 1063 | 1077 | String(rlimits.fileBlocks), |
| 1064 | 1078 | String(rlimits.files), |
| 1065 | 1079 | String(rlimits.tasks), |
| @@ -1117,6 +1131,7 @@ export function openCell( | ||
| 1117 | 1131 | env, |
| 1118 | 1132 | watch, |
| 1119 | 1133 | ...(cgroup === undefined ? {} : { cgroupDir: cgroup.dir }), |
| 1134 | ...(supervised === undefined ? {} : { supervised }), | |
| 1120 | 1135 | breach: () => breach ?? killed(), |
| 1121 | 1136 | close: () => { |
| 1122 | 1137 | const kernel = breach === undefined ? killed() : undefined; |
| @@ -1124,6 +1139,7 @@ export function openCell( | ||
| 1124 | 1139 | cgroup?.remove(); |
| 1125 | 1140 | unwatch?.(); |
| 1126 | 1141 | for (const o of opened) o.close(); |
| 1142 | supervised?.close(); | |
| 1127 | 1143 | rmSync(sockets, { recursive: true, force: true }); |
| 1128 | 1144 | }, |
| 1129 | 1145 | }; |
src/core/capacity.ts +22 −6
| @@ -225,7 +225,10 @@ function explicitDefault(key: CapacityKey): string | undefined { | ||
| 225 | 225 | |
| 226 | 226 | /** Most shards a check takes by default: past four, a test run waits on git, not CPU. */ |
| 227 | 227 | const SHARDS_DEFAULT_MAX = 4; |
| 228 | /** CPUs a shard is given by default: its bun test, the git children it starts, and the forge it boots. */ | |
| 228 | /** | |
| 229 | * CPUs a shard is given by default: its bun test, the git children it starts, and the forge it | |
| 230 | * boots. | |
| 231 | */ | |
| 229 | 232 | const CPUS_PER_SHARD = 3; |
| 230 | 233 | |
| 231 | 234 | /** Shards per check by default: `min(4, cpus)`, at least one. 80 CPUs give 4; 2 give 2. */ |
| @@ -501,7 +504,10 @@ export function followsSpeed(key: CapacityKey): key is SpeedDerivedKey { | ||
| 501 | 504 | return key === "pace.floor" || key === "queue.depth"; |
| 502 | 505 | } |
| 503 | 506 | |
| 504 | /** What Speed implies for a dial that follows it: half of Speed, at least 1 for the floor; half rounded up, 1 to 4, for the depth. */ | |
| 507 | /** | |
| 508 | * What Speed implies for a dial that follows it: half of Speed, at least 1 for the floor; half | |
| 509 | * rounded up, 1 to 4, for the depth. | |
| 510 | */ | |
| 505 | 511 | export function derivedFromSpeed(db: Database, key: SpeedDerivedKey): number { |
| 506 | 512 | const speed = Number(capacityValue(db, "crew.concurrency")); |
| 507 | 513 | if (key === "pace.floor") return Math.max(1, Math.floor(speed / 2)); |
| @@ -562,12 +568,17 @@ const MODEL_HOSTS: Readonly<Record<string, readonly string[]>> = { | ||
| 562 | 568 | both: ["openrouter.ai:443", "api.anthropic.com:443"], |
| 563 | 569 | }; |
| 564 | 570 | |
| 565 | /** Whether a child forge must have a filesystem of its own to start (`children.ownFilesystem`); off by default. */ | |
| 571 | /** | |
| 572 | * Whether a child forge must have a filesystem of its own to start (`children.ownFilesystem`); off | |
| 573 | * by default. | |
| 574 | */ | |
| 566 | 575 | export function childNeedsOwnFilesystem(db: Database): boolean { |
| 567 | 576 | return capacityValue(db, "children.ownFilesystem") === "on"; |
| 568 | 577 | } |
| 569 | 578 | |
| 570 | /** The package registry a child forge cell may reach (`children.registryEgress`); none by default. */ | |
| 579 | /** | |
| 580 | * The package registry a child forge cell may reach (`children.registryEgress`); none by default. | |
| 581 | */ | |
| 571 | 582 | export function childRegistryHosts(db: Database): readonly string[] { |
| 572 | 583 | const value = capacityValue(db, "children.registryEgress"); |
| 573 | 584 | return value === "build" ? BUILD_HOSTS : value === "npm" ? ["registry.npmjs.org:443"] : []; |
| @@ -599,7 +610,10 @@ export function childOriginHosts(db: Database, parentUrl: string): readonly stri | ||
| 599 | 610 | |
| 600 | 611 | const GIT_HOSTS: readonly string[] = ["github.com", "api.github.com", "codeload.github.com", "gitlab.com", "codeberg.org", "bitbucket.org"]; |
| 601 | 612 | |
| 602 | /** The payment provider hosts a child forge cell may reach (`children.paymentEgress`); the cell's ceiling, each app still declares its own. */ | |
| 613 | /** | |
| 614 | * The payment provider hosts a child forge cell may reach (`children.paymentEgress`); the cell's | |
| 615 | * ceiling, each app still declares its own. | |
| 616 | */ | |
| 603 | 617 | export function childPaymentHosts(db: Database): readonly string[] { |
| 604 | 618 | return capacityValue(db, "children.paymentEgress") === "stripe" ? PAYMENT_HOSTS : []; |
| 605 | 619 | } |
| @@ -670,7 +684,9 @@ export function setCapacity( | ||
| 670 | 684 | key: CapacityKey, |
| 671 | 685 | value: string | null, |
| 672 | 686 | actorId: string, |
| 673 | /** Why, in the admin's words, already cut to REASON_MAX by the action; kept beside who (#10778). */ | |
| 687 | /** | |
| 688 | * Why, in the admin's words, already cut to REASON_MAX by the action; kept beside who (#10778). | |
| 689 | */ | |
| 674 | 690 | reason = "", |
| 675 | 691 | ): { from: string; to: string } { |
| 676 | 692 | const from = capacityValue(db, key); |
src/kernel/destroy_reach.ts +2
| @@ -82,6 +82,8 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [ | ||
| 82 | 82 | { unit: "src/git/repos.ts:detachFork", why: "removes the alternates file of the fork just made" }, |
| 83 | 83 | { unit: "src/git/env.ts:trustedCaBundle", why: "removes the process's own temporary CA bundle at exit" }, |
| 84 | 84 | { unit: "src/apps/contain.ts:openCell", why: "clears the cell's socket directory before the cell opens" }, |
| 85 | { unit: "src/spawn/supervisor.ts:startSupervisor", why: "removes the cell's own control directory (a fresh temp directory it made) when the host closes the cell" }, | |
| 86 | { unit: "src/spawn/supervisor.ts:runSupervisor", why: "removes the cell's own socket directory and, when no host is left, its control directory: both fresh temp directories made for that cell" }, | |
| 85 | 87 | { unit: "src/apps/cgroup.ts:removeDir", why: "removes an empty cgroup directory" }, |
| 86 | 88 | { unit: "src/actions/spawn.ts:unmake", why: "removes the directory and ledger of a child whose first attempt failed before it was ever finished, and only that call's own (never a finished or resumed one): nothing of anyone's work is in it (#13135)" }, |
| 87 | 89 | { unit: "src/spawn/own_code.ts:checkout", why: "removes a half-made export of the forge's own code, keyed by commit" }, |
src/spawn/cell.ts +54 −10
| @@ -28,6 +28,15 @@ import { liveChildDisk, type ChildDisk } from "./disk_mount.ts"; | ||
| 28 | 28 | import { oneAtATime, sourcesFor, startFailure, startFirstThatWorks, useTree, type CodeSource } from "./own_code.ts"; |
| 29 | 29 | import { childGate, type ChildGate } from "./child_reach.ts"; |
| 30 | 30 | |
| 31 | /** | |
| 32 | * Whether a child's cell outlives this process (`FORGE_CELLS_OUTLIVE=on`; #13755): its proxy and | |
| 33 | * relays run in a supervisor of their own, the cell is not tied to its parent, and a stop of this | |
| 34 | * process leaves it running. Off by default and undocumented: nothing yet re-adopts or reaps a cell | |
| 35 | * on start (#13750 step 2), its beat door (`beat-door.ts`) still runs in this process, and its | |
| 36 | * egress bytes are counted in the supervisor, out of this process's gauges (#13921). | |
| 37 | */ | |
| 38 | export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on"; | |
| 39 | ||
| 31 | 40 | export const CELL_PREFIX = "cell-"; |
| 32 | 41 | export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX); |
| 33 | 42 | |
| @@ -179,6 +188,7 @@ export function cellSpecFor(input: { | ||
| 179 | 188 | // cannot starve the forge's database of the disk (#10854). |
| 180 | 189 | io: SIZE_IO[input.size], |
| 181 | 190 | listen: input.port, |
| 191 | ...(cellsOutlive() ? { outlive: true } : {}), | |
| 182 | 192 | label: input.label, |
| 183 | 193 | // Its shell's prompt names the child, never the parent's pod (#10850). |
| 184 | 194 | hostname: input.name, |
| @@ -272,12 +282,45 @@ export const SLEEP_AT_ONCE = 8; | ||
| 272 | 282 | |
| 273 | 283 | const registries = new WeakMap<App, CellChildren>(); |
| 274 | 284 | |
| 275 | export function cellChildren(parent: App): CellChildren { | |
| 285 | /** | |
| 286 | * The parent's one registry of child cells. `launcher` stands in for the cell launch the first time | |
| 287 | * the registry is made (tests); later calls ignore it. | |
| 288 | */ | |
| 289 | export function cellChildren(parent: App, launcher?: Launcher): CellChildren { | |
| 276 | 290 | let r = registries.get(parent); |
| 277 | if (r === undefined) registries.set(parent, (r = makeRegistry(parent))); | |
| 291 | if (r === undefined) registries.set(parent, (r = makeRegistry(parent, launcher))); | |
| 278 | 292 | return r; |
| 279 | 293 | } |
| 280 | 294 | |
| 295 | /** | |
| 296 | * Run `command` in `cell` and wire its output to `log`. A supervised cell (one that outlives this | |
| 297 | * process, #13755) writes to its supervisor's file, is adopted as the supervisor's root, and is | |
| 298 | * followed there; any other is read from pipes this process holds. `unfollow` stops the following. | |
| 299 | */ | |
| 300 | export function spawnInCell( | |
| 301 | cell: Pick<Cell, "argv" | "env" | "supervised">, | |
| 302 | command: string, | |
| 303 | at: { cwd: string; env: Record<string, string> }, | |
| 304 | log: (line: string) => void, | |
| 305 | ): { proc: ReturnType<typeof Bun.spawn>; unfollow: () => void } { | |
| 306 | const sup = cell.supervised; | |
| 307 | const proc = Bun.spawn(cell.argv(command), { | |
| 308 | cwd: at.cwd, | |
| 309 | env: cell.env(at.env), | |
| 310 | stdin: "ignore", | |
| 311 | // A cell that outlives this process writes to a file, not a pipe this process reads. | |
| 312 | stdout: sup?.logFd ?? "pipe", | |
| 313 | stderr: sup?.logFd ?? "pipe", | |
| 314 | }); | |
| 315 | if (sup === undefined) { | |
| 316 | void drain(proc.stdout, log); | |
| 317 | void drain(proc.stderr, log); | |
| 318 | return { proc, unfollow: () => {} }; | |
| 319 | } | |
| 320 | sup.adopt(proc.pid); | |
| 321 | return { proc, unfollow: sup.follow(log) }; | |
| 322 | } | |
| 323 | ||
| 281 | 324 | /** |
| 282 | 325 | * Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not. |
| 283 | 326 | */ |
| @@ -403,14 +446,12 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 403 | 446 | throw error; |
| 404 | 447 | } |
| 405 | 448 | const command = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(source.main)}`; |
| 406 | const proc = Bun.spawn(cell.argv(command), { | |
| 449 | const { proc, unfollow } = spawnInCell(cell, command, { | |
| 407 | 450 | cwd: dirname(source.main), |
| 408 | env: cell.env(childEnvFor({ port, dataDir: dir, url, parentPort, size: input.size, ...(cell.cgroupDir === undefined ? {} : { cgroupDir: cell.cgroupDir }) })), | |
| 409 | stdin: "ignore", | |
| 410 | stdout: "pipe", | |
| 411 | stderr: "pipe", | |
| 412 | }); | |
| 451 | env: childEnvFor({ port, dataDir: dir, url, parentPort, size: input.size, ...(cell.cgroupDir === undefined ? {} : { cgroupDir: cell.cgroupDir }) }), | |
| 452 | }, log); | |
| 413 | 453 | const exited = proc.exited.then((code) => { |
| 454 | unfollow(); | |
| 414 | 455 | const why = cell.breach(); |
| 415 | 456 | if (why !== undefined) { |
| 416 | 457 | parent.log.warn("child forge stopped", { name: input.name, why }); |
| @@ -424,8 +465,6 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 424 | 465 | return code; |
| 425 | 466 | }); |
| 426 | 467 | cell.watch(proc.pid, () => killTree(proc.pid)); |
| 427 | void drain(proc.stdout, log); | |
| 428 | void drain(proc.stderr, log); | |
| 429 | 468 | const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc, door }; |
| 430 | 469 | let gone = false; |
| 431 | 470 | void exited.then(() => (gone = true)); |
| @@ -617,6 +656,11 @@ export async function stopCellChildren(parent: App): Promise<void> { | ||
| 617 | 656 | const r = registries.get(parent); |
| 618 | 657 | if (r === undefined) return; |
| 619 | 658 | registries.delete(parent); |
| 659 | // A cell that outlives this process is left running; its supervisor holds its way in and out. | |
| 660 | if (cellsOutlive()) { | |
| 661 | closeChildDb(); | |
| 662 | return; | |
| 663 | } | |
| 620 | 664 | await Promise.all(r.list().map((c) => r.stop(c.bodyId))); |
| 621 | 665 | // The workers that ask the children's databases go with them; one is started again when next |
| 622 | 666 | // asked. |
src/spawn/supervisor.ts added +396
| @@ -0,0 +1,396 @@ | ||
| 1 | /** | |
| 2 | * A cell's supervisor: the process that owns what the cell talks to (the egress proxy, the loopback | |
| 3 | * relays, the way in) and the files the host reads it by, so the cell can outlive the host process | |
| 4 | * (#13755, step 1 of #13750). | |
| 5 | * | |
| 6 | * It runs in its own session (`setsid`), not as a child that dies with the host. The host talks to | |
| 7 | * it through plain files in one control directory, never to the cell: | |
| 8 | * | |
| 9 | * config.json what to serve (host writes, once) | |
| 10 | * host.pid the host that started it (host writes) | |
| 11 | * cell.pid the cell's root process (host writes once it is spawned) | |
| 12 | * cell.log the cell's stdout and stderr, as a file the cell holds open | |
| 13 | * allowed.json egress entries allowed since the cell started (host rewrites when they change) | |
| 14 | * events.jsonl refusals and undeclared hosts (supervisor appends; the host tails) | |
| 15 | * | |
| 16 | * It ends when the cell's root process is gone, on SIGTERM, or when no cell appears in a minute. | |
| 17 | * The host side of this file is `startSupervisor`; `bun supervisor.ts <ctl dir>` is the other. | |
| 18 | */ | |
| 19 | import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, truncateSync, writeFileSync } from "node:fs"; | |
| 20 | import { join } from "node:path"; | |
| 21 | import { during } from "../perf/jobs.ts"; | |
| 22 | import { PROXY_VARS, relay, startEgress, type Egress } from "../apps/egress.ts"; | |
| 23 | import { proxyOptions, type CellPolicy } from "../apps/cell_policy.ts"; | |
| 24 | ||
| 25 | /** Plain data: what a supervisor serves. */ | |
| 26 | export interface SupervisorConfig extends CellPolicy { | |
| 27 | /** The directory the cell sees at `/run/forge`; the sockets are made in it. */ | |
| 28 | readonly sockets: string; | |
| 29 | /** Tests: the proxy listens on this loopback port, where a unix socket cannot be bound. */ | |
| 30 | readonly outPort?: number; | |
| 31 | } | |
| 32 | ||
| 33 | export type SupervisorEvent = | |
| 34 | | { e: "refused"; host: string; port: number; private: boolean } | |
| 35 | | { e: "undeclared"; host: string; port: number } | |
| 36 | | { e: "log"; line: string }; | |
| 37 | ||
| 38 | const WAIT_FOR_CELL_MS = 60_000; | |
| 39 | const TICK_MS = 1_000; | |
| 40 | /** | |
| 41 | * The most the cell's output file may hold; past it the file starts over (the cell keeps writing). | |
| 42 | */ | |
| 43 | export const LOG_MAX_BYTES = 4_000_000; | |
| 44 | /** The most the events file may hold. */ | |
| 45 | export const EVENTS_MAX_BYTES = 1_000_000; | |
| 46 | /** Events the supervisor writes in a minute; past it they are dropped. */ | |
| 47 | export const EVENTS_PER_MINUTE = 300; | |
| 48 | ||
| 49 | /** | |
| 50 | * Start `path` over when it is past `max` bytes: a writer holding it open in append mode goes on. | |
| 51 | */ | |
| 52 | export function capFile(path: string, max: number): boolean { | |
| 53 | try { | |
| 54 | if (statSync(path).size <= max) return false; | |
| 55 | truncateSync(path, 0); | |
| 56 | return true; | |
| 57 | } catch { | |
| 58 | return false; | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | /** A counter that says yes to `limit` calls in each minute, and no to the rest. */ | |
| 63 | export function perMinute(limit: number, now: () => number = Date.now): () => boolean { | |
| 64 | let start = now(); | |
| 65 | let n = 0; | |
| 66 | return () => { | |
| 67 | const t = now(); | |
| 68 | if (t - start >= 60_000) { | |
| 69 | start = t; | |
| 70 | n = 0; | |
| 71 | } | |
| 72 | return ++n <= limit; | |
| 73 | }; | |
| 74 | } | |
| 75 | ||
| 76 | export const files = (ctl: string) => ({ | |
| 77 | config: join(ctl, "config.json"), | |
| 78 | host: join(ctl, "host.pid"), | |
| 79 | cell: join(ctl, "cell.pid"), | |
| 80 | log: join(ctl, "cell.log"), | |
| 81 | allowed: join(ctl, "allowed.json"), | |
| 82 | events: join(ctl, "events.jsonl"), | |
| 83 | }); | |
| 84 | ||
| 85 | export function alive(pid: number): boolean { | |
| 86 | if (!Number.isInteger(pid) || pid <= 0) return false; | |
| 87 | try { | |
| 88 | process.kill(pid, 0); | |
| 89 | return true; | |
| 90 | } catch (error) { | |
| 91 | return (error as NodeJS.ErrnoException).code === "EPERM"; | |
| 92 | } | |
| 93 | } | |
| 94 | ||
| 95 | /** | |
| 96 | * When a process started, in clock ticks since boot (`/proc/<pid>/stat`); undefined where unknown. | |
| 97 | */ | |
| 98 | export function startTime(pid: number): string | undefined { | |
| 99 | try { | |
| 100 | const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); | |
| 101 | return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19]; | |
| 102 | } catch { | |
| 103 | return undefined; | |
| 104 | } | |
| 105 | } | |
| 106 | ||
| 107 | /** A pid file holds `pid`, and `pid:start` when known: a pid whose start differs was reused. */ | |
| 108 | export const pidRecord = (pid: number): string => { | |
| 109 | const start = startTime(pid); | |
| 110 | return start === undefined ? String(pid) : `${pid}:${start}`; | |
| 111 | }; | |
| 112 | ||
| 113 | /** The pid in a pid file that is still the same process; undefined when it is gone or reused. */ | |
| 114 | export function readPid(path: string): number | undefined { | |
| 115 | try { | |
| 116 | const [a, start] = readFileSync(path, "utf8").trim().split(":"); | |
| 117 | const n = Number(a); | |
| 118 | if (!Number.isInteger(n) || n <= 0) return undefined; | |
| 119 | if (start !== undefined && startTime(n) !== undefined && startTime(n) !== start) return undefined; | |
| 120 | return n; | |
| 121 | } catch { | |
| 122 | return undefined; | |
| 123 | } | |
| 124 | } | |
| 125 | ||
| 126 | /** The entries of `allowed.json`, only ever strings; what is not is ignored. */ | |
| 127 | export function readAllowed(path: string): string[] { | |
| 128 | try { | |
| 129 | const value: unknown = JSON.parse(readFileSync(path, "utf8")); | |
| 130 | return Array.isArray(value) ? value.filter((v): v is string => typeof v === "string") : []; | |
| 131 | } catch { | |
| 132 | return []; | |
| 133 | } | |
| 134 | } | |
| 135 | ||
| 136 | const isPort = (v: unknown): v is number => Number.isInteger(v) && (v as number) > 0 && (v as number) < 65_536; | |
| 137 | const strings = (v: unknown): v is string[] => Array.isArray(v) && v.every((x) => typeof x === "string"); | |
| 138 | ||
| 139 | /** A `config.json` checked field by field; undefined when it is not one. */ | |
| 140 | export function parseConfig(text: string): SupervisorConfig | undefined { | |
| 141 | try { | |
| 142 | const v = JSON.parse(text) as Record<string, unknown>; | |
| 143 | if (typeof v["sockets"] !== "string" || v["sockets"] === "" || !strings(v["hosts"])) return undefined; | |
| 144 | if (v["granted"] !== undefined && !strings(v["granted"])) return undefined; | |
| 145 | if (v["label"] !== undefined && typeof v["label"] !== "string") return undefined; | |
| 146 | if (typeof v["report"] !== "boolean") return undefined; | |
| 147 | if (!Array.isArray(v["loops"]) || !v["loops"].every(isPort)) return undefined; | |
| 148 | if (v["listen"] !== undefined && !isPort(v["listen"])) return undefined; | |
| 149 | if (v["outPort"] !== undefined && !isPort(v["outPort"])) return undefined; | |
| 150 | const origins = v["origins"]; | |
| 151 | if (!Array.isArray(origins)) return undefined; | |
| 152 | for (const o of origins as Record<string, unknown>[]) if (typeof o !== "object" || o === null || typeof o["host"] !== "string" || !isPort(o["port"])) return undefined; | |
| 153 | return v as unknown as SupervisorConfig; | |
| 154 | } catch { | |
| 155 | return undefined; | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | /** Parse one line of `events.jsonl`; undefined for anything that is not an event. */ | |
| 160 | export function parseEvent(line: string): SupervisorEvent | undefined { | |
| 161 | try { | |
| 162 | const v = JSON.parse(line) as Record<string, unknown>; | |
| 163 | if (v["e"] === "log" && typeof v["line"] === "string") return { e: "log", line: v["line"].slice(0, 500) }; | |
| 164 | if (typeof v["host"] !== "string" || typeof v["port"] !== "number") return undefined; | |
| 165 | if (v["e"] === "undeclared") return { e: "undeclared", host: v["host"], port: v["port"] }; | |
| 166 | if (v["e"] === "refused") return { e: "refused", host: v["host"], port: v["port"], private: v["private"] === true }; | |
| 167 | } catch { | |
| 168 | // not an event | |
| 169 | } | |
| 170 | return undefined; | |
| 171 | } | |
| 172 | ||
| 173 | /** | |
| 174 | * Serve `config` until the cell is gone or `stop` is called. Resolves once everything is closed. | |
| 175 | */ | |
| 176 | export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT_FOR_CELL_MS): { done: Promise<void>; stop: () => void } { | |
| 177 | const f = files(ctl); | |
| 178 | const eventAllowed = perMinute(EVENTS_PER_MINUTE); | |
| 179 | const say = (event: SupervisorEvent): void => { | |
| 180 | if (!eventAllowed()) return; | |
| 181 | try { | |
| 182 | appendFileSync(f.events, `${JSON.stringify(event)}\n`); | |
| 183 | } catch { | |
| 184 | // the control directory is gone: the host closed it | |
| 185 | } | |
| 186 | }; | |
| 187 | let cache: { at: number; list: string[] } = { at: -1, list: [] }; | |
| 188 | const allowed = (): readonly string[] => { | |
| 189 | try { | |
| 190 | const at = statSync(f.allowed).mtimeMs; | |
| 191 | if (at !== cache.at) cache = { at, list: readAllowed(f.allowed) }; | |
| 192 | } catch { | |
| 193 | // none yet | |
| 194 | } | |
| 195 | return cache.list; | |
| 196 | }; | |
| 197 | const opened: Egress[] = []; | |
| 198 | // TODO(#13921): `recordEgress` counts this proxy's bytes in this process, so the host's egress | |
| 199 | // gauges miss an outliving cell's traffic until they are summed here and sent back as events. | |
| 200 | const policy = proxyOptions(config, { | |
| 201 | allowed, | |
| 202 | refused: (host, port, isPrivate) => say({ e: "refused", host, port, private: isPrivate }), | |
| 203 | undeclared: (host, port) => say({ e: "undeclared", host, port }), | |
| 204 | }); | |
| 205 | const out = config.outPort === undefined ? join(config.sockets, "out.sock") : { tcp: config.outPort }; | |
| 206 | opened.push(startEgress(out, policy, (line) => say({ e: "log", line }))); | |
| 207 | for (const port of new Set(config.loops)) opened.push(relay({ unix: join(config.sockets, `loop-${port}.sock`) }, { tcp: port })); | |
| 208 | if (config.listen !== undefined) opened.push(relay({ tcp: config.listen }, { unix: join(config.sockets, "in.sock") })); | |
| 209 | ||
| 210 | const began = Date.now(); | |
| 211 | let finish: () => void = () => {}; | |
| 212 | const done = new Promise<void>((resolve) => (finish = resolve)); | |
| 213 | let closed = false; | |
| 214 | const stop = (): void => { | |
| 215 | if (closed) return; | |
| 216 | closed = true; | |
| 217 | clearInterval(timer); | |
| 218 | for (const o of opened) o.close(); | |
| 219 | rmSync(config.sockets, { recursive: true, force: true }); | |
| 220 | // The host reads the last of the log and events when it is there; with no host, nobody will. | |
| 221 | const host = readPid(f.host); | |
| 222 | if (host === undefined || !alive(host)) rmSync(ctl, { recursive: true, force: true }); | |
| 223 | finish(); | |
| 224 | }; | |
| 225 | // Its own process has no job ring: this is the only timer it has. | |
| 226 | const timer = setInterval(() => { | |
| 227 | capFile(f.log, LOG_MAX_BYTES); | |
| 228 | capFile(f.events, EVENTS_MAX_BYTES); | |
| 229 | const cell = readPid(f.cell); | |
| 230 | if (cell !== undefined ? !alive(cell) : Date.now() - began > wait) stop(); | |
| 231 | }, TICK_MS); | |
| 232 | return { done, stop }; | |
| 233 | } | |
| 234 | ||
| 235 | // ---- the host's side ---- | |
| 236 | ||
| 237 | export interface SupervisorHooks { | |
| 238 | /** | |
| 239 | * Egress entries allowed since the cell started; rewritten to the supervisor when they change. | |
| 240 | */ | |
| 241 | readonly allowed?: (() => readonly string[]) | undefined; | |
| 242 | readonly onRefused?: ((host: string, port: number, isPrivate: boolean) => void) | undefined; | |
| 243 | readonly onUndeclared?: ((host: string, port: number) => void) | undefined; | |
| 244 | readonly log: (line: string) => void; | |
| 245 | } | |
| 246 | ||
| 247 | export interface Supervised { | |
| 248 | readonly ctl: string; | |
| 249 | readonly pid: number; | |
| 250 | /** An open file the cell writes its stdout and stderr to; hand it to the spawn, then `follow`. */ | |
| 251 | readonly logFd: number; | |
| 252 | /** Tell the supervisor which process is the cell's root: it ends when that is gone. */ | |
| 253 | readonly adopt: (pid: number) => void; | |
| 254 | /** Call `line` with each new line of the cell's output; returns what stops it. */ | |
| 255 | readonly follow: (line: (text: string) => void) => () => void; | |
| 256 | /** Stop the supervisor and remove the control directory. */ | |
| 257 | readonly close: () => void; | |
| 258 | } | |
| 259 | ||
| 260 | /** | |
| 261 | * Call `line` for each new line appended to `path`, polling; the returned function stops, after a | |
| 262 | * last read. | |
| 263 | */ | |
| 264 | export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES): () => void { | |
| 265 | let offset = 0; | |
| 266 | let rest = ""; | |
| 267 | const read = (): void => { | |
| 268 | let fd: number; | |
| 269 | try { | |
| 270 | fd = openSync(path, "r"); | |
| 271 | } catch { | |
| 272 | return; | |
| 273 | } | |
| 274 | try { | |
| 275 | const size = statSync(path).size; | |
| 276 | // The file was started over (`capFile`): read it again from its top. | |
| 277 | if (size < offset) offset = 0; | |
| 278 | while (offset < size) { | |
| 279 | const buf = Buffer.alloc(Math.min(65_536, size - offset)); | |
| 280 | const n = readSync(fd, buf, 0, buf.length, offset); | |
| 281 | if (n <= 0) break; | |
| 282 | offset += n; | |
| 283 | rest += buf.subarray(0, n).toString("utf8"); | |
| 284 | const cut = rest.split("\n"); | |
| 285 | rest = cut.pop() ?? ""; | |
| 286 | if (rest.length > 4_000) { | |
| 287 | cut.push(rest); | |
| 288 | rest = ""; | |
| 289 | } | |
| 290 | for (const l of cut) if (l !== "") line(l); | |
| 291 | } | |
| 292 | } finally { | |
| 293 | closeSync(fd); | |
| 294 | } | |
| 295 | }; | |
| 296 | // The host bounds the file too: the supervisor may be gone, and the cell not. | |
| 297 | const timer = setInterval(() => void during("cell output tail", () => (read(), capFile(path, max))), everyMs); | |
| 298 | timer.unref(); | |
| 299 | return () => { | |
| 300 | clearInterval(timer); | |
| 301 | read(); | |
| 302 | if (rest !== "") line(rest); | |
| 303 | rest = ""; | |
| 304 | }; | |
| 305 | } | |
| 306 | ||
| 307 | /** | |
| 308 | * The environment a supervisor needs: where to find tools, and the way out a child forge's cell | |
| 309 | * has. | |
| 310 | */ | |
| 311 | function supervisorEnv(): Record<string, string> { | |
| 312 | const env: Record<string, string> = {}; | |
| 313 | for (const key of ["PATH", "FORGE_PARENT_LOCAL", "FORGE_TEST_SOCKET", ...PROXY_VARS]) { | |
| 314 | const v = process.env[key]; | |
| 315 | if (v !== undefined) env[key] = v; | |
| 316 | } | |
| 317 | return env; | |
| 318 | } | |
| 319 | ||
| 320 | /** Start a supervisor in its own session, and the host's side of talking to it. */ | |
| 321 | export function startSupervisor(ctl: string, config: SupervisorConfig, hooks: SupervisorHooks): Supervised { | |
| 322 | const f = files(ctl); | |
| 323 | mkdirSync(ctl, { recursive: true, mode: 0o700 }); | |
| 324 | writeFileSync(f.config, JSON.stringify(config)); | |
| 325 | writeFileSync(f.host, pidRecord(process.pid)); | |
| 326 | writeFileSync(f.events, ""); | |
| 327 | const logFd = openSync(f.log, "a"); | |
| 328 | const writeAllowed = (() => { | |
| 329 | let last = ""; | |
| 330 | return (): void => { | |
| 331 | const now = JSON.stringify([...(hooks.allowed?.() ?? [])]); | |
| 332 | if (now === last) return; | |
| 333 | last = now; | |
| 334 | writeFileSync(f.allowed, now); | |
| 335 | }; | |
| 336 | })(); | |
| 337 | writeAllowed(); | |
| 338 | const proc = Bun.spawn(["setsid", process.execPath, import.meta.path, ctl], { | |
| 339 | env: supervisorEnv(), | |
| 340 | stdin: "ignore", | |
| 341 | stdout: "ignore", | |
| 342 | stderr: "ignore", | |
| 343 | }); | |
| 344 | proc.unref(); | |
| 345 | const sync = setInterval(() => void during("cell allowed sync", writeAllowed), TICK_MS); | |
| 346 | sync.unref(); | |
| 347 | const stopEvents = tailLines(f.events, (text) => { | |
| 348 | const event = parseEvent(text); | |
| 349 | if (event === undefined) return; | |
| 350 | if (event.e === "log") hooks.log(event.line); | |
| 351 | else if (event.e === "refused") hooks.onRefused?.(event.host, event.port, event.private); | |
| 352 | else hooks.onUndeclared?.(event.host, event.port); | |
| 353 | }); | |
| 354 | let closed = false; | |
| 355 | return { | |
| 356 | ctl, | |
| 357 | pid: proc.pid, | |
| 358 | logFd, | |
| 359 | adopt: (pid) => writeFileSync(f.cell, pidRecord(pid)), | |
| 360 | follow: (line) => tailLines(f.log, line), | |
| 361 | close: () => { | |
| 362 | if (closed) return; | |
| 363 | closed = true; | |
| 364 | clearInterval(sync); | |
| 365 | stopEvents(); | |
| 366 | try { | |
| 367 | process.kill(proc.pid, "SIGTERM"); | |
| 368 | } catch { | |
| 369 | // already gone | |
| 370 | } | |
| 371 | try { | |
| 372 | closeSync(logFd); | |
| 373 | } catch { | |
| 374 | // closed | |
| 375 | } | |
| 376 | rmSync(ctl, { recursive: true, force: true }); | |
| 377 | }, | |
| 378 | }; | |
| 379 | } | |
| 380 | ||
| 381 | if (import.meta.main) { | |
| 382 | const ctl = process.argv[2]; | |
| 383 | if (ctl === undefined || !existsSync(files(ctl).config)) { | |
| 384 | console.error("usage: supervisor.ts <control directory>"); | |
| 385 | process.exit(2); | |
| 386 | } | |
| 387 | const config = parseConfig(readFileSync(files(ctl).config, "utf8")); | |
| 388 | if (config === undefined) { | |
| 389 | console.error("supervisor: config.json is not a config"); | |
| 390 | process.exit(2); | |
| 391 | } | |
| 392 | const s = runSupervisor(ctl, config); | |
| 393 | for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, s.stop); | |
| 394 | await s.done; | |
| 395 | process.exit(0); | |
| 396 | } |
test/cell-outlives-host.test.ts added +312
| @@ -0,0 +1,312 @@ | ||
| 1 | /** | |
| 2 | * A cell's supervisor owns its proxy, relays and way in outside the host process (#13755): with the | |
| 3 | * host killed, the cell and its listener stay up, and the supervisor goes when the cell does. | |
| 4 | */ | |
| 5 | import { afterEach, expect, test } from "bun:test"; | |
| 6 | import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync, appendFileSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { cellPolicy, openCell } from "../src/apps/contain.ts"; | |
| 10 | import { proxyOptions } from "../src/apps/cell_policy.ts"; | |
| 11 | import { createApp } from "../src/app.ts"; | |
| 12 | import { freePort } from "../src/apps/process.ts"; | |
| 13 | import { cellChildren, cellLimits, cellSpecFor, cellsOutlive, spawnInCell, stopCellChildren } from "../src/spawn/cell.ts"; | |
| 14 | import { alive, files, parseConfig, parseEvent, readAllowed, tailLines, type Supervised } from "../src/spawn/supervisor.ts"; | |
| 15 | ||
| 16 | const killed: number[] = []; | |
| 17 | afterEach(() => { | |
| 18 | for (const pid of killed.splice(0)) | |
| 19 | try { | |
| 20 | process.kill(pid, "SIGKILL"); | |
| 21 | } catch { | |
| 22 | // gone | |
| 23 | } | |
| 24 | }); | |
| 25 | ||
| 26 | const until = async (what: () => boolean, ms = 8_000): Promise<boolean> => { | |
| 27 | const end = Date.now() + ms; | |
| 28 | while (Date.now() < end) { | |
| 29 | if (what()) return true; | |
| 30 | await Bun.sleep(50); // real-timer: polling a real child process | |
| 31 | } | |
| 32 | return what(); | |
| 33 | }; | |
| 34 | ||
| 35 | const accepts = (port: number): Promise<boolean> => | |
| 36 | new Promise((resolve) => { | |
| 37 | Bun.connect({ hostname: "127.0.0.1", port, socket: { open: (s) => (s.end(), resolve(true)), data() {}, error: () => resolve(false), connectError: () => resolve(false) } }).catch(() => resolve(false)); | |
| 38 | }); | |
| 39 | ||
| 40 | const pgid = (pid: number): string => Bun.spawnSync(["ps", "-o", "pgid=", "-p", String(pid)]).stdout.toString().trim(); | |
| 41 | ||
| 42 | test("events and the allowed list are read as plain data, and anything else is ignored", () => { | |
| 43 | expect(parseEvent('{"e":"refused","host":"a.com","port":443,"private":true}')).toEqual({ e: "refused", host: "a.com", port: 443, private: true }); | |
| 44 | expect(parseEvent('{"e":"undeclared","host":"a.com","port":80}')).toEqual({ e: "undeclared", host: "a.com", port: 80 }); | |
| 45 | expect(parseEvent('{"e":"log","line":"hi"}')).toEqual({ e: "log", line: "hi" }); | |
| 46 | expect(parseEvent('{"e":"refused","host":1}')).toBeUndefined(); | |
| 47 | expect(parseEvent("not json")).toBeUndefined(); | |
| 48 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 49 | const path = join(dir, "allowed.json"); | |
| 50 | writeFileSync(path, '["a.com:443", 7, "b.com"]'); | |
| 51 | expect(readAllowed(path)).toEqual(["a.com:443", "b.com"]); | |
| 52 | expect(readAllowed(join(dir, "missing"))).toEqual([]); | |
| 53 | }); | |
| 54 | ||
| 55 | test("a file is followed by line, and its last line is read when it stops", async () => { | |
| 56 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 57 | const path = join(dir, "cell.log"); | |
| 58 | writeFileSync(path, "one\ntw"); | |
| 59 | const heard: string[] = []; | |
| 60 | const stop = tailLines(path, (l) => heard.push(l), 20); | |
| 61 | await until(() => heard.length === 1); | |
| 62 | appendFileSync(path, "o\nthree"); | |
| 63 | await until(() => heard.length === 2); | |
| 64 | stop(); | |
| 65 | expect(heard).toEqual(["one", "two", "three"]); | |
| 66 | }); | |
| 67 | ||
| 68 | test("kill the host: the cell and its listener stay up, and the supervisor ends with the cell", async () => { | |
| 69 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 70 | const ctl = join(dir, "ctl"); | |
| 71 | const sockets = join(dir, "sockets"); | |
| 72 | const listen = freePort(); | |
| 73 | const outPort = freePort(); | |
| 74 | const script = ` | |
| 75 | import { startSupervisor } from ${JSON.stringify(join(import.meta.dir, "../src/spawn/supervisor.ts"))}; | |
| 76 | const s = startSupervisor(${JSON.stringify(ctl)}, { sockets: ${JSON.stringify(sockets)}, hosts: [], origins: [], report: false, loops: [], listen: ${listen}, outPort: ${outPort} }, { log: () => {} }); | |
| 77 | const cell = Bun.spawn(["sleep", "300"], { stdout: s.logFd, stderr: s.logFd }); | |
| 78 | cell.unref(); | |
| 79 | s.adopt(cell.pid); | |
| 80 | console.log(JSON.stringify({ supervisor: s.pid, cell: cell.pid })); | |
| 81 | setInterval(() => {}, 1000); | |
| 82 | `; | |
| 83 | const host = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "inherit", env: { PATH: process.env["PATH"] ?? "", FORGE_TEST_SOCKET: "port" } }); | |
| 84 | killed.push(host.pid); | |
| 85 | const text = await new Promise<string>(async (resolve) => { | |
| 86 | let got = ""; | |
| 87 | for await (const chunk of host.stdout) { | |
| 88 | got += new TextDecoder().decode(chunk); | |
| 89 | if (got.includes("\n")) return resolve(got); | |
| 90 | } | |
| 91 | resolve(got); | |
| 92 | }); | |
| 93 | const { supervisor, cell } = JSON.parse(text) as { supervisor: number; cell: number }; | |
| 94 | killed.push(supervisor, cell); | |
| 95 | expect(await until(() => existsSync(files(ctl).cell))).toBe(true); | |
| 96 | expect(await until(() => alive(supervisor))).toBe(true); | |
| 97 | // Its own session: not the host's group, so a signal to the host's group never reaches it. | |
| 98 | expect(pgid(supervisor)).not.toBe(pgid(host.pid)); | |
| 99 | expect(pgid(supervisor)).toBe(String(supervisor)); | |
| 100 | ||
| 101 | let up = false; | |
| 102 | for (let i = 0; i < 100 && !up; i++) { | |
| 103 | up = await accepts(listen); | |
| 104 | if (!up) await Bun.sleep(50); // real-timer: polling a real listening socket | |
| 105 | } | |
| 106 | expect(up).toBe(true); | |
| 107 | ||
| 108 | host.kill("SIGKILL"); | |
| 109 | await host.exited; | |
| 110 | await Bun.sleep(500); // real-timer: grace period to prove a real supervisor survives its host being killed | |
| 111 | expect(alive(supervisor)).toBe(true); | |
| 112 | expect(alive(cell)).toBe(true); | |
| 113 | expect(await accepts(listen)).toBe(true); | |
| 114 | expect(await accepts(outPort)).toBe(true); | |
| 115 | ||
| 116 | process.kill(cell, "SIGKILL"); | |
| 117 | expect(await until(() => !alive(supervisor), 8_000)).toBe(true); | |
| 118 | // No host to read what is left: the control directory is gone with the supervisor. | |
| 119 | expect(existsSync(ctl)).toBe(false); | |
| 120 | expect(await accepts(listen)).toBe(false); | |
| 121 | }); | |
| 122 | ||
| 123 | const spec = (dir: string) => cellSpecFor({ dir, parentDataDir: join(dir, "p"), parentUrl: "https://f.test", size: "small", port: freePort(), label: "forge:t" }); | |
| 124 | ||
| 125 | // Runs under FORGE_TEST_SOCKET too: an outliving cell binds no socket in this process (its | |
| 126 | // supervisor, a process of its own, does), and nothing here waits on that. | |
| 127 | test("FORGE_CELLS_OUTLIVE=on makes a child's cell its own session, not tied to its parent", () => { | |
| 128 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 129 | expect(cellsOutlive({})).toBe(false); | |
| 130 | expect(cellsOutlive({ FORGE_CELLS_OUTLIVE: "on" })).toBe(true); | |
| 131 | expect(spec(dir).outlive).toBeUndefined(); | |
| 132 | const cell = openCell({ ...spec(dir), outlive: true }); | |
| 133 | try { | |
| 134 | const argv = cell.argv("true"); | |
| 135 | expect(cell.supervised).toBeDefined(); | |
| 136 | expect(argv[0]).toBe("setsid"); | |
| 137 | expect(argv).not.toContain("--die-with-parent"); | |
| 138 | } finally { | |
| 139 | cell.close(); | |
| 140 | } | |
| 141 | expect(existsSync(cell.supervised?.ctl ?? "")).toBe(false); | |
| 142 | }); | |
| 143 | ||
| 144 | // Unix sockets cannot be bound where FORGE_TEST_SOCKET is set; the forge's check runs this. | |
| 145 | test.skipIf(process.env["FORGE_TEST_SOCKET"] !== undefined)("the default cell dies with its parent and is not made a session", () => { | |
| 146 | const cell = openCell(spec(mkdtempSync(join(tmpdir(), "outlive-")))); | |
| 147 | try { | |
| 148 | expect(cell.supervised).toBeUndefined(); | |
| 149 | expect(cell.argv("true")).toContain("--die-with-parent"); | |
| 150 | expect(cell.argv("true")[0]).not.toBe("setsid"); | |
| 151 | } finally { | |
| 152 | cell.close(); | |
| 153 | } | |
| 154 | }); | |
| 155 | ||
| 156 | test("a cell's output file is bounded: past the cap it starts over and the tail reads the new top", async () => { | |
| 157 | const { capFile, perMinute, parseConfig, pidRecord, readPid } = await import("../src/spawn/supervisor.ts"); | |
| 158 | const dir = mkdtempSync(join(tmpdir(), "fc-cap-")); | |
| 159 | const log = join(dir, "cell.log"); | |
| 160 | writeFileSync(log, "x".repeat(2_000)); | |
| 161 | expect(capFile(log, 5_000)).toBe(false); | |
| 162 | appendFileSync(log, "y".repeat(4_000)); | |
| 163 | expect(capFile(log, 5_000)).toBe(true); | |
| 164 | expect(Bun.file(log).size).toBe(0); | |
| 165 | const heard: string[] = []; | |
| 166 | const stop = tailLines(log, (l) => heard.push(l), 20, 1_000); | |
| 167 | appendFileSync(log, `${"z".repeat(900)}\n`); | |
| 168 | expect(await until(() => heard.length === 1)).toBe(true); | |
| 169 | appendFileSync(log, `${"w".repeat(900)}\n`); // past 1,000 bytes: the tail starts the file over | |
| 170 | expect(await until(() => heard.length === 2)).toBe(true); | |
| 171 | expect(await until(() => Bun.file(log).size === 0)).toBe(true); | |
| 172 | appendFileSync(log, "after\n"); | |
| 173 | expect(await until(() => heard.at(-1) === "after")).toBe(true); | |
| 174 | stop(); | |
| 175 | ||
| 176 | let t = 0; | |
| 177 | const ok = perMinute(2, () => t); | |
| 178 | expect([ok(), ok(), ok()]).toEqual([true, true, false]); | |
| 179 | t = 60_000; | |
| 180 | expect(ok()).toBe(true); | |
| 181 | ||
| 182 | // a config is checked field by field; a pid file with another start time is another process | |
| 183 | const good = { sockets: "/s", hosts: ["a"], origins: [{ host: "h", port: 80 }], report: false, loops: [8080] }; | |
| 184 | expect(parseConfig(JSON.stringify(good))).toBeDefined(); | |
| 185 | expect(parseConfig(JSON.stringify({ ...good, loops: ["8080"] }))).toBeUndefined(); | |
| 186 | expect(parseConfig(JSON.stringify({ ...good, hosts: [1] }))).toBeUndefined(); | |
| 187 | expect(parseConfig("nope")).toBeUndefined(); | |
| 188 | const pidFile = join(dir, "cell.pid"); | |
| 189 | writeFileSync(pidFile, pidRecord(process.pid)); | |
| 190 | expect(readPid(pidFile)).toBe(process.pid); | |
| 191 | writeFileSync(pidFile, `${process.pid}:1`); | |
| 192 | expect(readPid(pidFile)).toBeUndefined(); | |
| 193 | }); | |
| 194 | ||
| 195 | /** A cell stand-in that runs `command` under sh, with the supervision given. */ | |
| 196 | const shCell = (supervised: Supervised | undefined) => ({ | |
| 197 | argv: (command: string) => ["sh", "-c", command], | |
| 198 | env: (extra: Record<string, string>) => ({ PATH: process.env["PATH"] ?? "", ...extra }), | |
| 199 | ...(supervised === undefined ? {} : { supervised }), | |
| 200 | }); | |
| 201 | ||
| 202 | test("a supervised cell writes to its supervisor's file, is adopted, and is followed there (fake supervisor)", async () => { | |
| 203 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 204 | const path = join(dir, "cell.log"); | |
| 205 | writeFileSync(path, ""); | |
| 206 | const adopted: number[] = []; | |
| 207 | let unfollowed = false; | |
| 208 | const fake: Supervised = { | |
| 209 | ctl: dir, | |
| 210 | pid: process.pid, | |
| 211 | logFd: openSync(path, "a"), | |
| 212 | adopt: (pid) => void adopted.push(pid), | |
| 213 | follow: (line) => { | |
| 214 | const stop = tailLines(path, line, 20); | |
| 215 | return () => ((unfollowed = true), stop()); | |
| 216 | }, | |
| 217 | close: () => {}, | |
| 218 | }; | |
| 219 | const heard: string[] = []; | |
| 220 | const { proc, unfollow } = spawnInCell(shCell(fake), "echo out; echo err >&2", { cwd: dir, env: {} }, (l) => void heard.push(l)); | |
| 221 | await proc.exited; | |
| 222 | expect(adopted).toEqual([proc.pid]); | |
| 223 | // The output went to the file, not to a pipe this process holds. | |
| 224 | expect(proc.stdout).toBe(fake.logFd); | |
| 225 | expect(readFileSync(path, "utf8")).toBe("out\nerr\n"); | |
| 226 | expect(await until(() => heard.length === 2)).toBe(true); | |
| 227 | unfollow(); | |
| 228 | expect(unfollowed).toBe(true); | |
| 229 | expect(heard).toEqual(["out", "err"]); | |
| 230 | }); | |
| 231 | ||
| 232 | test("an unsupervised cell is read from pipes, and nothing is adopted", async () => { | |
| 233 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 234 | const heard: string[] = []; | |
| 235 | const { proc, unfollow } = spawnInCell(shCell(undefined), "echo piped", { cwd: dir, env: {} }, (l) => void heard.push(l)); | |
| 236 | await proc.exited; | |
| 237 | expect(await until(() => heard.includes("piped"))).toBe(true); | |
| 238 | unfollow(); | |
| 239 | }); | |
| 240 | ||
| 241 | test("an outliving cell's real supervisor adopts the spawned root and its output reaches the host's log", async () => { | |
| 242 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 243 | const cell = openCell({ ...spec(dir), outlive: true }); | |
| 244 | const sup = cell.supervised; | |
| 245 | expect(sup).toBeDefined(); | |
| 246 | if (sup === undefined) return; | |
| 247 | killed.push(sup.pid); | |
| 248 | try { | |
| 249 | const heard: string[] = []; | |
| 250 | const { proc, unfollow } = spawnInCell(shCell(sup), "echo from-the-cell; sleep 30", { cwd: dir, env: {} }, (l) => void heard.push(l)); | |
| 251 | killed.push(proc.pid); | |
| 252 | expect(await until(() => heard.includes("from-the-cell"))).toBe(true); | |
| 253 | expect(await until(() => existsSync(files(sup.ctl).cell) && readFileSync(files(sup.ctl).cell, "utf8").startsWith(`${proc.pid}`))).toBe(true); | |
| 254 | unfollow(); | |
| 255 | proc.kill("SIGKILL"); | |
| 256 | await proc.exited; | |
| 257 | } finally { | |
| 258 | cell.close(); | |
| 259 | } | |
| 260 | }); | |
| 261 | ||
| 262 | test("the host and a supervisor serve one policy: the supervisor's config is the cell's policy", () => { | |
| 263 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 264 | const s = { ...spec(dir), egress: ["api.example.com"], granted: ["10.0.0.5:5432"], egressMode: "report" as const, reach: ["http://127.0.0.1:4100", "https://f.test"], outlive: true }; | |
| 265 | const cell = openCell(s); | |
| 266 | try { | |
| 267 | const ctl = cell.supervised?.ctl ?? ""; | |
| 268 | const config = parseConfig(readFileSync(files(ctl).config, "utf8")); | |
| 269 | expect(config).toBeDefined(); | |
| 270 | const { sockets: _sockets, outPort: _outPort, ...served } = config ?? { sockets: "" }; | |
| 271 | expect(served).toEqual(cellPolicy(s)); | |
| 272 | expect(cellPolicy(s).loops).toEqual([4100]); | |
| 273 | } finally { | |
| 274 | cell.close(); | |
| 275 | } | |
| 276 | // The one mapping to proxy options: report is a hook only when the policy reports. | |
| 277 | const told: string[] = []; | |
| 278 | const opts = proxyOptions(cellPolicy(s), { undeclared: (h, p) => void told.push(`${h}:${p}`) }); | |
| 279 | opts.report?.("x.test", 443); | |
| 280 | expect(told).toEqual(["x.test:443"]); | |
| 281 | expect(opts.hosts).toEqual(["api.example.com"]); | |
| 282 | expect(opts.granted).toEqual(["10.0.0.5:5432"]); | |
| 283 | expect(proxyOptions({ ...cellPolicy(s), report: false }, {}).report).toBeUndefined(); | |
| 284 | }); | |
| 285 | ||
| 286 | const stopsWith = async (outlive: boolean): Promise<boolean> => { | |
| 287 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 288 | const app = createApp({ dataDir: dir, port: 0 }); | |
| 289 | const before = process.env["FORGE_CELLS_OUTLIVE"]; | |
| 290 | try { | |
| 291 | const registry = cellChildren(app, async (input) => { | |
| 292 | const proc = Bun.spawn(["sleep", "30"], { stdout: "ignore", stderr: "ignore" }); | |
| 293 | killed.push(proc.pid); | |
| 294 | return { ...input, port: 1, limits: cellLimits(input.size), pid: proc.pid, exited: proc.exited }; | |
| 295 | }); | |
| 296 | const child = await registry.start({ bodyId: "cell-kept", name: "kept", size: "small" }); | |
| 297 | expect(child).toBeDefined(); | |
| 298 | if (outlive) process.env["FORGE_CELLS_OUTLIVE"] = "on"; | |
| 299 | else delete process.env["FORGE_CELLS_OUTLIVE"]; | |
| 300 | await stopCellChildren(app); | |
| 301 | return !alive(child?.pid ?? 0) || (await Promise.race([child?.exited.then(() => true), Bun.sleep(200).then(() => false)])) === true; // real-timer: a real child either exited or is still up | |
| 302 | } finally { | |
| 303 | if (before === undefined) delete process.env["FORGE_CELLS_OUTLIVE"]; | |
| 304 | else process.env["FORGE_CELLS_OUTLIVE"] = before; | |
| 305 | app.close(); | |
| 306 | } | |
| 307 | }; | |
| 308 | ||
| 309 | test("stopping the host leaves an outliving cell running, and stops a default one", async () => { | |
| 310 | expect(await stopsWith(true)).toBe(false); | |
| 311 | expect(await stopsWith(false)).toBe(true); | |
| 312 | }); |
test/job-slices.test.ts +2
| @@ -292,6 +292,8 @@ const UNTIMED: Record<string, string> = { | ||
| 292 | 292 | "roll-probe": "cluster only (FORGE_RELEASE_KEEPER=on): one HTTP probe", |
| 293 | 293 | "apps-sweep": "AppsRuntime's sweep over running app processes; the fixture has no apps", |
| 294 | 294 | "profiler-drain": "drains the sampling profiler's stack traces on a timer (perf/sample.ts) so they do not pile up; runs only where the profiler is on, and the fixture samples nothing", |
| 295 | "cell output tail": "runs in a cell's supervisor process, not the host: reads a bounded tail of the child's output file and trims it; needs a supervised cell", | |
| 296 | "cell allowed sync": "runs in a cell's supervisor process, not the host: one small file write of the allowed hosts; needs a supervised cell", | |
| 295 | 297 | "cell-watch": "watches running app cells' cgroups; needs cells, skipped in this sandbox mode", |
| 296 | 298 | "cell-probe-deadline": "a cell's start probe running out of time: needs cells, skipped in this sandbox mode", |
| 297 | 299 | "cell-kill-root": "kills a cell's root process after its grace: needs cells, skipped in this sandbox mode", |
test/spawn-sites.test.ts +2
| @@ -93,6 +93,8 @@ const SITES: Record<string, string> = { | ||
| 93 | 93 | "bun (process.execPath) on the forge's own child_db_reader.ts, a fixed argv with an empty environment; the directory asked about and what a request presented travel over the IPC channel, never argv", |
| 94 | 94 | "src/spawn/cell.ts": |
| 95 | 95 | "a child forge launched from a launch spec the parent built, wrapped by its cell", |
| 96 | "src/spawn/supervisor.ts": | |
| 97 | "setsid with the forge's own bun and its own supervisor.ts, one control directory the host made as argv; nothing from a request", | |
| 96 | 98 | "src/tui/screens/external.ts": |
| 97 | 99 | "the operator's $EDITOR resolved with Bun.which, refused when it holds whitespace, and a file path the TUI made, as argv", |
| 98 | 100 | "src/tui/term/terminal.ts": |
test/timers-named.test.ts +3
| @@ -64,6 +64,9 @@ const allow = (call: Call, why: string, at: readonly (readonly [string, string]) | ||
| 64 | 64 | * does work on the loop: a span around each would only fill the ring with nothing. |
| 65 | 65 | */ |
| 66 | 66 | const ALLOWED: readonly Allowed[] = [ |
| 67 | ...allow("setInterval", "the cell supervisor's own tick: a separate process with no job ring, its one timer", [ | |
| 68 | ["src/spawn/supervisor.ts", "() => { capFile(f.log"], | |
| 69 | ]), | |
| 67 | 70 | ...allow("setInterval", `SSE heartbeat, ${PER_CONNECTION}`, [["src/http/sse.ts", "() => write("]]), |
| 68 | 71 | ...allow("setInterval", `a talk stream's poll, ${PER_CONNECTION}`, [["src/web/pages/talk.ts", "tick"]]), |
| 69 | 72 | ...allow("setInterval", `a shell session's idle check, ${PER_CONNECTION}`, [["src/machines/shell.ts", "() => { if (idle(session"]]), |
new above
offline
read(tailLines) andwriteAllowedrun in the host process, not the supervisor process, so 'no job ring' is false for them. Name those jobs; allow only the supervisor's own tick.