Public: anyone can read this; the crew can change it.
A child forge runs as its own process in its own cell, held to its size #6019
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2926-a-child-forge-runs-as-its-own-process-in → main 3 commits fork 9 threads · 9 unresolved
docs/SPAWN.md +1
| @@ -16,6 +16,7 @@ and user accounts. The child mints its own. A test in #420 and #422 proves it. | ||
| 16 | 16 | | ----- | ------------------------------------------------------- | ---------------- | |
| 17 | 17 | | #420 | spawn plan: one validated value, dry-run | #383 | |
| 18 | 18 | | #427 | placement seam; hosted inside the parent today | #420 | |
| 19 | | #2926 | `cell` placement: a child as its own `bun src/main.ts` in a cell held to its size (`spawn/cell.ts`); `hosted` where the host cannot hold a cell. The spawn answer and the children list say `placement` | #2925 | | |
| 19 | 20 | | #421 | spawn action: org, instance, apps as repos, briefs | #420, #427 | |
| 20 | 21 | |
| 21 | 22 | `src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token). |
src/actions/spawn.ts +15 −2
| @@ -52,7 +52,7 @@ import { Database } from "bun:sqlite"; | ||
| 52 | 52 | import { loadKey, open } from "../kernel/secret.ts"; |
| 53 | 53 | import { loadOrCreateHostKey } from "../ssh/keys.ts"; |
| 54 | 54 | import { claimLink, hasAdmin, mintClaim, spawnedOrg } from "../spawn/claim.ts"; |
| 55 | import { childPublicUrl, parentOrigin, type Placement } from "../spawn/placement.ts"; | |
| 55 | import { childPublicUrl, parentOrigin, type Placement, type PlacementKind } from "../spawn/placement.ts"; | |
| 56 | 56 | import { quotaRefusal } from "../spawn/quota.ts"; |
| 57 | 57 | import { planSpawn, type SpawnInput, type SpawnPlan, type SpawnSource } from "../spawn/plan.ts"; |
| 58 | 58 | import { DEFAULT_SIZE, isSize, type Size } from "../spawn/sizes.ts"; |
| @@ -64,6 +64,8 @@ export interface Spawned { | ||
| 64 | 64 | readonly org: string; |
| 65 | 65 | readonly url: string; |
| 66 | 66 | readonly bodyId: string; |
| 67 | /** Where the child runs: in this process, or as a process in its own cell. */ | |
| 68 | readonly placement: PlacementKind; | |
| 67 | 69 | /** True when this call found the child already made and changed nothing. */ |
| 68 | 70 | readonly existed: boolean; |
| 69 | 71 | /** |
| @@ -112,6 +114,10 @@ interface Ledger { | ||
| 112 | 114 | readonly createdAt?: string; |
| 113 | 115 | /** Absent on a ledger written before sizes; reads as small. */ |
| 114 | 116 | readonly size?: Size; |
| 117 | /** Absent on a ledger written before cells; reads as hosted. */ | |
| 118 | readonly placement?: PlacementKind; | |
| 119 | /** Why a child that wanted a cell is hosted instead (`spawn/host.ts`). */ | |
| 120 | readonly placementNote?: string; | |
| 115 | 121 | /** |
| 116 | 122 | * What the parent shows the child to learn its private facts (when anything |
| 117 | 123 | * last happened); the child keeps only its hash. Never in an answer. |
| @@ -234,7 +240,7 @@ async function make( | ||
| 234 | 240 | }); |
| 235 | 241 | } |
| 236 | 242 | if (ledger.done) |
| 237 | return ok({ name: plan.name, org: plan.org, url: liveChildUrl(app, plan.name), bodyId: ledger.bodyId, existed: true }); | |
| 243 | return ok({ name: plan.name, org: plan.org, url: liveChildUrl(app, plan.name), bodyId: ledger.bodyId, placement: ledger.placement ?? "hosted", existed: true }); | |
| 238 | 244 | if (ledger.statusToken === undefined || ledger.beatKey === undefined) { |
| 239 | 245 | ledger.statusToken ??= secret(); |
| 240 | 246 | ledger.beatKey ??= secret(); |
| @@ -261,6 +267,7 @@ async function make( | ||
| 261 | 267 | apps: plan.apps.map((a) => a.ref), |
| 262 | 268 | createdAt: new Date().toISOString(), |
| 263 | 269 | size: plan.size ?? DEFAULT_SIZE, |
| 270 | placement: placed.value.kind ?? "hosted", | |
| 264 | 271 | statusToken: secret(), |
| 265 | 272 | beatKey: secret(), |
| 266 | 273 | }; |
| @@ -284,6 +291,7 @@ async function make( | ||
| 284 | 291 | org: plan.org, |
| 285 | 292 | url: liveChildUrl(app, plan.name), |
| 286 | 293 | bodyId: ledger.bodyId, |
| 294 | placement: ledger.placement ?? "hosted", | |
| 287 | 295 | existed: false, |
| 288 | 296 | claimUrl, |
| 289 | 297 | }); |
| @@ -665,6 +673,9 @@ export interface Child { | ||
| 665 | 673 | /** When it was begun; null on a ledger older than the client list. */ |
| 666 | 674 | readonly createdAt: string | null; |
| 667 | 675 | readonly size: Size; |
| 676 | readonly placement: PlacementKind; | |
| 677 | /** Why it is hosted when it asked for a cell; null otherwise. */ | |
| 678 | readonly placementNote: string | null; | |
| 668 | 679 | } |
| 669 | 680 | |
| 670 | 681 | /** The children this forge has made or begun, from its ledger, by name. */ |
| @@ -684,6 +695,8 @@ export async function listChildren(app: App): Promise<Child[]> { | ||
| 684 | 695 | apps: ledger.apps ?? [], |
| 685 | 696 | createdAt: ledger.createdAt ?? null, |
| 686 | 697 | size: isSize(ledger.size) ? ledger.size : DEFAULT_SIZE, |
| 698 | placement: ledger.placement ?? "hosted", | |
| 699 | placementNote: ledger.placementNote ?? null, | |
| 687 | 700 | }); |
| 688 | 701 | } |
| 689 | 702 | return out; |
src/apps/contain.ts +3 −1
| @@ -189,6 +189,8 @@ export interface CellSpec { | ||
| 189 | 189 | /** Origins always reachable: the forge's own URL. */ |
| 190 | 190 | readonly reach: readonly string[]; |
| 191 | 191 | readonly limits: Limits; |
| 192 | /** Read-only paths to keep visible when they sit under a hidden directory (a forge's own code). */ | |
| 193 | readonly reveal?: readonly string[] | undefined; | |
| 192 | 194 | /** The port an app listens on; the forge relays `127.0.0.1:<port>` in. */ |
| 193 | 195 | readonly listen?: number | undefined; |
| 194 | 196 | /** Keep the controlling terminal (an app's `tty`). */ |
| @@ -405,7 +407,7 @@ export function openCell( | ||
| 405 | 407 | const blank = hidden.filter((path) => !isDirectory(path)); |
| 406 | 408 | // The runtime itself must stay visible when it lives in a hidden home |
| 407 | 409 | // (`~/.bun/bin/bun` on a developer's machine). |
| 408 | const reveal = [dirname(process.execPath)].filter((path) => | |
| 410 | const reveal = [dirname(process.execPath), ...(spec.reveal ?? [])].filter((path) => | |
| 409 | 411 | hide.some((h) => path === h || path.startsWith(`${h}/`)), |
| 410 | 412 | ); |
| 411 | 413 | const writable = [...new Set([spec.dir, spec.home, ...spec.writable])]; |
src/http/server.ts +15 −1
| @@ -532,8 +532,9 @@ import { | ||
| 532 | 532 | resolveAppHost, |
| 533 | 533 | } from "../apps/proxy.ts"; |
| 534 | 534 | |
| 535 | import { stopCellChildren } from "../spawn/cell.ts"; | |
| 535 | 536 | import { portInsteadOfSocket, tcpAddress } from "../git/hook-address.ts"; |
| 536 | import { closeHostedChildren, resolveHostedChild, resolveHostedChildApp } from "../spawn/host.ts"; | |
| 537 | import { answerFromCell, closeHostedChildren, resolveHostedChild, resolveHostedChildApp } from "../spawn/host.ts"; | |
| 537 | 538 | import { getVerifiedDomain } from "../core/domains.ts"; |
| 538 | 539 | |
| 539 | 540 | // ---- auth ---- |
| @@ -2277,6 +2278,18 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2277 | 2278 | const child = await resolveHostedChild(app, request); |
| 2278 | 2279 | if (child === "unknown") |
| 2279 | 2280 | return securityFloor(new Response("not found", { status: 404 })); |
| 2281 | if (child === "cell") { | |
| 2282 | // The child sees only this forge's relay, so it is told who the client is. | |
| 2283 | notePeer(request, server.requestIP(request)?.address, trust); | |
| 2284 | armClose(request, () => server.timeout(request, 1)); | |
| 2285 | const response = await answered(app, request, () => answerFromCell(app, request)); | |
| 2286 | counted(app, request, response, started); | |
| 2287 | logged(app.config.dataDir, request, response, started); | |
| 2288 | await settleBody(request); | |
| 2289 | if (!mustClose(request)) return response; | |
| 2290 | server.timeout(request, 1); | |
| 2291 | return withHeader(response, "connection", "close"); | |
| 2292 | } | |
| 2280 | 2293 | if (child !== undefined) { |
| 2281 | 2294 | const key = child.config.dataDir; |
| 2282 | 2295 | let http = childHttps.get(key); |
| @@ -2395,6 +2408,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2395 | 2408 | for (const http of childHttps.values()) http.close(); |
| 2396 | 2409 | childHttps.clear(); |
| 2397 | 2410 | closeHostedChildren(app); |
| 2411 | void stopCellChildren(app); | |
| 2398 | 2412 | flushAccess(); |
| 2399 | 2413 | web.stop(true); |
| 2400 | 2414 | internal.stop(true); |
src/spawn/cell.ts added +299
| @@ -0,0 +1,299 @@ | ||
| 1 | /** | |
| 2 | * The `cell` placement: a child forge as a process of its own. It is | |
| 3 | * `bun src/main.ts` with its data directory, started from `childEnv` | |
| 4 | * (invariant 11) inside a cell (`apps/contain.ts`) whose memory, CPU and | |
| 5 | * disk are its size's (`spawn/sizes.ts`). The parent keeps answering | |
| 6 | * `<name>--<host>` and forwards to the loopback port the cell relays in on | |
| 7 | * (`forward`). Where the host cannot hold a cell the child is hosted | |
| 8 | * (`spawn/placement.ts` `fallbackPlacement`), and the ledger says which. | |
| 9 | */ | |
| 10 | ||
| 11 | import { dirname, join } from "node:path"; | |
| 12 | import { clientAddress } from "../http/peer.ts"; | |
| 13 | import type { App } from "../app.ts"; | |
| 14 | import { cellsAvailable, killTree, openCell, type Cell, type CellSpec, type Limits } from "../apps/contain.ts"; | |
| 15 | import { childEnv, freePort } from "../apps/process.ts"; | |
| 16 | import { childDirOf } from "../actions/spawn.ts"; | |
| 17 | import { hostedPlacement, parentOrigin, childPublicUrl, type BodyStore, type Placement } from "./placement.ts"; | |
| 18 | import { SIZES, type Size } from "./sizes.ts"; | |
| 19 | ||
| 20 | export const CELL_PREFIX = "cell-"; | |
| 21 | export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX); | |
| 22 | ||
| 23 | /** After a child fails to start, requests are answered 503 at once for this long. */ | |
| 24 | export const START_BACKOFF_MS = 30_000; | |
| 25 | ||
| 26 | const MAIN = join(import.meta.dir, "..", "main.ts"); | |
| 27 | const CODE = join(import.meta.dir, "..", ".."); | |
| 28 | ||
| 29 | /** A size's limits as a cell holds them. */ | |
| 30 | export function cellLimits(size: Size): Limits { | |
| 31 | const s = SIZES[size]; | |
| 32 | return { memoryMiB: s.memoryMiB, cpus: s.cpu, diskMiB: s.diskMiB }; | |
| 33 | } | |
| 34 | ||
| 35 | /** The cell a child of `size` runs in; plain data, so a test reads the limits off it. */ | |
| 36 | export function cellSpecFor(input: { | |
| 37 | dir: string; | |
| 38 | parentDataDir: string; | |
| 39 | parentUrl: string; | |
| 40 | size: Size; | |
| 41 | port: number; | |
| 42 | label: string; | |
| 43 | }): CellSpec { | |
| 44 | return { | |
| 45 | dir: input.dir, | |
| 46 | writable: [], | |
| 47 | home: join(input.dir, ".home"), | |
| 48 | // The parent's data is the one thing a child must not see; its own | |
| 49 | // directory sits under it and is bound back, deepest last. | |
| 50 | hide: [input.parentDataDir], | |
| 51 | // The forge's own code may live under the parent's data (a release checkout). | |
| 52 | reveal: [CODE], | |
| 53 | egress: [], | |
| 54 | reach: [input.parentUrl], | |
| 55 | limits: cellLimits(input.size), | |
| 56 | listen: input.port, | |
| 57 | label: input.label, | |
| 58 | }; | |
| 59 | } | |
| 60 | ||
| 61 | /** The child forge's environment: short, explicit, and its own. */ | |
| 62 | export function cellEnv(input: { port: number; dataDir: string; url: string }): Record<string, string> { | |
| 63 | const env = childEnv({ | |
| 64 | PORT: String(input.port), | |
| 65 | FORGE_HOST: "127.0.0.1", | |
| 66 | FORGE_APP_URL: input.url, | |
| 67 | FORGE_APP_DATA: input.dataDir, | |
| 68 | NODE_ENV: "production", | |
| 69 | }); | |
| 70 | delete env["USER"]; | |
| 71 | delete env["LOGNAME"]; | |
| 72 | return env; | |
| 73 | } | |
| 74 | ||
| 75 | export interface CellChild { | |
| 76 | readonly bodyId: string; | |
| 77 | readonly name: string; | |
| 78 | readonly size: Size; | |
| 79 | readonly port: number; | |
| 80 | readonly limits: Limits; | |
| 81 | readonly pid: number; | |
| 82 | readonly exited: Promise<number>; | |
| 83 | } | |
| 84 | ||
| 85 | interface Held extends CellChild { | |
| 86 | readonly cell: Cell; | |
| 87 | readonly proc: ReturnType<typeof Bun.spawn>; | |
| 88 | } | |
| 89 | ||
| 90 | export interface CellChildren { | |
| 91 | /** Start the child (idempotent); resolves once it answers, or says why not. */ | |
| 92 | start(input: { bodyId: string; name: string; size: Size }): Promise<CellChild | undefined>; | |
| 93 | stop(bodyId: string): Promise<void>; | |
| 94 | get(bodyId: string): CellChild | undefined; | |
| 95 | /** Why the last start failed, in a sentence a person can read; undefined when it did not. */ | |
| 96 | failure(bodyId: string): string | undefined; | |
| 97 | list(): readonly CellChild[]; | |
| 98 | } | |
| 99 | ||
| 100 | const registries = new WeakMap<App, CellChildren>(); | |
| 101 | ||
| 102 | export function cellChildren(parent: App): CellChildren { | |
| 103 | let r = registries.get(parent); | |
| 104 | if (r === undefined) registries.set(parent, (r = makeRegistry(parent))); | |
| 105 | return r; | |
| 106 | } | |
| 107 | ||
| 108 | async function answers(port: number, gone: Promise<unknown>): Promise<boolean> { | |
| 109 | let over = false; | |
| 110 | void gone.then(() => (over = true)); | |
| 111 | for (let i = 0; i < 300 && !over; i++) { | |
| 112 | try { | |
| 113 | await fetch(`http://127.0.0.1:${port}/`, { redirect: "manual", signal: AbortSignal.timeout(500) }); | |
| 114 | return true; | |
| 115 | } catch { | |
| 116 | await Bun.sleep(100); | |
| 117 | } | |
| 118 | } | |
| 119 | return false; | |
| 120 | } | |
| 121 | ||
| 122 | function makeRegistry(parent: App): CellChildren { | |
| 123 | const held = new Map<string, Held>(); | |
| 124 | const starting = new Map<string, Promise<CellChild | undefined>>(); | |
| 125 | /** When a start last failed: a broken child is not relaunched on every request. */ | |
| 126 | const failed = new Map<string, { at: number; why: string }>(); | |
| 127 | /** Held once it answers, never before: a start in flight is in `starting`. */ | |
| 128 | ||
| 129 | const launch = async (input: { bodyId: string; name: string; size: Size }): Promise<CellChild | undefined> => { | |
| 130 | const cells = await cellsAvailable(); | |
| 131 | if (!cells.ok) { | |
| 132 | parent.log.warn("child cell not started", { name: input.name, reason: cells.reason }); | |
| 133 | failed.set(input.bodyId, { at: Date.now(), why: cells.reason }); | |
| 134 | return undefined; | |
| 135 | } | |
| 136 | const recent: string[] = []; | |
| 137 | const port = freePort(); | |
| 138 | const dir = childDirOf(parent, input.bodyId); | |
| 139 | const url = childPublicUrl(input.name, parentOrigin(parent.config)); | |
| 140 | const spec = cellSpecFor({ | |
| 141 | dir, | |
| 142 | parentDataDir: parent.config.dataDir, | |
| 143 | parentUrl: parentOrigin(parent.config), | |
| 144 | size: input.size, | |
outdated
On the preview this process exits within about 1 s of every launch, and the child never answers (503 on every request). Before trusting the paths here, prove that |
||
| 145 | port, | |
| 146 | label: `forge:${input.name}`, | |
| 147 | }); | |
| 148 | const log = (line: string): void => { | |
| 149 | recent.push(line); | |
| 150 | if (recent.length > 5) recent.shift(); | |
| 151 | parent.log.info("child forge", { name: input.name, line }); | |
| 152 | }; | |
| 153 | const cell = openCell(spec, log); | |
| 154 | const command = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(MAIN)}`; | |
| 155 | const proc = Bun.spawn(cell.argv(command), { | |
| 156 | cwd: dirname(MAIN), | |
| 157 | env: cell.env(cellEnv({ port, dataDir: dir, url })), | |
| 158 | stdin: "ignore", | |
| 159 | stdout: "pipe", | |
| 160 | stderr: "pipe", | |
| 161 | }); | |
outdated
This is held before it answers. A concurrent |
||
| 162 | const exited = proc.exited.then((code) => { | |
| 163 | const why = cell.breach(); | |
| 164 | if (why !== undefined) parent.log.warn("child forge stopped", { name: input.name, why }); | |
| 165 | cell.close(); | |
| 166 | if (held.get(input.bodyId)?.proc === proc) held.delete(input.bodyId); | |
| 167 | if (why !== undefined) recent.push(why); | |
| 168 | return code; | |
| 169 | }); | |
| 170 | cell.watch(proc.pid, () => killTree(proc.pid)); | |
| 171 | void drain(proc.stdout, log); | |
| 172 | void drain(proc.stderr, log); | |
| 173 | const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc }; | |
| 174 | if (!(await answers(port, exited))) { | |
| 175 | const why = `its cell did not come up: ${recent.slice(-3).join(" | ") || "no output"}`.slice(0, 400); | |
| 176 | parent.log.warn("child forge did not answer", { name: input.name, why }); | |
| 177 | failed.set(input.bodyId, { at: Date.now(), why }); | |
| 178 | killTree(proc.pid); | |
outdated
If |
||
| 179 | return undefined; | |
| 180 | } | |
| 181 | held.set(input.bodyId, child); | |
| 182 | return child; | |
| 183 | }; | |
| 184 | ||
| 185 | return { | |
| 186 | start(input) { | |
| 187 | const there = held.get(input.bodyId); | |
| 188 | if (there !== undefined) return Promise.resolve(there); | |
| 189 | const lastFailure = failed.get(input.bodyId); | |
| 190 | if (lastFailure !== undefined && Date.now() - lastFailure.at < START_BACKOFF_MS) return Promise.resolve(undefined); | |
| 191 | let flight = starting.get(input.bodyId); | |
| 192 | if (flight === undefined) { | |
| 193 | flight = launch(input) | |
| 194 | .catch((error: unknown) => { | |
| 195 | const why = `its cell could not be made: ${String(error)}`.slice(0, 400); | |
| 196 | parent.log.warn("child forge could not start", { name: input.name, why }); | |
| 197 | failed.set(input.bodyId, { at: Date.now(), why }); | |
| 198 | return undefined; | |
| 199 | }) | |
| 200 | .then((child) => { | |
| 201 | if (child !== undefined) failed.delete(input.bodyId); | |
| 202 | return child; | |
| 203 | }); | |
| 204 | starting.set(input.bodyId, flight); | |
| 205 | void flight.finally(() => starting.delete(input.bodyId)).catch(() => {}); | |
| 206 | } | |
| 207 | return flight; | |
| 208 | }, | |
| 209 | async stop(bodyId) { | |
| 210 | const child = held.get(bodyId); | |
| 211 | if (child === undefined) return; | |
| 212 | held.delete(bodyId); | |
| 213 | killTree(child.pid); | |
| 214 | await child.exited; | |
| 215 | }, | |
| 216 | get: (bodyId) => held.get(bodyId), | |
| 217 | failure: (bodyId) => failed.get(bodyId)?.why, | |
| 218 | list: () => [...held.values()], | |
| 219 | }; | |
| 220 | } | |
| 221 | ||
| 222 | async function drain(stream: ReadableStream<Uint8Array> | number | null | undefined, log: (line: string) => void): Promise<void> { | |
| 223 | if (stream === null || stream === undefined || typeof stream === "number") return; | |
| 224 | const text = new TextDecoder(); | |
| 225 | let rest = ""; | |
| 226 | try { | |
| 227 | for await (const chunk of stream) { | |
| 228 | rest += text.decode(chunk, { stream: true }); | |
| 229 | const lines = rest.split("\n"); | |
| 230 | rest = lines.pop() ?? ""; | |
| 231 | for (const line of lines) if (line !== "") log(line.slice(0, 500)); | |
| 232 | } | |
| 233 | } catch { | |
| 234 | // the process is gone | |
| 235 | } | |
| 236 | } | |
| 237 | ||
| 238 | /** Stop every child cell this parent holds: when the parent stops, they go with it. */ | |
| 239 | export async function stopCellChildren(parent: App): Promise<void> { | |
outdated
High: this copies the client's |
||
| 240 | const r = registries.get(parent); | |
| 241 | if (r === undefined) return; | |
| 242 | registries.delete(parent); | |
| 243 | await Promise.all(r.list().map((c) => r.stop(c.bodyId))); | |
| 244 | } | |
| 245 | ||
| 246 | /** The cell placement: a body in the shared store, stopped when retired. */ | |
| 247 | export function cellPlacement(parent: App, store: BodyStore, maxChildren: () => number): Placement { | |
| 248 | const base = hostedPlacement({ | |
| 249 | host: parentOrigin(parent.config), | |
| 250 | store, | |
| 251 | maxChildren, | |
| 252 | kind: "cell", | |
| 253 | newId: () => `${CELL_PREFIX}${crypto.randomUUID()}`, | |
| 254 | }); | |
| 255 | return { | |
| 256 | ...(base.room === undefined ? {} : { room: base.room }), | |
| 257 | place: base.place, | |
| 258 | async retire(bodyId) { | |
| 259 | await cellChildren(parent).stop(bodyId); | |
| 260 | return base.retire(bodyId); | |
| 261 | }, | |
| 262 | }; | |
| 263 | } | |
| 264 | ||
| 265 | // ---- forwarding ---- | |
| 266 | ||
| 267 | /** Headers that say who the client is: the child believes them from its relay, so only the forge writes them. */ | |
| 268 | const WHO = new Set(["x-forwarded-for", "forwarded", "x-real-ip", "cf-connecting-ip", "true-client-ip", "x-forwarded-host", "x-forwarded-proto"]); | |
| 269 | ||
| 270 | const HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "content-length"]); | |
| 271 | ||
| 272 | /** Carry a request to the child's loopback port and its answer back, streaming. */ | |
| 273 | export async function forward( | |
| 274 | child: Pick<CellChild, "port">, | |
| 275 | request: Request, | |
| 276 | clientHeader?: string, | |
| 277 | ): Promise<Response> { | |
| 278 | const at = new URL(request.url); | |
| 279 | const headers = new Headers(); | |
| 280 | const named = clientHeader?.toLowerCase(); | |
| 281 | for (const [k, v] of request.headers) if (!HOP.has(k) && !WHO.has(k) && k !== named) headers.set(k, v); | |
| 282 | // The address the forge resolved (`http/peer.ts`), the only one the child hears. | |
| 283 | headers.set("x-forwarded-for", clientAddress(request)); | |
| 284 | const hasBody = request.method !== "GET" && request.method !== "HEAD"; | |
| 285 | try { | |
| 286 | const res = await fetch(`http://127.0.0.1:${child.port}${at.pathname}${at.search}`, { | |
| 287 | method: request.method, | |
| 288 | headers, | |
| 289 | body: hasBody ? request.body : null, | |
| 290 | redirect: "manual", | |
| 291 | decompress: false, | |
| 292 | } as RequestInit); | |
| 293 | const out = new Headers(); | |
| 294 | for (const [k, v] of res.headers) if (!HOP.has(k)) out.append(k, v); | |
| 295 | return new Response(res.body, { status: res.status, headers: out }); | |
| 296 | } catch { | |
| 297 | return new Response("this forge is not answering", { status: 502 }); | |
| 298 | } | |
| 299 | } |
src/spawn/door.ts +11 −1
| @@ -19,6 +19,7 @@ import type { Ctx } from "../http/context.ts"; | ||
| 19 | 19 | import { takeDoor } from "../http/strangers.ts"; |
| 20 | 20 | import { |
| 21 | 21 | childPublicUrl, |
| 22 | fallbackPlacement, | |
| 22 | 23 | hostedPlacement, |
| 23 | 24 | memoryBodies, |
| 24 | 25 | parentOrigin, |
| @@ -27,6 +28,8 @@ import { | ||
| 27 | 28 | } from "./placement.ts"; |
| 28 | 29 | import { dryRun, planSpawn, KEY_BUDGET_NOTE, MAX_APPS, type CatalogApp, type CrewBrief, type SpawnSource } from "./plan.ts"; |
| 29 | 30 | import { bootHostedChild } from "./host.ts"; |
| 31 | import { cellPlacement, ownsBody } from "./cell.ts"; | |
| 32 | import { cellsAvailable } from "../apps/contain.ts"; | |
| 30 | 33 | import { ANON_SPAWNS_PER_DAY, checkAnonSpawn, noteAnonSpawn } from "./rate.ts"; |
| 31 | 34 | import { readdirSync, readFileSync } from "node:fs"; |
| 32 | 35 | import { readdir, stat } from "node:fs/promises"; |
| @@ -101,7 +104,14 @@ function placementFor(app: App): Placement { | ||
| 101 | 104 | if (!p) { |
| 102 | 105 | const store = memoryBodies(); |
| 103 | 106 | seedBodies(app, store); |
| 104 | p = hostedPlacement({ host: parentOrigin(app.config), store, maxChildren: () => maxChildren(app.db) }); | |
| 107 | const cap = () => maxChildren(app.db); | |
| 108 | // A cell where the host can hold one, the parent's own process where it cannot. | |
| 109 | p = fallbackPlacement( | |
| 110 | cellPlacement(app, store, cap), | |
| 111 | hostedPlacement({ host: parentOrigin(app.config), store, maxChildren: cap }), | |
| 112 | async () => (await cellsAvailable()).ok, | |
| 113 | ownsBody, | |
| 114 | ); | |
| 105 | 115 | placements.set(app, p); |
| 106 | 116 | } |
| 107 | 117 | return p; |
src/spawn/family.ts +7 −1
| @@ -9,12 +9,15 @@ import { parentForgeFor } from "../core/parent_releases.ts"; | ||
| 9 | 9 | import { running } from "../kernel/version.ts"; |
| 10 | 10 | import type { ChildAppsSummary } from "./child-apps.ts"; |
| 11 | 11 | import { childStandings, parentUpstream, type ChildStanding } from "./heartbeat.ts"; |
| 12 | import { listChildren } from "../actions/spawn.ts"; | |
| 12 | 13 | import { hostedChildApps } from "./host.ts"; |
| 13 | 14 | |
| 14 | 15 | /** A child as the family shows it: its standing, and what its apps are doing. */ |
| 15 | 16 | export interface FamilyChild extends ChildStanding { |
| 16 | 17 | /** "1 of 2 apps running — bakery asleep", or null when it was given no apps. */ |
| 17 | 18 | readonly apps: string | null; |
| 19 | /** Where it runs: "own cell", "hosted", or "hosted: <why its cell did not start>". */ | |
| 20 | readonly placement: string | null; | |
| 18 | 21 | } |
| 19 | 22 | |
| 20 | 23 | const REASONS = new Set(["asleep", "failed", "stopped", "deploying"]); |
| @@ -57,6 +60,9 @@ export async function familyView( | ||
| 57 | 60 | ): Promise<FamilyView> { |
| 58 | 61 | const known = parentForgeFor(app.config); |
| 59 | 62 | const url = app.config.parentUrl ?? (await parentUpstream(app)); |
| 63 | const placed = new Map<string, string>(); | |
| 64 | if (isAdmin) | |
| 65 | for (const c of await listChildren(app)) placed.set(c.name, c.placementNote ?? (c.placement === "cell" ? "own cell" : "hosted")); | |
| 60 | 66 | const kids = isAdmin ? await childStandings(app) : { items: [], more: 0 }; |
| 61 | 67 | return { |
| 62 | 68 | parent: |
| @@ -67,7 +73,7 @@ export async function familyView( | ||
| 67 | 73 | host: app.config.url === undefined ? null : new URL(app.config.url).host, |
| 68 | 74 | version: running().version, |
| 69 | 75 | }, |
| 70 | children: kids.items.map((c) => ({ ...c, apps: appsLine(appsOf(app, c.name)) })), | |
| 76 | children: kids.items.map((c) => ({ ...c, apps: appsLine(appsOf(app, c.name)), placement: placed.get(c.name) ?? null })), | |
| 71 | 77 | more: kids.more, |
| 72 | 78 | }; |
| 73 | 79 | } |
src/spawn/host.ts +64 −5
| @@ -19,14 +19,19 @@ import { | ||
| 19 | 19 | parentOrigin, |
| 20 | 20 | parseHostedChildAppLabel, |
| 21 | 21 | parseHostedChildName, |
| 22 | type PlacementKind, | |
| 22 | 23 | } from "./placement.ts"; |
| 23 | import { readdirSync, readFileSync } from "node:fs"; | |
| 24 | import { DEFAULT_SIZE, isSize, type Size } from "./sizes.ts"; | |
| 25 | import { cellChildren, forward } from "./cell.ts"; | |
| 26 | import { readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 24 | 27 | import { join } from "node:path"; |
| 25 | 28 | |
| 26 | 29 | interface ChildLedger { |
| 27 | 30 | readonly name: string; |
| 28 | 31 | readonly bodyId: string; |
| 29 | 32 | readonly done: boolean; |
| 33 | readonly placement: PlacementKind; | |
| 34 | readonly size: Size; | |
| 30 | 35 | } |
| 31 | 36 | |
| 32 | 37 | function readLedgerSync(app: App, name: string): ChildLedger | undefined { |
| @@ -36,9 +41,17 @@ function readLedgerSync(app: App, name: string): ChildLedger | undefined { | ||
| 36 | 41 | name?: unknown; |
| 37 | 42 | bodyId?: unknown; |
| 38 | 43 | done?: unknown; |
| 44 | placement?: unknown; | |
| 45 | size?: unknown; | |
| 39 | 46 | }; |
| 40 | 47 | if (typeof parsed.name !== "string" || typeof parsed.bodyId !== "string") return undefined; |
| 41 | return { name: parsed.name, bodyId: parsed.bodyId, done: parsed.done === true }; | |
| 48 | return { | |
| 49 | name: parsed.name, | |
| 50 | bodyId: parsed.bodyId, | |
| 51 | done: parsed.done === true, | |
| 52 | placement: parsed.placement === "cell" ? "cell" : "hosted", | |
| 53 | size: isSize(parsed.size) ? parsed.size : DEFAULT_SIZE, | |
| 54 | }; | |
| 42 | 55 | } catch { |
| 43 | 56 | return undefined; |
| 44 | 57 | } |
| @@ -92,22 +105,49 @@ async function openChild(parent: App, ledger: ChildLedger): Promise<App> { | ||
| 92 | 105 | * Resolve a request Host to a hosted child's App. |
| 93 | 106 | * - `undefined` — not a child host (parent / app / other continues). |
| 94 | 107 | * - `"unknown"` — shaped like a child host but no finished ledger. |
| 108 | * - `"cell"` — a process of its own: `answerFromCell` carries the request. | |
| 95 | 109 | * - `App` — answer with this child's routes. |
| 96 | 110 | */ |
| 97 | 111 | export async function resolveHostedChild( |
| 98 | 112 | parent: App, |
| 99 | 113 | request: Request, |
| 100 | ): Promise<App | "unknown" | undefined> { | |
| 114 | ): Promise<App | "cell" | "unknown" | undefined> { | |
| 101 | 115 | const host = request.headers.get("host"); |
| 102 | 116 | if (host === null) return undefined; |
| 103 | 117 | const name = parseHostedChildName(host, childHost(parent)); |
| 104 | 118 | if (name === undefined) return undefined; |
| 105 | 119 | const ledger = readLedgerSync(parent, name); |
| 106 | 120 | if (ledger === undefined || !ledger.done) return "unknown"; |
| 107 | ||
| 121 | // A child in its own cell is not an App here: the request is carried to it. | |
| 122 | if (ledger.placement === "cell") return settleCell(parent, ledger); | |
| 108 | 123 | return openOnce(parent, ledger); |
| 109 | 124 | } |
| 110 | 125 | |
| 126 | /** | |
| 127 | * Start a child's cell. When the cell cannot come up the child is not left | |
| 128 | * dead: the ledger turns it hosted, says why (`placementNote`, which | |
| 129 | * `/forges` lists), and the child is opened in this process. | |
| 130 | */ | |
| 131 | async function settleCell(parent: App, ledger: ChildLedger): Promise<App | "cell"> { | |
| 132 | const cells = cellChildren(parent); | |
| 133 | if ((await cells.start(ledger)) !== undefined) return "cell"; | |
On the preview this start runs inside the visitor's request and waits the full |
||
| 134 | const why = cells.failure(ledger.bodyId) ?? "its cell did not start"; | |
| 135 | demote(parent, ledger.name, why); | |
| 136 | parent.log.warn("child forge placed hosted instead", { name: ledger.name, why }); | |
| 137 | return openOnce(parent, { ...ledger, placement: "hosted" }); | |
| 138 | } | |
| 139 | ||
| 140 | function demote(parent: App, name: string, why: string): void { | |
| 141 | const path = join(parent.config.dataDir, "spawn", `${name}.json`); | |
| 142 | try { | |
| 143 | const raw = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>; | |
| 144 | writeFileSync(`${path}.tmp`, JSON.stringify({ ...raw, placement: "hosted", placementNote: `hosted: ${why}` })); | |
| 145 | renameSync(`${path}.tmp`, path); | |
| 146 | } catch (error) { | |
| 147 | parent.log.error("could not record the placement", { name, error: String(error) }); | |
| 148 | } | |
| 149 | } | |
| 150 | ||
| 111 | 151 | function openOnce(parent: App, ledger: ChildLedger): Promise<App> | App { |
| 112 | 152 | const cache = cacheOf(parent); |
| 113 | 153 | const hit = cache.get(ledger.bodyId); |
| @@ -131,8 +171,13 @@ function openOnce(parent: App, ledger: ChildLedger): Promise<App> | App { | ||
| 131 | 171 | * the first visit after a claim finds them awake. Never throws. |
| 132 | 172 | */ |
| 133 | 173 | export async function bootHostedChild(parent: App, name: string): Promise<void> { |
| 134 | const ledger = readLedgerSync(parent, name); | |
| 174 | let ledger = readLedgerSync(parent, name); | |
| 135 | 175 | if (ledger === undefined || !ledger.done) return; |
| 176 | if (ledger.placement === "cell") { | |
| 177 | const settled = await settleCell(parent, ledger); | |
| 178 | if (settled === "cell") return; | |
| 179 | ledger = { ...ledger, placement: "hosted" }; | |
| 180 | } | |
| 136 | 181 | try { |
| 137 | 182 | await startChildApps(parent, await openOnce(parent, ledger), name); |
| 138 | 183 | } catch (error) { |
| @@ -198,3 +243,17 @@ export async function resolveHostedChildApp( | ||
| 198 | 243 | |
| 199 | 244 | /** What a child's apps are doing, for `/forges`; undefined when none run here. */ |
| 200 | 245 | export const hostedChildApps = childAppsSummary; |
| 246 | ||
| 247 | /** | |
| 248 | * Answer a request for a child that runs in its own cell: start it if it is | |
| 249 | * not up (a parent that restarted, a child that stopped), then forward. | |
| 250 | */ | |
| 251 | export async function answerFromCell(parent: App, request: Request): Promise<Response> { | |
| 252 | const name = parseHostedChildName(request.headers.get("host") ?? "", childHost(parent)); | |
| 253 | const ledger = name === undefined ? undefined : readLedgerSync(parent, name); | |
| 254 | if (ledger === undefined || ledger.placement !== "cell") return new Response("not found", { status: 404 }); | |
| 255 | const child = await cellChildren(parent).start(ledger); | |
| 256 | // Unreachable once a cell has failed (the ledger turned hosted), but never a hang. | |
| 257 | if (child === undefined) return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5" } }); | |
| 258 | return forward(child, request, parent.config.clientIpHeader); | |
| 259 | } |
src/spawn/placement.ts +28 −1
| @@ -12,9 +12,14 @@ | ||
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | 13 | import type { SpawnPlan } from "./plan.ts"; |
| 14 | 14 | |
| 15 | /** Where a child runs: inside the parent's process, or as a process in its own cell. */ | |
| 16 | export type PlacementKind = "hosted" | "cell"; | |
| 17 | ||
| 15 | 18 | export interface Placed { |
| 16 | 19 | readonly url: string; |
| 17 | 20 | readonly bodyId: string; |
| 21 | /** Absent reads as hosted. */ | |
| 22 | readonly kind?: PlacementKind; | |
| 18 | 23 | } |
| 19 | 24 | |
| 20 | 25 | export type PlacementRefusal = { readonly reason: "children" | "depth" | "exists" | "quota" | "unknown"; readonly message: string }; |
| @@ -143,6 +148,8 @@ export interface HostedOptions { | ||
| 143 | 148 | readonly store?: BodyStore; |
| 144 | 149 | /** Injectable so tests need no randomness. */ |
| 145 | 150 | readonly newId?: () => string; |
| 151 | /** What `place` says it did; hosted unless the caller is a cell placement. */ | |
| 152 | readonly kind?: PlacementKind; | |
| 146 | 153 | } |
| 147 | 154 | |
| 148 | 155 | const refuse = (reason: PlacementRefusal["reason"], message: string) => |
| @@ -182,7 +189,7 @@ export function hostedPlacement(opts: HostedOptions): Placement { | ||
| 182 | 189 | const url = childPublicUrl(plan.name, opts.host); |
| 183 | 190 | const bodyId = newId(); |
| 184 | 191 | store.add({ bodyId, name: plan.name, url, depth: depth + 1, agents: 1, ssh: false }); |
| 185 | return ok({ url, bodyId }); | |
| 192 | return ok({ url, bodyId, ...(opts.kind === undefined ? {} : { kind: opts.kind }) }); | |
| 186 | 193 | }, |
| 187 | 194 | async retire(bodyId) { |
| 188 | 195 | store.remove(bodyId); |
| @@ -190,3 +197,23 @@ export function hostedPlacement(opts: HostedOptions): Placement { | ||
| 190 | 197 | }, |
| 191 | 198 | }; |
| 192 | 199 | } |
| 200 | ||
| 201 | /** | |
| 202 | * Two placements behind one: `preferred` where `holds()` says this host can, | |
| 203 | * `fallback` where it cannot. A body is retired by whichever made it, told | |
| 204 | * by its id's prefix. Capacity is the fallback's (they share one store). | |
| 205 | */ | |
| 206 | export function fallbackPlacement( | |
| 207 | preferred: Placement, | |
| 208 | fallback: Placement, | |
| 209 | holds: () => Promise<boolean>, | |
| 210 | owns: (bodyId: string) => boolean, | |
| 211 | ): Placement { | |
| 212 | return { | |
| 213 | ...(fallback.room === undefined ? {} : { room: fallback.room }), | |
| 214 | async place(plan) { | |
| 215 | return (await holds()) ? preferred.place(plan) : fallback.place(plan); | |
| 216 | }, | |
| 217 | retire: (bodyId) => (owns(bodyId) ? preferred : fallback).retire(bodyId), | |
| 218 | }; | |
| 219 | } |
src/web/pages/family.ts +1
| @@ -49,6 +49,7 @@ function standingNode(c: FamilyChild): Node { | ||
| 49 | 49 | const follow = followLine(c); |
| 50 | 50 | if (follow !== null) meta.push(html`<span data-follow>${follow}</span>`); |
| 51 | 51 | if (c.apps !== null) meta.push(html`<span data-apps>${c.apps}</span>`); |
| 52 | if (c.placement !== null) meta.push(html`<span data-placement>${c.placement}</span>`); | |
The page gets the placement, but |
||
| 52 | 53 | return { |
| 53 | 54 | label: c.name, |
| 54 | 55 | href: c.url, |
test/spawn-api.test.ts +2
| @@ -165,6 +165,8 @@ describe("spawn over the API and MCP", () => { | ||
| 165 | 165 | apps: expect.any(Array), |
| 166 | 166 | createdAt: expect.any(String), |
| 167 | 167 | size: "small", |
| 168 | placement: "hosted", | |
| 169 | placementNote: null, | |
| 168 | 170 | }, |
| 169 | 171 | ]); |
| 170 | 172 | // The same request again is answered, not made twice. |
test/spawn-cell.test.ts added +204
| @@ -0,0 +1,204 @@ | ||
| 1 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 2 | import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { cellsAvailable, openCell } from "../src/apps/contain.ts"; | |
| 6 | import { closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts"; | |
| 7 | import { createApp } from "../src/app.ts"; | |
| 8 | import { cellChildren, stopCellChildren, cellEnv, cellLimits, cellPlacement, cellSpecFor, forward, ownsBody } from "../src/spawn/cell.ts"; | |
| 9 | import { fallbackPlacement, hostedPlacement, memoryBodies } from "../src/spawn/placement.ts"; | |
| 10 | import type { SpawnPlan } from "../src/spawn/plan.ts"; | |
| 11 | import { SIZES } from "../src/spawn/sizes.ts"; | |
| 12 | import { notePeer } from "../src/http/peer.ts"; | |
| 13 | import type { App } from "../src/app.ts"; | |
| 14 | ||
| 15 | const plan = (name: string): SpawnPlan => ({ name, org: "o", apps: [], briefs: [], subscriptions: [], direction: "", keepAccess: false, budgetUsd: null }); | |
| 16 | const tmp = mkdtempSync(join(tmpdir(), "spawn-cell-")); | |
| 17 | afterAll(() => rmSync(tmp, { recursive: true, force: true })); | |
| 18 | ||
| 19 | describe("a child forge's cell", () => { | |
| 20 | test("holds the size's memory, CPU and disk", () => { | |
| 21 | for (const size of ["small", "medium", "large"] as const) { | |
| 22 | expect(cellLimits(size)).toEqual({ memoryMiB: SIZES[size].memoryMiB, cpus: SIZES[size].cpu, diskMiB: SIZES[size].diskMiB }); | |
| 23 | } | |
| 24 | const spec = cellSpecFor({ dir: "/d/children/c1", parentDataDir: "/d", parentUrl: "https://f.test", size: "small", port: 4100, label: "forge:kid" }); | |
| 25 | expect(spec.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 }); | |
| 26 | expect(spec.hide).toEqual(["/d"]); | |
| 27 | expect(spec.listen).toBe(4100); | |
| 28 | expect(spec.dir).toBe("/d/children/c1"); | |
| 29 | }); | |
| 30 | ||
| 31 | test("starts from childEnv: the parent's secrets are not handed on", () => { | |
| 32 | process.env["FORGE_EDGE_TOKEN"] = "parent-secret"; | |
| 33 | try { | |
| 34 | const env = cellEnv({ port: 4100, dataDir: "/d/children/c1", url: "https://kid--f.test" }); | |
| 35 | expect(env["FORGE_EDGE_TOKEN"]).toBeUndefined(); | |
| 36 | expect(env["FORGE_APP_URL"]).toBe("https://kid--f.test"); | |
| 37 | expect(env["FORGE_APP_DATA"]).toBe("/d/children/c1"); | |
| 38 | expect(env["PORT"]).toBe("4100"); | |
| 39 | } finally { | |
| 40 | delete process.env["FORGE_EDGE_TOKEN"]; | |
| 41 | } | |
| 42 | }); | |
| 43 | }); | |
| 44 | ||
| 45 | describe("the placement seam", () => { | |
| 46 | const app = { config: { dataDir: tmp, url: "https://f.test" } } as unknown as App; | |
| 47 | const build = (cells: boolean) => { | |
| 48 | const store = memoryBodies(); | |
| 49 | const cap = () => 10; | |
| 50 | return fallbackPlacement( | |
| 51 | cellPlacement(app, store, cap), | |
| 52 | hostedPlacement({ host: "https://f.test", store, maxChildren: cap, newId: () => "hosted-1" }), | |
| 53 | async () => cells, | |
| 54 | ownsBody, | |
| 55 | ); | |
| 56 | }; | |
| 57 | ||
| 58 | test("a cell where the host can hold one, and it says so", async () => { | |
| 59 | const r = await build(true).place(plan("kid")); | |
| 60 | if (!r.ok) throw new Error("refused"); | |
| 61 | expect(r.value.kind).toBe("cell"); | |
| 62 | expect(ownsBody(r.value.bodyId)).toBe(true); | |
| 63 | expect(r.value.url).toBe("https://kid--f.test"); | |
| 64 | }); | |
| 65 | ||
| 66 | test("hosted where it cannot", async () => { | |
| 67 | const r = await build(false).place(plan("kid")); | |
| 68 | if (!r.ok) throw new Error("refused"); | |
| 69 | expect(r.value.kind ?? "hosted").toBe("hosted"); | |
| 70 | expect(ownsBody(r.value.bodyId)).toBe(false); | |
| 71 | }); | |
| 72 | ||
| 73 | test("a retired cell child frees its room", async () => { | |
| 74 | const p = build(true); | |
| 75 | const r = await p.place(plan("kid")); | |
| 76 | if (!r.ok) throw new Error("refused"); | |
| 77 | expect((await p.retire(r.value.bodyId)).ok).toBe(true); | |
| 78 | expect((await p.place(plan("kid"))).ok).toBe(true); | |
| 79 | }); | |
| 80 | }); | |
| 81 | ||
| 82 | describe("forwarding to a child", () => { | |
| 83 | test("carries method, path, query, body and the answer back", async () => { | |
| 84 | const server = Bun.serve({ | |
| 85 | port: 0, | |
| 86 | async fetch(req) { | |
| 87 | const u = new URL(req.url); | |
| 88 | return new Response(`${req.method} ${u.pathname}${u.search} ${await req.text()} ${req.headers.get("host")}`, { | |
| 89 | status: 201, | |
| 90 | headers: { "set-cookie": "a=b; HttpOnly" }, | |
| 91 | }); | |
| 92 | }, | |
| 93 | }); | |
| 94 | try { | |
| 95 | const res = await forward({ port: server.port! }, new Request("http://kid--f.test/x?y=1", { method: "POST", body: "hi", headers: { host: "kid--f.test" } })); | |
| 96 | expect(res.status).toBe(201); | |
| 97 | expect(await res.text()).toBe("POST /x?y=1 hi kid--f.test"); | |
| 98 | expect(res.headers.get("set-cookie")).toBe("a=b; HttpOnly"); | |
| 99 | } finally { | |
| 100 | server.stop(true); | |
| 101 | } | |
| 102 | }); | |
| 103 | ||
| 104 | test("a child that is not listening answers 502", async () => { | |
| 105 | const res = await forward({ port: 1 }, new Request("http://kid--f.test/")); | |
| 106 | expect(res.status).toBe(502); | |
| 107 | }); | |
| 108 | }); | |
| 109 | ||
| 110 | const cells = await cellsAvailable(); | |
outdated
This test runs |
||
| 111 | describe("a real cell", () => { | |
| 112 | test.skipIf(!cells.ok)("runs the child with the small limits and cannot see the parent's data", async () => { | |
| 113 | const parent = join(tmp, "parent"); | |
| 114 | const dir = join(parent, "children", "c1"); | |
| 115 | mkdirSync(dir, { recursive: true }); | |
| 116 | writeFileSync(join(parent, "forge.db"), "secret"); | |
| 117 | const spec = cellSpecFor({ dir, parentDataDir: parent, parentUrl: "https://f.test", size: "small", port: 4999, label: "forge:test" }); | |
| 118 | expect(spec.limits.memoryMiB).toBe(512); | |
| 119 | const cell = openCell(spec); | |
| 120 | const proc = Bun.spawn(cell.argv(`ls ${parent}; echo own > ${dir}/mine`), { env: cell.env({ PATH: process.env["PATH"] ?? "" }), stdout: "pipe", stderr: "pipe" }); | |
| 121 | const out = await new Response(proc.stdout).text(); | |
| 122 | await proc.exited; | |
| 123 | cell.close(); | |
| 124 | expect(out).not.toContain("forge.db"); | |
| 125 | expect(out).toContain("children"); | |
| 126 | }); | |
| 127 | }); | |
| 128 | ||
| 129 | describe("who the child hears", () => { | |
| 130 | test("a spoofed client header never reaches the child; it hears the address the forge resolved", async () => { | |
| 131 | let seen: Headers | undefined; | |
| 132 | const server = Bun.serve({ port: 0, fetch(req) { seen = req.headers; return new Response("ok"); } }); | |
| 133 | try { | |
| 134 | const request = new Request("http://kid--f.test/", { | |
| 135 | headers: { | |
| 136 | host: "kid--f.test", | |
| 137 | "x-forwarded-for": "6.6.6.6", | |
| 138 | forwarded: "for=6.6.6.6", | |
| 139 | "x-real-ip": "6.6.6.6", | |
| 140 | "cf-connecting-ip": "6.6.6.6", | |
| 141 | "x-edge-ip": "6.6.6.6", | |
| 142 | }, | |
| 143 | }); | |
| 144 | notePeer(request, "203.0.113.9"); | |
| 145 | await forward({ port: server.port! }, request, "x-edge-ip"); | |
| 146 | expect(seen?.get("x-forwarded-for")).toBe("203.0.113.9"); | |
| 147 | for (const h of ["forwarded", "x-real-ip", "cf-connecting-ip", "x-edge-ip"]) expect(seen?.get(h)).toBeNull(); | |
| 148 | } finally { | |
| 149 | server.stop(true); | |
| 150 | } | |
| 151 | }); | |
| 152 | }); | |
| 153 | ||
| 154 | describe("a child that will not start", () => { | |
| 155 | test.skipIf(cells.ok)("is not relaunched on every request", async () => { | |
| 156 | const warned: string[] = []; | |
| 157 | const app = { config: { dataDir: tmp, url: "https://f.test" }, log: { warn: (m: string) => warned.push(m), info() {}, error() {} } } as unknown as App; | |
| 158 | const children = cellChildren(app); | |
| 159 | const input = { bodyId: "cell-x", name: "kid", size: "small" } as const; | |
| 160 | expect(await children.start(input)).toBeUndefined(); | |
| 161 | expect(await children.start(input)).toBeUndefined(); | |
| 162 | expect(warned.length).toBe(1); | |
| 163 | }); | |
| 164 | }); | |
| 165 | ||
| 166 | describe("a real child forge", () => { | |
| 167 | test.skipIf(!cells.ok)("boots in its cell and answers /healthz through the forward", async () => { | |
| 168 | const parent = createApp({ dataDir: join(tmp, "real-parent"), url: "https://f.test" }); | |
| 169 | try { | |
| 170 | const children = cellChildren(parent); | |
| 171 | const child = await children.start({ bodyId: "cell-real", name: "kid", size: "small" }); | |
| 172 | expect(children.failure("cell-real")).toBeUndefined(); | |
| 173 | if (child === undefined) throw new Error(`the child did not start: ${children.failure("cell-real")}`); | |
| 174 | expect(child.limits).toEqual({ memoryMiB: 512, cpus: 1, diskMiB: 2048 }); | |
| 175 | const res = await forward(child, new Request("http://kid--f.test/healthz", { headers: { host: "kid--f.test" } })); | |
| 176 | expect(res.status).toBe(200); | |
| 177 | await children.stop("cell-real"); | |
| 178 | } finally { | |
| 179 | await stopCellChildren(parent); | |
| 180 | parent.close(); | |
| 181 | } | |
| 182 | }, 60_000); | |
| 183 | }); | |
| 184 | ||
| 185 | describe("a child whose cell will not come up", () => { | |
| 186 | test.skipIf(cells.ok)("is hosted instead, and the ledger says why", async () => { | |
| 187 | const dataDir = join(tmp, "demote-parent"); | |
| 188 | const parent = createApp({ dataDir, url: "https://f.test" }); | |
| 189 | try { | |
| 190 | mkdirSync(join(dataDir, "spawn"), { recursive: true }); | |
| 191 | const file = join(dataDir, "spawn", "kid.json"); | |
| 192 | writeFileSync(file, JSON.stringify({ name: "kid", bodyId: "cell-d", done: true, placement: "cell", size: "small" })); | |
| 193 | const child = await resolveHostedChild(parent, new Request("https://kid--f.test/", { headers: { host: "kid--f.test" } })); | |
| 194 | expect(child).not.toBe("cell"); | |
| 195 | expect(child).not.toBe("unknown"); | |
| 196 | const ledger = JSON.parse(readFileSync(file, "utf8")); | |
| 197 | expect(ledger.placement).toBe("hosted"); | |
| 198 | expect(ledger.placementNote).toStartWith("hosted: "); | |
| 199 | } finally { | |
| 200 | await closeHostedChildren(parent); | |
| 201 | parent.close(); | |
| 202 | } | |
| 203 | }); | |
| 204 | }); |
test/spawn-host.test.ts +1 −1
| @@ -150,7 +150,7 @@ describe("a parent on plain http with a port", () => { | ||
| 150 | 150 | parent, |
| 151 | 151 | new Request("http://forge.lan:8080/healthz", { headers: { host: "kid--forge.lan:8080" } }), |
| 152 | 152 | ); |
| 153 | if (child === undefined || child === "unknown") throw new Error("child not found"); | |
| 153 | if (child === undefined || child === "unknown" || child === "cell") throw new Error("child not found"); | |
| 154 | 154 | expect(child.config.url).toBe("http://kid--forge.lan:8080"); |
| 155 | 155 | }); |
| 156 | 156 | }); |
test/spawn-pipes.test.ts +1
| @@ -16,6 +16,7 @@ import { join } from "node:path"; | ||
| 16 | 16 | const PIPING: Record<string, { sites: string[]; reason: string }> = { |
| 17 | 17 | "src/apps/contain.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" }, |
| 18 | 18 | "src/apps/process.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "pump() reads stdout and stderr of the app until exit" }, |
| 19 | "src/spawn/cell.ts": { sites: ["stdout+stderr"], reason: "drain() reads the child forge's stdout and stderr to its log until exit" }, | |
| 19 | 20 | "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" }, |
| 20 | 21 | "src/crew/claude.ts": { sites: ["stdout+stderr", "stdout"], reason: "session pumps stdout and stderr; --version awaits stdout, stderr ignored" }, |
| 21 | 22 | "src/crew/plan.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" }, |
High (pairs with cell.ts:239): no
notePeercall before this, so the parent has no resolved client to carry. Medium: this also bypassesarmClose,answered,counted,loggedandsettleBody, which the hosted branch below runs.