Public: anyone can read this; the crew can change it.
A cell child inside a preview reaches its parent for heartbeats, and the real child forge test fails rather than skips where it can run #6031
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6030-a-cell-child-inside-a-preview-reaches-it → main 2 commits fork 1 thread · 1 unresolved
5 files
changed +155
−8
forge.json +2 −2
| @@ -15,8 +15,8 @@ | ||
| 15 | 15 | } |
| 16 | 16 | }, |
| 17 | 17 | "check": { |
| 18 | "fast": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"", | |
| 19 | "run": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --gate --impact", | |
| 18 | "fast": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && FORGE_REQUIRE_CELLS=1 bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"", | |
| 19 | "run": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && FORGE_REQUIRE_CELLS=1 bun scripts/test-shards.ts --timeout 60000 --gate --impact", | |
| 20 | 20 | "timeout": 7200, |
| 21 | 21 | "limits": { |
| 22 | 22 | "memory": 8192, |
src/spawn/beat-door.ts added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | /** | |
| 2 | * The one door a cell child has to its parent: a loopback listener that | |
| 3 | * answers `POST /api/v1/heartbeat` and nothing else. The child's cell relays | |
| 4 | * only this port in, so untrusted child code never reaches the parent's web | |
| 5 | * surface, and since the door never asks who the client is, no forwarding | |
| 6 | * header it sends counts for anything. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { App } from "../app.ts"; | |
| 10 | import { receiveBeat, SIGNATURE_HEADER } from "./heartbeat.ts"; | |
| 11 | ||
| 12 | const MAX_BODY = 1024; | |
| 13 | ||
| 14 | /** Where a child reaches its parent's door from inside its cell. */ | |
| 15 | export const parentLocal = (port: number): string => `http://127.0.0.1:${port}`; | |
| 16 | ||
| 17 | /** `FORGE_PARENT_LOCAL` as `parentLocal` writes it, and nothing else. */ | |
| 18 | export const isParentLocal = (value: string): boolean => /^http:\/\/127\.0\.0\.1:\d{1,5}$/.test(value); | |
| 19 | ||
| 20 | export interface BeatDoor { | |
| 21 | readonly port: number; | |
| 22 | close(): void; | |
| 23 | } | |
| 24 | ||
| 25 | export function openBeatDoor(app: App): BeatDoor { | |
| 26 | const server = Bun.serve({ | |
| 27 | port: 0, | |
| 28 | hostname: "127.0.0.1", | |
| 29 | maxRequestBodySize: MAX_BODY, | |
| 30 | async fetch(req) { | |
| 31 | const refused = new Response("no such door", { status: 404 }); | |
| 32 | if (req.method !== "POST" || new URL(req.url).pathname !== "/api/v1/heartbeat") return refused; | |
| 33 | const raw = await req.text(); | |
| 34 | if (raw.length > MAX_BODY) return refused; | |
| 35 | const kept = await receiveBeat(app, raw, req.headers.get(SIGNATURE_HEADER)); | |
| 36 | return kept ? Response.json({ ok: true }) : refused; | |
| 37 | }, | |
| 38 | }); | |
| 39 | return { port: server.port!, close: () => server.stop(true) }; | |
| 40 | } |
src/spawn/cell.ts +20 −5
| @@ -14,6 +14,7 @@ import type { App } from "../app.ts"; | ||
| 14 | 14 | import { cellsAvailable, killTree, openCell, type Cell, type CellSpec, type Limits } from "../apps/contain.ts"; |
| 15 | 15 | import { childEnv, freePort } from "../apps/process.ts"; |
| 16 | 16 | import { childDirOf } from "../actions/spawn.ts"; |
| 17 | import { openBeatDoor, parentLocal, type BeatDoor } from "./beat-door.ts"; | |
| 17 | 18 | import { hostedPlacement, parentOrigin, childPublicUrl, type BodyStore, type Placement } from "./placement.ts"; |
| 18 | 19 | import { SIZES, type Size } from "./sizes.ts"; |
| 19 | 20 | |
| @@ -37,6 +38,8 @@ export function cellSpecFor(input: { | ||
| 37 | 38 | dir: string; |
| 38 | 39 | parentDataDir: string; |
| 39 | 40 | parentUrl: string; |
| 41 | /** The parent's heartbeat door (`beat-door.ts`): the only way to it that needs no public address (a preview's cell has none). */ | |
| 42 | parentPort?: number; | |
| 40 | 43 | size: Size; |
| 41 | 44 | port: number; |
| 42 | 45 | label: string; |
| @@ -51,7 +54,7 @@ export function cellSpecFor(input: { | ||
| 51 | 54 | // The forge's own code may live under the parent's data (a release checkout). |
| 52 | 55 | reveal: [CODE], |
| 53 | 56 | egress: [], |
| 54 | reach: [input.parentUrl], | |
| 57 | reach: input.parentPort === undefined ? [input.parentUrl] : [input.parentUrl, parentLocal(input.parentPort)], | |
| 55 | 58 | limits: cellLimits(input.size), |
| 56 | 59 | listen: input.port, |
| 57 | 60 | label: input.label, |
| @@ -59,13 +62,14 @@ export function cellSpecFor(input: { | ||
| 59 | 62 | } |
| 60 | 63 | |
| 61 | 64 | /** The child forge's environment: short, explicit, and its own. */ |
| 62 | export function cellEnv(input: { port: number; dataDir: string; url: string }): Record<string, string> { | |
| 65 | export function cellEnv(input: { port: number; dataDir: string; url: string; parentPort?: number }): Record<string, string> { | |
| 63 | 66 | const env = childEnv({ |
| 64 | 67 | PORT: String(input.port), |
| 65 | 68 | FORGE_HOST: "127.0.0.1", |
| 66 | 69 | FORGE_APP_URL: input.url, |
| 67 | 70 | FORGE_APP_DATA: input.dataDir, |
| 68 | 71 | NODE_ENV: "production", |
| 72 | ...(input.parentPort === undefined ? {} : { FORGE_PARENT_LOCAL: parentLocal(input.parentPort) }), | |
| 69 | 73 | }); |
| 70 | 74 | delete env["USER"]; |
| 71 | 75 | delete env["LOGNAME"]; |
| @@ -83,6 +87,7 @@ export interface CellChild { | ||
| 83 | 87 | } |
| 84 | 88 | |
| 85 | 89 | interface Held extends CellChild { |
| 90 | readonly door: BeatDoor; | |
| 86 | 91 | readonly cell: Cell; |
| 87 | 92 | readonly proc: ReturnType<typeof Bun.spawn>; |
| 88 | 93 | } |
| @@ -137,10 +142,13 @@ function makeRegistry(parent: App): CellChildren { | ||
| 137 | 142 | const port = freePort(); |
| 138 | 143 | const dir = childDirOf(parent, input.bodyId); |
| 139 | 144 | const url = childPublicUrl(input.name, parentOrigin(parent.config)); |
| 145 | const door = openBeatDoor(parent); | |
| 146 | const parentPort = door.port; | |
| 140 | 147 | const spec = cellSpecFor({ |
| 141 | 148 | dir, |
| 142 | 149 | parentDataDir: parent.config.dataDir, |
| 143 | 150 | parentUrl: parentOrigin(parent.config), |
| 151 | parentPort, | |
| 144 | 152 | size: input.size, |
| 145 | 153 | port, |
| 146 | 154 | label: `forge:${input.name}`, |
| @@ -150,11 +158,17 @@ function makeRegistry(parent: App): CellChildren { | ||
| 150 | 158 | if (recent.length > 5) recent.shift(); |
| 151 | 159 | parent.log.info("child forge", { name: input.name, line }); |
| 152 | 160 | }; |
| 153 | const cell = openCell(spec, log); | |
| 161 | let cell: Cell; | |
| 162 | try { | |
| 163 | cell = openCell(spec, log); | |
| 164 | } catch (error) { | |
| 165 | door.close(); | |
| 166 | throw error; | |
| 167 | } | |
| 154 | 168 | const command = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(MAIN)}`; |
| 155 | 169 | const proc = Bun.spawn(cell.argv(command), { |
| 156 | 170 | cwd: dirname(MAIN), |
| 157 | env: cell.env(cellEnv({ port, dataDir: dir, url })), | |
| 171 | env: cell.env(cellEnv({ port, dataDir: dir, url, parentPort })), | |
| 158 | 172 | stdin: "ignore", |
| 159 | 173 | stdout: "pipe", |
| 160 | 174 | stderr: "pipe", |
| @@ -163,6 +177,7 @@ function makeRegistry(parent: App): CellChildren { | ||
| 163 | 177 | const why = cell.breach(); |
| 164 | 178 | if (why !== undefined) parent.log.warn("child forge stopped", { name: input.name, why }); |
| 165 | 179 | cell.close(); |
| 180 | door.close(); | |
| 166 | 181 | if (held.get(input.bodyId)?.proc === proc) held.delete(input.bodyId); |
| 167 | 182 | if (why !== undefined) recent.push(why); |
| 168 | 183 | return code; |
| @@ -170,7 +185,7 @@ function makeRegistry(parent: App): CellChildren { | ||
| 170 | 185 | cell.watch(proc.pid, () => killTree(proc.pid)); |
| 171 | 186 | void drain(proc.stdout, log); |
| 172 | 187 | void drain(proc.stderr, log); |
| 173 | const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc }; | |
| 188 | const child: Held = { ...input, port, limits: spec.limits, pid: proc.pid, exited, cell, proc, door }; | |
| 174 | 189 | if (!(await answers(port, exited))) { |
| 175 | 190 | const why = `its cell did not come up: ${recent.slice(-3).join(" | ") || "no output"}`.slice(0, 400); |
| 176 | 191 | parent.log.warn("child forge did not answer", { name: input.name, why }); |
src/spawn/heartbeat.ts +8 −1
| @@ -21,6 +21,7 @@ import { spentTodayEverywhere } from "../crew/autonomy.ts"; | ||
| 21 | 21 | import { beatKeyOf, listChildren } from "../actions/spawn.ts"; |
| 22 | 22 | import releases from "../core/releases.json" with { type: "json" }; |
| 23 | 23 | import { forgeLinkOf, forgeTipOf, FOLLOW_MODES, type FollowMode, type ForgeLink } from "./chain.ts"; |
| 24 | import { isParentLocal } from "./beat-door.ts"; | |
| 24 | 25 | import { upstreamState } from "./upstream.ts"; |
| 25 | 26 | import { progress } from "../core/next-steps.ts"; |
| 26 | 27 | import { running } from "../kernel/version.ts"; |
| @@ -273,6 +274,12 @@ async function forkReport(app: App, forge: ForgeLink) { | ||
| 273 | 274 | }; |
| 274 | 275 | } |
| 275 | 276 | |
| 277 | /** In a cell the parent's public address is out of reach; its relayed loopback port (`FORGE_PARENT_LOCAL`) is not. */ | |
| 278 | function beatBase(upstream: string): string { | |
| 279 | const local = process.env["FORGE_PARENT_LOCAL"] ?? ""; | |
| 280 | return isParentLocal(local) ? local : upstream; | |
| 281 | } | |
| 282 | ||
| 276 | 283 | type FetchLike = (url: string, init?: RequestInit) => Promise<Response>; |
| 277 | 284 | let lastAt = 0; |
| 278 | 285 | |
| @@ -299,7 +306,7 @@ export async function sendBeat( | ||
| 299 | 306 | : await forkReport(app, forge)), |
| 300 | 307 | }); |
| 301 | 308 | try { |
| 302 | const res = await fetcher(`${link.upstream}/api/v1/heartbeat`, { | |
| 309 | const res = await fetcher(`${beatBase(link.upstream)}/api/v1/heartbeat`, { | |
| 303 | 310 | method: "POST", |
| 304 | 311 | headers: { "content-type": "application/json", [SIGNATURE_HEADER]: sign(link.key, body) }, |
| 305 | 312 | body, |
test/spawn-cell.test.ts +85
| @@ -11,6 +11,8 @@ import type { SpawnPlan } from "../src/spawn/plan.ts"; | ||
| 11 | 11 | import { SIZES } from "../src/spawn/sizes.ts"; |
| 12 | 12 | import { notePeer } from "../src/http/peer.ts"; |
| 13 | 13 | import type { App } from "../src/app.ts"; |
| 14 | import { openBeatDoor, parentLocal } from "../src/spawn/beat-door.ts"; | |
| 15 | import { lastHeard, sendBeat } from "../src/spawn/heartbeat.ts"; | |
| 14 | 16 | |
| 15 | 17 | const plan = (name: string): SpawnPlan => ({ name, org: "o", apps: [], briefs: [], subscriptions: [], direction: "", keepAccess: false, budgetUsd: null }); |
| 16 | 18 | const tmp = mkdtempSync(join(tmpdir(), "spawn-cell-")); |
| @@ -42,6 +44,57 @@ describe("a child forge's cell", () => { | ||
| 42 | 44 | }); |
| 43 | 45 | }); |
| 44 | 46 | |
| 47 | describe("a child reaching its parent from inside its cell", () => { | |
| 48 | test("the cell relays the parent's heartbeat door in, and the child is told where", () => { | |
| 49 | const spec = cellSpecFor({ dir: "/d/c", parentDataDir: "/d", parentUrl: "https://f.test", parentPort: 3000, size: "small", port: 4100, label: "forge:kid" }); | |
| 50 | expect(spec.reach).toEqual(["https://f.test", parentLocal(3000)]); | |
| 51 | expect(cellEnv({ port: 4100, dataDir: "/d/c", url: "https://kid--f.test", parentPort: 3000 })["FORGE_PARENT_LOCAL"]).toBe("http://127.0.0.1:3000"); | |
| 52 | expect(cellEnv({ port: 4100, dataDir: "/d/c", url: "https://kid--f.test" })["FORGE_PARENT_LOCAL"]).toBeUndefined(); | |
| 53 | }); | |
| 54 | ||
| 55 | test("the door answers the heartbeat route and nothing else, and ignores who the client claims to be", async () => { | |
| 56 | const parent = createApp({ dataDir: join(tmp, "door-parent"), url: "https://f.test" }); | |
| 57 | const door = openBeatDoor(parent); | |
| 58 | const at = (path: string, init?: RequestInit) => fetch(`http://127.0.0.1:${door.port}${path}`, init); | |
| 59 | try { | |
| 60 | for (const path of ["/", "/healthz", "/api/v1/forges", "/api/v1/heartbeat/x"]) { | |
| 61 | for (const method of ["GET", "POST"]) { | |
| 62 | const res = await at(path, { method, headers: { "x-forwarded-for": "6.6.6.6" } }); | |
| 63 | expect(res.status).toBe(404); | |
| 64 | expect(await res.text()).toBe("no such door"); | |
| 65 | } | |
| 66 | } | |
| 67 | expect((await at("/api/v1/heartbeat", { method: "GET" })).status).toBe(404); | |
| 68 | // An unsigned beat is refused alike, whoever it says it is. | |
| 69 | const beat = await at("/api/v1/heartbeat", { method: "POST", body: "{}", headers: { "x-forwarded-for": "6.6.6.6" } }); | |
| 70 | expect(beat.status).toBe(404); | |
| 71 | } finally { | |
| 72 | door.close(); | |
| 73 | parent.close(); | |
| 74 | } | |
| 75 | }); | |
| 76 | ||
| 77 | test("a heartbeat goes to the relayed port when one is named, and to the public address otherwise", async () => { | |
| 78 | const dataDir = join(tmp, "beat-child"); | |
| 79 | mkdirSync(dataDir, { recursive: true }); | |
| 80 | writeFileSync(join(dataDir, "parent.json"), JSON.stringify({ name: "kid", upstream: "https://f.test/", beatKey: "k" })); | |
| 81 | const child = createApp({ dataDir }); | |
| 82 | const urls: string[] = []; | |
| 83 | const fetcher = async (url: string) => (urls.push(url), new Response("{}")); | |
| 84 | try { | |
| 85 | await sendBeat(child, fetcher); | |
| 86 | process.env["FORGE_PARENT_LOCAL"] = "http://127.0.0.1:3000"; | |
| 87 | await sendBeat(child, fetcher); | |
| 88 | process.env["FORGE_PARENT_LOCAL"] = "http://evil.test"; | |
| 89 | await sendBeat(child, fetcher); | |
| 90 | expect(urls).toEqual(["https://f.test/api/v1/heartbeat", "http://127.0.0.1:3000/api/v1/heartbeat", "https://f.test/api/v1/heartbeat"]); | |
| 91 | } finally { | |
| 92 | delete process.env["FORGE_PARENT_LOCAL"]; | |
| 93 | child.close(); | |
| 94 | } | |
| 95 | }); | |
| 96 | }); | |
| 97 | ||
| 45 | 98 | describe("the placement seam", () => { |
| 46 | 99 | const app = { config: { dataDir: tmp, url: "https://f.test" } } as unknown as App; |
| 47 | 100 | const build = (cells: boolean) => { |
| @@ -108,6 +161,14 @@ describe("forwarding to a child", () => { | ||
| 108 | 161 | }); |
| 109 | 162 | |
| 110 | 163 | const cells = await cellsAvailable(); |
| 164 | // The check host sets FORGE_REQUIRE_CELLS: there a cell that cannot come up is a failure, never a skip. | |
| 165 | if (!cells.ok && process.env["FORGE_REQUIRE_CELLS"] === "1") { | |
| 166 | describe("the host holds cells", () => { | |
| 167 | test("cells are available", () => { | |
| 168 | throw new Error(`FORGE_REQUIRE_CELLS is set but this host refuses a cell: ${cells.reason}`); | |
| 169 | }); | |
| 170 | }); | |
| 171 | } | |
| 111 | 172 | describe("a real cell", () => { |
| 112 | 173 | test.skipIf(!cells.ok)("runs the child with the small limits and cannot see the parent's data", async () => { |
| 113 | 174 | const parent = join(tmp, "parent"); |
| @@ -180,6 +241,30 @@ describe("a real child forge", () => { | ||
| 180 | 241 | parent.close(); |
| 181 | 242 | } |
| 182 | 243 | }, 60_000); |
| 244 | ||
| 245 | test.skipIf(!cells.ok)("its heartbeat reaches the parent through the cell, with no public address", async () => { | |
| 246 | // The parent's public url is not reachable from here, as in a preview's cell. | |
| 247 | const parent = createApp({ dataDir: join(tmp, "beat-parent"), url: "https://unreachable.invalid" }); | |
| 248 | try { | |
| 249 | mkdirSync(join(parent.config.dataDir, "spawn"), { recursive: true }); | |
| 250 | writeFileSync(join(parent.config.dataDir, "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "cell-beat", done: true, beatKey: "beat-key" })); | |
| 251 | const dir = join(parent.config.dataDir, "children", "cell-beat"); | |
| 252 | mkdirSync(dir, { recursive: true }); | |
| 253 | writeFileSync(join(dir, "parent.json"), JSON.stringify({ name: "kid", upstream: "https://unreachable.invalid", beatKey: "beat-key" })); | |
| 254 | const children = cellChildren(parent); | |
| 255 | const child = await children.start({ bodyId: "cell-beat", name: "kid", size: "small" }); | |
| 256 | if (child === undefined) throw new Error(`the child did not start: ${children.failure("cell-beat")}`); | |
| 257 | let heard = await lastHeard(parent, "kid"); | |
| 258 | for (let i = 0; i < 100 && heard === undefined; i++) { | |
| 259 | await Bun.sleep(100); | |
| 260 | heard = await lastHeard(parent, "kid"); | |
| 261 | } | |
| 262 | expect(heard?.beat.name).toBe("kid"); | |
| 263 | } finally { | |
| 264 | await stopCellChildren(parent); | |
| 265 | parent.close(); | |
| 266 | } | |
| 267 | }, 60_000); | |
| 183 | 268 | }); |
| 184 | 269 | |
| 185 | 270 | describe("a child whose cell will not come up", () => { |
new above
offline
This reach entry opens the parent's full web port to the cell. Restrict it to the heartbeat route (dedicated listener or path filter) and make sure forwarded headers from it are not trusted.