Public: anyone can read this; the crew can change it.
Stopping a forge takes 15 s, and a request during those 15 s starts a second forge on the same database #13824
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13814-stopping-a-forge-takes-15-s-and-a-reques → main 2 commits fork 2 threads · 2 unresolved
2 files
changed +129
−22
src/spawn/cell.ts +76 −22
| @@ -31,7 +31,8 @@ import { childGate, type ChildGate } from "./child_reach.ts"; | ||
| 31 | 31 | export const CELL_PREFIX = "cell-"; |
| 32 | 32 | export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX); |
| 33 | 33 | |
| 34 | // Read off the serving thread in production (`child_db.ts`); here for the tests that read a file directly. | |
| 34 | // Read off the serving thread in production (`child_db.ts`); here for the tests that read a file | |
| 35 | // directly. | |
| 35 | 36 | export { crewAtWork, nextRunDue } from "./child_runs.ts"; |
| 36 | 37 | import { childCrewAtWork, closeChildDb } from "./child_db.ts"; |
| 37 | 38 | |
| @@ -92,23 +93,38 @@ export function cellLimits(size: Size): Limits { | ||
| 92 | 93 | export function cellSpecFor(input: { |
| 93 | 94 | dir: string; |
| 94 | 95 | parentDataDir: string; |
| 95 | /** The parent's own home, where its crew's sessions and model login live; `HOME` unless a test names one. */ | |
| 96 | /** | |
| 97 | * The parent's own home, where its crew's sessions and model login live; `HOME` unless a test | |
| 98 | * names one. | |
| 99 | */ | |
| 96 | 100 | parentHome?: string; |
| 97 | 101 | parentUrl: string; |
| 98 | /** The child's own public address, which its pages and checks call back on; the parent answers it. */ | |
| 102 | /** | |
| 103 | * The child's own public address, which its pages and checks call back on; the parent answers it. | |
| 104 | */ | |
| 99 | 105 | url?: string; |
| 100 | /** The parent's heartbeat door (`beat-door.ts`): the only way to it that needs no public address (a preview's cell has none). */ | |
| 106 | /** | |
| 107 | * The parent's heartbeat door (`beat-door.ts`): the only way to it that needs no public address | |
| 108 | * (a preview's cell has none). | |
| 109 | */ | |
| 101 | 110 | parentPort?: number; |
| 102 | 111 | size: Size; |
| 103 | 112 | port: number; |
| 104 | 113 | label: string; |
| 105 | 114 | /** The child's name: its cell's hostname. */ |
| 106 | 115 | name?: string; |
| 107 | /** The model provider hosts an admin let children reach (`children.modelEgress`); none by default. */ | |
| 116 | /** | |
| 117 | * The model provider hosts an admin let children reach (`children.modelEgress`); none by default. | |
| 118 | */ | |
| 108 | 119 | modelHosts?: readonly string[]; |
| 109 | /** The package registry an admin let children reach (`children.registryEgress`); none by default. */ | |
| 120 | /** | |
| 121 | * The package registry an admin let children reach (`children.registryEgress`); none by default. | |
| 122 | */ | |
| 110 | 123 | registryHosts?: readonly string[]; |
| 111 | /** The payment hosts children may reach (`children.paymentEgress`): a ceiling only an app declaring the host passes. */ | |
| 124 | /** | |
| 125 | * The payment hosts children may reach (`children.paymentEgress`): a ceiling only an app | |
| 126 | * declaring the host passes. | |
| 127 | */ | |
| 112 | 128 | paymentHosts?: readonly string[]; |
| 113 | 129 | /** |
| 114 | 130 | * The git hosts and neighbouring forges children may reach (`children.originEgress`), so an |
| @@ -123,9 +139,15 @@ export function cellSpecFor(input: { | ||
| 123 | 139 | * what it always was. |
| 124 | 140 | */ |
| 125 | 141 | disk?: ChildDisk; |
| 126 | /** A checked copy of the child's own forge code, when it runs that (`own_code.ts`); the parent's code is revealed either way, for its dependencies. */ | |
| 142 | /** | |
| 143 | * A checked copy of the child's own forge code, when it runs that (`own_code.ts`); the parent's | |
| 144 | * code is revealed either way, for its dependencies. | |
| 145 | */ | |
| 127 | 146 | ownRoot?: string; |
| 128 | /** What an admin allowed this child since, asked on every connection, and its refusals said (`child_reach.ts`): a grant needs no restart. */ | |
| 147 | /** | |
| 148 | * What an admin allowed this child since, asked on every connection, and its refusals said | |
| 149 | * (`child_reach.ts`): a grant needs no restart. | |
| 150 | */ | |
| 129 | 151 | gate?: ChildGate; |
| 130 | 152 | }): CellSpec { |
| 131 | 153 | return { |
| @@ -175,6 +197,8 @@ export function cellEnv(input: { port: number; dataDir: string; url: string; par | ||
| 175 | 197 | FORGE_APP_URL: input.url, |
| 176 | 198 | FORGE_APP_DATA: input.dataDir, |
| 177 | 199 | NODE_ENV: "production", |
| 200 | // Nothing routes to a child's port but its parent: it leaves at once on SIGTERM (#13814). | |
| 201 | FORGE_DRAIN_LINGER_S: "0", | |
| 178 | 202 | ...(input.size === undefined ? {} : { [SIZE_ENV]: input.size }), |
| 179 | 203 | ...(input.cgroupDir === undefined ? {} : { [CGROUP_DIR_ENV]: input.cgroupDir }), |
| 180 | 204 | ...(input.parentPort === undefined ? {} : { FORGE_PARENT_LOCAL: parentLocal(input.parentPort) }), |
| @@ -188,9 +212,9 @@ export function cellEnv(input: { port: number; dataDir: string; url: string; par | ||
| 188 | 212 | } |
| 189 | 213 | |
| 190 | 214 | /** |
| 191 | * What a child is started with: `cellEnv`, plus the parent's runner (`FORGE_AGENT_RUNNER`, or the older `FORGE_CLAUDE`) | |
| 192 | * when it has one. The child's crew reads it (`main.ts`), so a parent run with | |
| 193 | * Claude Code somewhere off `PATH` has children that find it too. | |
| 215 | * What a child is started with: `cellEnv`, plus the parent's runner (`FORGE_AGENT_RUNNER`, or the | |
| 216 | * older `FORGE_CLAUDE`) when it has one. The child's crew reads it (`main.ts`), so a parent run | |
| 217 | * with Claude Code somewhere off `PATH` has children that find it too. | |
| 194 | 218 | */ |
| 195 | 219 | export function childEnvFor(input: Parameters<typeof cellEnv>[0], parentEnv: Record<string, string | undefined> = process.env): Record<string, string> { |
| 196 | 220 | const claude = namedRunner(parentEnv); |
| @@ -226,11 +250,17 @@ export interface CellChildren { | ||
| 226 | 250 | */ |
| 227 | 251 | forgetFailure(bodyId: string): void; |
| 228 | 252 | list(): readonly CellChild[]; |
| 229 | /** Mark a request under way for this child; the returned function ends it. A child with one is not put to sleep. */ | |
| 253 | /** | |
| 254 | * Mark a request under way for this child; the returned function ends it. A child with one is not | |
| 255 | * put to sleep. | |
| 256 | */ | |
| 230 | 257 | hold(bodyId: string): () => void; |
| 231 | 258 | /** Note that a request just reached this child, so it is not idle. */ |
| 232 | 259 | touch(bodyId: string, now?: number): void; |
| 233 | /** Stop every running child quiet for `idleMs`; the names of those stopped (their data stays on disk). */ | |
| 260 | /** | |
| 261 | * Stop every running child quiet for `idleMs`; the names of those stopped (their data stays on | |
| 262 | * disk). | |
| 263 | */ | |
| 234 | 264 | sleepIdle(idleMs: number, now?: number): Promise<string[]>; |
| 235 | 265 | } |
| 236 | 266 | |
| @@ -242,7 +272,9 @@ export function cellChildren(parent: App): CellChildren { | ||
| 242 | 272 | return r; |
| 243 | 273 | } |
| 244 | 274 | |
| 245 | /** Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not. */ | |
| 275 | /** | |
| 276 | * Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not. | |
| 277 | */ | |
| 246 | 278 | export type Launcher = (input: { bodyId: string; name: string; size: Size }, fail: (why: string) => void) => Promise<CellChild | undefined>; |
| 247 | 279 | |
| 248 | 280 | /** Probes a starting child gets, 100 ms apart: about a minute for a slow first boot. */ |
| @@ -323,6 +355,10 @@ export function launchSpecFor( | ||
| 323 | 355 | export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { |
| 324 | 356 | const held = new Map<string, CellChild>(); |
| 325 | 357 | const starting = new Map<string, Promise<CellChild | undefined>>(); |
| 358 | /** | |
| 359 | * A body told to go and not yet gone: a start for it waits for the exit, never runs beside it. | |
| 360 | */ | |
| 361 | const stopping = new Map<string, Promise<void>>(); | |
| 326 | 362 | /** When a start last failed: a broken child is not relaunched on every request. */ |
| 327 | 363 | const failed = new Map<string, { at: number; why: string }>(); |
| 328 | 364 | /** Held once it answers, never before: a start in flight is in `starting`. */ |
| @@ -388,7 +424,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 388 | 424 | let gone = false; |
| 389 | 425 | void exited.then(() => (gone = true)); |
| 390 | 426 | const up = await firstAnswer(child, port, () => { |
| 391 | // What it printed last goes to this forge's log and no further: the reason kept is the parent's own words. | |
| 427 | // What it printed last goes to this forge's log and no further: the reason kept is the | |
| 428 | // parent's own words. | |
| 392 | 429 | why = startFailure(cell.breach(), gone); |
| 393 | 430 | parent.log.warn("child forge did not answer", { name: input.name, why, said: (recent.slice(-3).join(" | ") || "no output").slice(0, 400) }); |
| 394 | 431 | }); |
| @@ -408,7 +445,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 408 | 445 | fail(cells.reason); |
| 409 | 446 | return undefined; |
| 410 | 447 | } |
| 411 | // The parent decides what the cell runs: its own code, the last that came up, or the parent's (`own_code.ts`). | |
| 448 | // The parent decides what the cell runs: its own code, the last that came up, or the parent's | |
| 449 | // (`own_code.ts`). | |
| 412 | 450 | const started = await oneAtATime(parent, input.name, async () => { |
| 413 | 451 | const sources = await sourcesFor(parent, { name: input.name, childDir: childDirOf(parent, input.bodyId) }, { main: MAIN, root: CODE }); |
| 414 | 452 | return startFirstThatWorks(parent, input.name, sources, (source) => startOn(input, source)); |
| @@ -436,6 +474,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 436 | 474 | |
| 437 | 475 | return { |
| 438 | 476 | start(input) { |
| 477 | const leaving = stopping.get(input.bodyId); | |
| 478 | if (leaving !== undefined) return leaving.then(() => this.start(input)); | |
| 439 | 479 | const there = held.get(input.bodyId); |
| 440 | 480 | if (there !== undefined) return Promise.resolve(there); |
| 441 | 481 | const lastFailure = failed.get(input.bodyId); |
| @@ -459,11 +499,15 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 459 | 499 | return flight; |
| 460 | 500 | }, |
| 461 | 501 | async stop(bodyId) { |
| 502 | const leaving = stopping.get(bodyId); | |
| 503 | if (leaving !== undefined) return leaving; | |
| 462 | 504 | const child = held.get(bodyId); |
| 463 | 505 | if (child === undefined) return; |
| 464 | 506 | held.delete(bodyId); |
| 465 | 507 | startedAt.delete(bodyId); |
| 466 | await stopGently(child); | |
| 508 | const gone = stopGently(child).finally(() => stopping.delete(bodyId)); | |
| 509 | stopping.set(bodyId, gone); | |
| 510 | await gone; | |
| 467 | 511 | }, |
| 468 | 512 | get: (bodyId) => held.get(bodyId), |
| 469 | 513 | failure: (bodyId) => failed.get(bodyId)?.why, |
| @@ -499,10 +543,16 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 499 | 543 | }; |
| 500 | 544 | } |
| 501 | 545 | |
| 502 | /** Lines a child forge may write to its parent's log in a minute; past it they are counted, not kept. */ | |
| 546 | /** | |
| 547 | * Lines a child forge may write to its parent's log in a minute; past it they are counted, not | |
| 548 | * kept. | |
| 549 | */ | |
| 503 | 550 | export const CHILD_LOG_PER_MINUTE = 600; |
| 504 | 551 | |
| 505 | /** A child's line with no control characters: it cannot move a terminal's cursor or paint over the parent's lines. */ | |
| 552 | /** | |
| 553 | * A child's line with no control characters: it cannot move a terminal's cursor or paint over the | |
| 554 | * parent's lines. | |
| 555 | */ | |
| 506 | 556 | export const plainLine = (line: string): string => line.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f]/g, "?"); |
| 507 | 557 | |
| 508 | 558 | /** |
| @@ -556,7 +606,8 @@ export async function stopCellChildren(parent: App): Promise<void> { | ||
| 556 | 606 | if (r === undefined) return; |
| 557 | 607 | registries.delete(parent); |
| 558 | 608 | await Promise.all(r.list().map((c) => r.stop(c.bodyId))); |
| 559 | // The workers that ask the children's databases go with them; one is started again when next asked. | |
| 609 | // The workers that ask the children's databases go with them; one is started again when next | |
| 610 | // asked. | |
| 560 | 611 | closeChildDb(); |
| 561 | 612 | } |
| 562 | 613 | |
| @@ -581,7 +632,10 @@ export function cellPlacement(parent: App, store: BodyStore, maxChildren: () => | ||
| 581 | 632 | |
| 582 | 633 | // ---- forwarding ---- |
| 583 | 634 | |
| 584 | /** Headers that say who the client is: the child believes them from its relay, so only the forge writes them. */ | |
| 635 | /** | |
| 636 | * Headers that say who the client is: the child believes them from its relay, so only the forge | |
| 637 | * writes them. | |
| 638 | */ | |
| 585 | 639 | const WHO = new Set(["x-forwarded-for", "forwarded", "x-real-ip", "cf-connecting-ip", "true-client-ip", "x-forwarded-host", "x-forwarded-proto"]); |
| 586 | 640 | |
| 587 | 641 | const HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "content-length"]); |
test/spawn-cell-stopping.test.ts added +53
| @@ -0,0 +1,53 @@ | ||
| 1 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp, type App } from "../src/app.ts"; | |
| 6 | import { cellEnv, cellLimits, makeRegistry } from "../src/spawn/cell.ts"; | |
| 7 | ||
| 8 | let dir = ""; | |
| 9 | let app: App; | |
| 10 | beforeAll(async () => { | |
| 11 | dir = await mkdtemp(join(tmpdir(), "forge-stopping-")); | |
| 12 | app = createApp({ dataDir: dir, port: 0 }); | |
| 13 | }); | |
| 14 | afterAll(async () => { | |
| 15 | app.close(); | |
| 16 | await rm(dir, { recursive: true, force: true }); | |
| 17 | }); | |
| 18 | ||
| 19 | test("a cell child is told not to linger: it leaves on SIGTERM at once", () => { | |
This only checks the env value. The 0 s exit itself is covered by main.ts's own tests, but note that a bad value would make the child exit at boot (main.ts:247), so this constant matters. |
||
| 20 | expect(cellEnv({ port: 4100, dataDir: "/d/c", url: "https://kid--f.test" })["FORGE_DRAIN_LINGER_S"]).toBe("0"); | |
| 21 | }); | |
| 22 | ||
| 23 | test("a request while a child is stopping waits for its exit: one process holds the directory", async () => { | |
| 24 | let live = 0; | |
| 25 | let most = 0; | |
| 26 | let launched = 0; | |
| 27 | const registry = makeRegistry(app, async (input) => { | |
| 28 | // A child that takes 400 ms to leave once told to. | |
| 29 | const proc = Bun.spawn(["bash", "-c", "trap 'sleep 0.4; exit 0' TERM; while :; do sleep 0.05; done"], { stdout: "ignore", stderr: "ignore" }); | |
| 30 | launched++; | |
| 31 | live++; | |
| 32 | most = Math.max(most, live); | |
| 33 | const exited = proc.exited.then((code) => { | |
| 34 | live--; | |
| 35 | return code; | |
| 36 | }); | |
| 37 | return { ...input, port: 1, limits: cellLimits(input.size), pid: proc.pid, exited }; | |
| 38 | }); | |
| 39 | const input = { bodyId: "cell-probe", name: "probe", size: "small" as const }; | |
| 40 | const first = await registry.start(input); | |
| 41 | expect(first).toBeDefined(); | |
| 42 | await Bun.sleep(200); // real-timer: a real bash child must install its TERM trap before we signal it | |
| 43 | const stopping = registry.stop(input.bodyId); | |
| 44 | const again = registry.start(input); | |
| 45 | await stopping; | |
| 46 | const second = await again; | |
| 47 | expect(launched).toBe(2); | |
| 48 | expect(most).toBe(1); | |
| 49 | expect(second).toBeDefined(); | |
| 50 | expect(second?.pid).not.toBe(first?.pid); | |
| 51 | await registry.stop(input.bodyId); | |
| 52 | expect(live).toBe(0); | |
| 53 | }); |
new above
offline
this.start(input)depends on the call being a method call; a destructuredconst { start } = registrywould break it. Safer to name the object and callregistry.start, or hoist the function.