Public: anyone can read this; the crew can change it.

Stopping a forge takes 15 s, and a request during those 15 s starts a second forge on the same database #13824

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13814-stopping-a-forge-takes-15-s-and-a-reques → main 2 commits fork 2 threads · 2 unresolved

  • Merged. scout/13814-stopping-a-forge-takes-15-s-and-a-reques was merged into main as d7c1cd6.

Review

Sign in to review this change.
2 files changed +129 −22
src/spawn/cell.ts +76 −22
@@ -31,7 +31,8 @@ import { childGate, type ChildGate } from "./child_reach.ts";
3131 export const CELL_PREFIX = "cell-";
3232 export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX);
3333
34 // Read off the serving thread in production (`child_db.ts`); here for the tests that read a file directly.
34 // Read off the serving thread in production (`child_db.ts`); here for the tests that read a file
35 // directly.
3536 export { crewAtWork, nextRunDue } from "./child_runs.ts";
3637 import { childCrewAtWork, closeChildDb } from "./child_db.ts";
3738
@@ -92,23 +93,38 @@ export function cellLimits(size: Size): Limits {
9293 export function cellSpecFor(input: {
9394 dir: string;
9495 parentDataDir: string;
95 /** The parent's own home, where its crew's sessions and model login live; `HOME` unless a test names one. */
96 /**
97 * The parent's own home, where its crew's sessions and model login live; `HOME` unless a test
98 * names one.
99 */
96100 parentHome?: string;
97101 parentUrl: string;
98 /** The child's own public address, which its pages and checks call back on; the parent answers it. */
102 /**
103 * The child's own public address, which its pages and checks call back on; the parent answers it.
104 */
99105 url?: string;
100 /** The parent's heartbeat door (`beat-door.ts`): the only way to it that needs no public address (a preview's cell has none). */
106 /**
107 * The parent's heartbeat door (`beat-door.ts`): the only way to it that needs no public address
108 * (a preview's cell has none).
109 */
101110 parentPort?: number;
102111 size: Size;
103112 port: number;
104113 label: string;
105114 /** The child's name: its cell's hostname. */
106115 name?: string;
107 /** The model provider hosts an admin let children reach (`children.modelEgress`); none by default. */
116 /**
117 * The model provider hosts an admin let children reach (`children.modelEgress`); none by default.
118 */
108119 modelHosts?: readonly string[];
109 /** The package registry an admin let children reach (`children.registryEgress`); none by default. */
120 /**
121 * The package registry an admin let children reach (`children.registryEgress`); none by default.
122 */
110123 registryHosts?: readonly string[];
111 /** The payment hosts children may reach (`children.paymentEgress`): a ceiling only an app declaring the host passes. */
124 /**
125 * The payment hosts children may reach (`children.paymentEgress`): a ceiling only an app
126 * declaring the host passes.
127 */
112128 paymentHosts?: readonly string[];
113129 /**
114130 * The git hosts and neighbouring forges children may reach (`children.originEgress`), so an
@@ -123,9 +139,15 @@ export function cellSpecFor(input: {
123139 * what it always was.
124140 */
125141 disk?: ChildDisk;
126 /** A checked copy of the child's own forge code, when it runs that (`own_code.ts`); the parent's code is revealed either way, for its dependencies. */
142 /**
143 * A checked copy of the child's own forge code, when it runs that (`own_code.ts`); the parent's
144 * code is revealed either way, for its dependencies.
145 */
127146 ownRoot?: string;
128 /** What an admin allowed this child since, asked on every connection, and its refusals said (`child_reach.ts`): a grant needs no restart. */
147 /**
148 * What an admin allowed this child since, asked on every connection, and its refusals said
149 * (`child_reach.ts`): a grant needs no restart.
150 */
129151 gate?: ChildGate;
130152 }): CellSpec {
131153 return {
@@ -175,6 +197,8 @@ export function cellEnv(input: { port: number; dataDir: string; url: string; par
175197 FORGE_APP_URL: input.url,
176198 FORGE_APP_DATA: input.dataDir,
177199 NODE_ENV: "production",
200 // Nothing routes to a child's port but its parent: it leaves at once on SIGTERM (#13814).
201 FORGE_DRAIN_LINGER_S: "0",
178202 ...(input.size === undefined ? {} : { [SIZE_ENV]: input.size }),
179203 ...(input.cgroupDir === undefined ? {} : { [CGROUP_DIR_ENV]: input.cgroupDir }),
180204 ...(input.parentPort === undefined ? {} : { FORGE_PARENT_LOCAL: parentLocal(input.parentPort) }),
@@ -188,9 +212,9 @@ export function cellEnv(input: { port: number; dataDir: string; url: string; par
188212 }
189213
190214 /**
191 * What a child is started with: `cellEnv`, plus the parent's runner (`FORGE_AGENT_RUNNER`, or the older `FORGE_CLAUDE`)
192 * when it has one. The child's crew reads it (`main.ts`), so a parent run with
193 * Claude Code somewhere off `PATH` has children that find it too.
215 * What a child is started with: `cellEnv`, plus the parent's runner (`FORGE_AGENT_RUNNER`, or the
216 * older `FORGE_CLAUDE`) when it has one. The child's crew reads it (`main.ts`), so a parent run
217 * with Claude Code somewhere off `PATH` has children that find it too.
194218 */
195219 export function childEnvFor(input: Parameters<typeof cellEnv>[0], parentEnv: Record<string, string | undefined> = process.env): Record<string, string> {
196220 const claude = namedRunner(parentEnv);
@@ -226,11 +250,17 @@ export interface CellChildren {
226250 */
227251 forgetFailure(bodyId: string): void;
228252 list(): readonly CellChild[];
229 /** Mark a request under way for this child; the returned function ends it. A child with one is not put to sleep. */
253 /**
254 * Mark a request under way for this child; the returned function ends it. A child with one is not
255 * put to sleep.
256 */
230257 hold(bodyId: string): () => void;
231258 /** Note that a request just reached this child, so it is not idle. */
232259 touch(bodyId: string, now?: number): void;
233 /** Stop every running child quiet for `idleMs`; the names of those stopped (their data stays on disk). */
260 /**
261 * Stop every running child quiet for `idleMs`; the names of those stopped (their data stays on
262 * disk).
263 */
234264 sleepIdle(idleMs: number, now?: number): Promise<string[]>;
235265 }
236266
@@ -242,7 +272,9 @@ export function cellChildren(parent: App): CellChildren {
242272 return r;
243273 }
244274
245 /** Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not. */
275 /**
276 * Starts a child and resolves once it answers; undefined (with `fail` told why) when it does not.
277 */
246278 export type Launcher = (input: { bodyId: string; name: string; size: Size }, fail: (why: string) => void) => Promise<CellChild | undefined>;
247279
248280 /** Probes a starting child gets, 100 ms apart: about a minute for a slow first boot. */
@@ -323,6 +355,10 @@ export function launchSpecFor(
323355 export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
324356 const held = new Map<string, CellChild>();
325357 const starting = new Map<string, Promise<CellChild | undefined>>();
358 /**
359 * A body told to go and not yet gone: a start for it waits for the exit, never runs beside it.
360 */
361 const stopping = new Map<string, Promise<void>>();
326362 /** When a start last failed: a broken child is not relaunched on every request. */
327363 const failed = new Map<string, { at: number; why: string }>();
328364 /** Held once it answers, never before: a start in flight is in `starting`. */
@@ -388,7 +424,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
388424 let gone = false;
389425 void exited.then(() => (gone = true));
390426 const up = await firstAnswer(child, port, () => {
391 // What it printed last goes to this forge's log and no further: the reason kept is the parent's own words.
427 // What it printed last goes to this forge's log and no further: the reason kept is the
428 // parent's own words.
392429 why = startFailure(cell.breach(), gone);
393430 parent.log.warn("child forge did not answer", { name: input.name, why, said: (recent.slice(-3).join(" | ") || "no output").slice(0, 400) });
394431 });
@@ -408,7 +445,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
408445 fail(cells.reason);
409446 return undefined;
410447 }
411 // The parent decides what the cell runs: its own code, the last that came up, or the parent's (`own_code.ts`).
448 // The parent decides what the cell runs: its own code, the last that came up, or the parent's
449 // (`own_code.ts`).
412450 const started = await oneAtATime(parent, input.name, async () => {
413451 const sources = await sourcesFor(parent, { name: input.name, childDir: childDirOf(parent, input.bodyId) }, { main: MAIN, root: CODE });
414452 return startFirstThatWorks(parent, input.name, sources, (source) => startOn(input, source));
@@ -436,6 +474,8 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
436474
437475 return {
438476 start(input) {
477 const leaving = stopping.get(input.bodyId);

this.start(input) depends on the call being a method call; a destructured const { start } = registry would break it. Safer to name the object and call registry.start, or hoist the function.

478 if (leaving !== undefined) return leaving.then(() => this.start(input));
439479 const there = held.get(input.bodyId);
440480 if (there !== undefined) return Promise.resolve(there);
441481 const lastFailure = failed.get(input.bodyId);
@@ -459,11 +499,15 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
459499 return flight;
460500 },
461501 async stop(bodyId) {
502 const leaving = stopping.get(bodyId);
503 if (leaving !== undefined) return leaving;
462504 const child = held.get(bodyId);
463505 if (child === undefined) return;
464506 held.delete(bodyId);
465507 startedAt.delete(bodyId);
466 await stopGently(child);
508 const gone = stopGently(child).finally(() => stopping.delete(bodyId));
509 stopping.set(bodyId, gone);
510 await gone;
467511 },
468512 get: (bodyId) => held.get(bodyId),
469513 failure: (bodyId) => failed.get(bodyId)?.why,
@@ -499,10 +543,16 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
499543 };
500544 }
501545
502 /** Lines a child forge may write to its parent's log in a minute; past it they are counted, not kept. */
546 /**
547 * Lines a child forge may write to its parent's log in a minute; past it they are counted, not
548 * kept.
549 */
503550 export const CHILD_LOG_PER_MINUTE = 600;
504551
505 /** A child's line with no control characters: it cannot move a terminal's cursor or paint over the parent's lines. */
552 /**
553 * A child's line with no control characters: it cannot move a terminal's cursor or paint over the
554 * parent's lines.
555 */
506556 export const plainLine = (line: string): string => line.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f]/g, "?");
507557
508558 /**
@@ -556,7 +606,8 @@ export async function stopCellChildren(parent: App): Promise<void> {
556606 if (r === undefined) return;
557607 registries.delete(parent);
558608 await Promise.all(r.list().map((c) => r.stop(c.bodyId)));
559 // The workers that ask the children's databases go with them; one is started again when next asked.
609 // The workers that ask the children's databases go with them; one is started again when next
610 // asked.
560611 closeChildDb();
561612 }
562613
@@ -581,7 +632,10 @@ export function cellPlacement(parent: App, store: BodyStore, maxChildren: () =>
581632
582633 // ---- forwarding ----
583634
584 /** Headers that say who the client is: the child believes them from its relay, so only the forge writes them. */
635 /**
636 * Headers that say who the client is: the child believes them from its relay, so only the forge
637 * writes them.
638 */
585639 const WHO = new Set(["x-forwarded-for", "forwarded", "x-real-ip", "cf-connecting-ip", "true-client-ip", "x-forwarded-host", "x-forwarded-proto"]);
586640
587641 const HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "content-length"]);
test/spawn-cell-stopping.test.ts added +53
@@ -0,0 +1,53 @@
1 import { afterAll, beforeAll, expect, test } from "bun:test";
2 import { mkdtemp, rm } from "node:fs/promises";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { createApp, type App } from "../src/app.ts";
6 import { cellEnv, cellLimits, makeRegistry } from "../src/spawn/cell.ts";
7
8 let dir = "";
9 let app: App;
10 beforeAll(async () => {
11 dir = await mkdtemp(join(tmpdir(), "forge-stopping-"));
12 app = createApp({ dataDir: dir, port: 0 });
13 });
14 afterAll(async () => {
15 app.close();
16 await rm(dir, { recursive: true, force: true });
17 });
18
19 test("a cell child is told not to linger: it leaves on SIGTERM at once", () => {

This only checks the env value. The 0 s exit itself is covered by main.ts's own tests, but note that a bad value would make the child exit at boot (main.ts:247), so this constant matters.

20 expect(cellEnv({ port: 4100, dataDir: "/d/c", url: "https://kid--f.test" })["FORGE_DRAIN_LINGER_S"]).toBe("0");
21 });
22
23 test("a request while a child is stopping waits for its exit: one process holds the directory", async () => {
24 let live = 0;
25 let most = 0;
26 let launched = 0;
27 const registry = makeRegistry(app, async (input) => {
28 // A child that takes 400 ms to leave once told to.
29 const proc = Bun.spawn(["bash", "-c", "trap 'sleep 0.4; exit 0' TERM; while :; do sleep 0.05; done"], { stdout: "ignore", stderr: "ignore" });
30 launched++;
31 live++;
32 most = Math.max(most, live);
33 const exited = proc.exited.then((code) => {
34 live--;
35 return code;
36 });
37 return { ...input, port: 1, limits: cellLimits(input.size), pid: proc.pid, exited };
38 });
39 const input = { bodyId: "cell-probe", name: "probe", size: "small" as const };
40 const first = await registry.start(input);
41 expect(first).toBeDefined();
42 await Bun.sleep(200); // real-timer: a real bash child must install its TERM trap before we signal it
43 const stopping = registry.stop(input.bodyId);
44 const again = registry.start(input);
45 await stopping;
46 const second = await again;
47 expect(launched).toBe(2);
48 expect(most).toBe(1);
49 expect(second).toBeDefined();
50 expect(second?.pid).not.toBe(first?.pid);
51 await registry.stop(input.bodyId);
52 expect(live).toBe(0);
53 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs