Public: anyone can read this; the crew can change it.

A production release does not cut a hosted forge's work: its checks and runs survive, or resume without losing their place #14200

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13750-a-production-release-does-not-cut-a-host → main 2 commits fork

  • Merged. scout/13750-a-production-release-does-not-cut-a-host was merged into main as bf26317.

Review

Sign in to review this change.
4 files changed +66 −10
docs/features/cells-outlive.md +2 −2
@@ -1,6 +1,6 @@
1 # Cells outlive the host (`FORGE_CELLS_OUTLIVE=on`)
1 # Cells outlive the host (`FORGE_CELLS_OUTLIVE`)
22
3 Off by default. With it on, a child's cell runs its proxy and relays in a supervisor of its own and is not tied to this process: stopping the host leaves the cell running (#13755).
3 On when the host boots (#13750), so a production release leaves a hosted forge's checks and runs running; set `FORGE_CELLS_OUTLIVE=off` to tie cells to the host again. With it on, a child's cell runs its proxy and relays in a supervisor of its own and is not tied to this process: stopping the host leaves the cell running (#13755).
44
55 **On start the host takes its cells back (#13756).** Each supervised cell writes `children/<body>/cell.json` (pid, port, supervisor control directory, start time, name, size) as soon as it is spawned, and clears it when it exits. At boot the host reads every record and, for each:
66
src/main.ts +5 −1
@@ -35,7 +35,7 @@ import { syncAllPillars } from "./actions/pillars.ts";
3535 import { bootHostedChildren, childAwake } from "./spawn/host.ts";
3636 import { announceSandboxed } from "./spawn/sandboxed.ts";
3737 import { keepSandboxing } from "./spawn/move_cell.ts";
38 import { stopCellChildren } from "./spawn/cell.ts";
38 import { applyBootDefaults, stopCellChildren } from "./spawn/cell.ts";
3939 import { backfillChildren } from "./spawn/forge_code_backfill.ts";
4040 import { firstBoot } from "./spawn/first-boot.ts";
4141 import { previewIdentityKey } from "./spawn/preview-owner.ts";
@@ -203,6 +203,10 @@ if (process.argv[2] === "--first-admin") {
203203 process.exit(answer.exit);
204204 }
205205
206 // Every boot, hosted or not: child cells outlive a restart of this process unless
207 // FORGE_CELLS_OUTLIVE=off (#13750).
208 applyBootDefaults(process.env);
209
206210 // A forge can be an app of a forge (docs/APPS.md, "The forge as an app of
207211 // itself"). The host's `FORGE_APP_*` say where this one lives and what it
208212 // answers to; `FORGE_URL` there names the host, not this forge. Hosted, the
src/spawn/cell.ts +24 −6
@@ -33,15 +33,33 @@ import { clearRecord, leaveRecord, readRecords, type CellRecord } from "./cell_r
3333 /**
3434 * Whether a child's cell outlives this process (`FORGE_CELLS_OUTLIVE=on`; #13755): its proxy and
3535 * relays run in a supervisor of their own, the cell is not tied to its parent, and a stop of this
36 * process leaves it running. Off by default (docs/features/cells-outlive.md): each cell leaves a
37 * record (`cell_record.ts`) that the next host adopts if the cell is still its supervisor's and
38 * answers, and stops if it does not answer (#13756); its beat door (`beat-door.ts`) is this
39 * process's, reached through the supervisor's relay while this process lives and answered by the
40 * supervisor's held door after (`held_door.ts`, #13906), and its egress bytes are summed in the
41 * supervisor and reach this process's gauges as events (#13922).
36 * process leaves it running. Off here by default, on at boot unless `off` is set (`outliveSetting`;
37 * docs/features/cells-outlive.md): each cell leaves a record (`cell_record.ts`) that the next host
38 * adopts if the cell is still its supervisor's and answers, and stops if it does not answer
39 * (#13756); its beat door (`beat-door.ts`) is this process's, reached through the supervisor's
40 * relay while this process lives and answered by the supervisor's held door after (`held_door.ts`,
41 * #13906), and its egress bytes are summed in the supervisor and reach this process's gauges as
42 * events (#13922).
4243 */
4344 export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on";
4445
46 /**
47 * The setting the booted host runs with (#13750): cells outlive it unless `FORGE_CELLS_OUTLIVE=off`
48 * is set, so a production release leaves a hosted forge's checks and runs running. Only `main.ts`
49 * applies it; the function itself still reads `on` alone, so a test or tool that imports this
50 * module keeps the old behaviour.
51 */
52 export const outliveSetting = (env: Record<string, string | undefined>): "on" | "off" => (env["FORGE_CELLS_OUTLIVE"] === "off" ? "off" : "on");
53
54 /**
55 * What boot sets in the environment before anything reads it (called by `main.ts`; every boot,
56 * hosted or not, since any host may hold cells). Mutates and returns `env`.
57 */
58 export const applyBootDefaults = (env: Record<string, string | undefined>): Record<string, string | undefined> => {
59 env["FORGE_CELLS_OUTLIVE"] = outliveSetting(env);
60 return env;
61 };
62
4563 export const CELL_PREFIX = "cell-";
4664 export const ownsBody = (bodyId: string): boolean => bodyId.startsWith(CELL_PREFIX);
4765
test/cell-outlives-host.test.ts +35 −1
@@ -11,7 +11,7 @@ import { proxyOptions } from "../src/apps/cell_policy.ts";
1111 import { createApp } from "../src/app.ts";
1212 import { freePort } from "../src/apps/process.ts";
1313 import { openHeldDoor } from "../src/spawn/held_door.ts";
14 import { cellChildren, cellLimits, cellSpecFor, cellsOutlive, spawnInCell, stopCellChildren } from "../src/spawn/cell.ts";
14 import { cellChildren, cellLimits, cellSpecFor, applyBootDefaults, cellsOutlive, outliveSetting, spawnInCell, stopCellChildren } from "../src/spawn/cell.ts";
1515 import { alive, doorTarget, files, parseConfig, parseEvent, plainLoops, readAllowed, tailLines, type Supervised } from "../src/spawn/supervisor.ts";
1616
1717 const killed: number[] = [];
@@ -508,3 +508,37 @@ test("a door in the config is a port, or the config is refused", () => {
508508 expect(parseConfig(JSON.stringify({ ...good, door: "8080" }))).toBeUndefined();
509509 expect(parseConfig(JSON.stringify({ ...good, doorAt: 70_000 }))).toBeUndefined();
510510 });
511
512 test("a booted host lets cells outlive it unless told off (#13750)", () => {
513 expect(outliveSetting({})).toBe("on");
514 expect(outliveSetting({ FORGE_CELLS_OUTLIVE: "on" })).toBe("on");
515 expect(outliveSetting({ FORGE_CELLS_OUTLIVE: "off" })).toBe("off");
516 expect(outliveSetting({ FORGE_CELLS_OUTLIVE: "x" })).toBe("on");
517 expect(cellsOutlive({})).toBe(false); // the module itself is unchanged for importers
518 });
519
520 test("boot defaults make an unset env outlive and a spawned cell supervised; off is kept (#13750)", () => {
521 const booted = applyBootDefaults({});
522 expect(booted["FORGE_CELLS_OUTLIVE"]).toBe("on");
523 expect(applyBootDefaults({ FORGE_CELLS_OUTLIVE: "off" })["FORGE_CELLS_OUTLIVE"]).toBe("off");
524 const before = process.env["FORGE_CELLS_OUTLIVE"];
525 try {
526 delete process.env["FORGE_CELLS_OUTLIVE"];
527 applyBootDefaults(process.env);
528 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
529 expect(spec(dir).outlive).toBe(true);
530 process.env["FORGE_CELLS_OUTLIVE"] = "off";
531 applyBootDefaults(process.env);
532 expect(spec(dir).outlive).toBeUndefined();
533 } finally {
534 if (before === undefined) delete process.env["FORGE_CELLS_OUTLIVE"];
535 else process.env["FORGE_CELLS_OUTLIVE"] = before;
536 }
537 });
538
539 test("main.ts applies the boot defaults before reading its environment", () => {
540 const main = readFileSync(join(import.meta.dir, "../src/main.ts"), "utf8");
541 const at = main.indexOf("applyBootDefaults(process.env)");
542 expect(at).toBeGreaterThan(0);
543 expect(at).toBeLessThan(main.indexOf("const hostedAt"));
544 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs