Public: anyone can read this; the crew can change it.
A real-cell check: a stopped cell dies with a killed forge #11938
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/11936-a-real-cell-check-a-stopped-cell-dies-wi → main 4 commits fork 3 threads · 3 unresolved
test/cell-stopped-dies-with-forge.test.ts added +149
| @@ -0,0 +1,149 @@ | ||
| 1 | /** | |
| 2 | * A stopped cell dies with a killed forge (#11936). Cores are held by | |
| 3 | * SIGSTOP/SIGCONT; a forge killed while a tree is stopped relies on bwrap's | |
| 4 | * `--die-with-parent`: SIGKILL ends a stopped process, so the tree goes. | |
| 5 | * | |
| 6 | * A parent process (test/helpers/cell-parent.ts) opens a real cell; this | |
| 7 | * file stops the cell's tree, kills the parent, and counts what is left. The | |
| 8 | * tree is told apart by a sleep length only this run uses. | |
| 9 | */ | |
| 10 | ||
| 11 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 12 | import { readdirSync, readFileSync } from "node:fs"; | |
| 13 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 14 | import { tmpdir } from "node:os"; | |
| 15 | import { join } from "node:path"; | |
| 16 | import type { CellSpec } from "../src/apps/contain.ts"; | |
| 17 | import { freePort, runStep } from "../src/apps/process.ts"; | |
| 18 | import { portInsteadOfSocket } from "../src/git/hook-address.ts"; | |
| 19 | import { missing } from "./helpers/tools.ts"; | |
| 20 | ||
| 21 | const noCells = | |
| 22 | process.env["FORGE_CELLS"] === "off" || | |
| 23 | process.platform !== "linux" || | |
| 24 | portInsteadOfSocket() || | |
| 25 | missing("bwrap", "socat"); | |
| 26 | ||
| 27 | /** Live (not zombie) processes whose command line carries `mark`, with their state. */ | |
| 28 | function tree(mark: string, except = 0): { pid: number; state: string }[] { | |
| 29 | const out: { pid: number; state: string }[] = []; | |
| 30 | for (const name of readdirSync("/proc")) { | |
| 31 | if (!/^\d+$/.test(name)) continue; | |
| 32 | try { | |
| 33 | const stat = readFileSync(`/proc/${name}/stat`, "utf8"); | |
| 34 | const state = stat.slice(stat.lastIndexOf(")") + 2)[0]!; | |
| 35 | if (state === "Z" || state === "X") continue; | |
| 36 | const cmd = readFileSync(`/proc/${name}/cmdline`, "utf8").replace(/\0/g, " "); | |
| 37 | if (Number(name) !== except && cmd.includes(mark)) out.push({ pid: Number(name), state }); | |
| 38 | } catch { | |
| 39 | // gone mid-read | |
| 40 | } | |
| 41 | } | |
| 42 | return out; | |
| 43 | } | |
| 44 | ||
| 45 | async function until(done: () => boolean, ms: number, what: string): Promise<void> { | |
| 46 | const deadline = Date.now() + ms; | |
| 47 | while (!done()) { | |
| 48 | if (Date.now() > deadline) throw new Error(`never came: ${what}`); | |
| 49 | await Bun.sleep(50); // real-timer: polling /proc for a real process tree | |
| 50 | } | |
| 51 | } | |
| 52 | ||
| 53 | describe.skipIf(noCells)("cells: a stopped tree dies with its killed parent", () => { | |
| 54 | let dir: string; | |
| 55 | // Ephemeral binds: a port the machine just gave, not one guessed. | |
| 56 | const port = freePort(); | |
| 57 | const length = 600_000 + Math.floor(Math.random() * 99_999); | |
| 58 | let parentPid = 0; | |
| 59 | const mark = `sleep ${length}`; | |
| 60 | ||
| 61 | beforeAll(async () => { | |
| 62 | dir = await mkdtemp(join(tmpdir(), "forge-stopped-")); | |
| 63 | }); | |
| 64 | afterAll(async () => { | |
| 65 | for (const { pid } of tree(mark, parentPid)) | |
| 66 | try { | |
| 67 | process.kill(pid, "SIGKILL"); | |
| 68 | } catch { | |
| 69 | // gone | |
| 70 | } | |
| 71 | await rm(dir, { recursive: true, force: true }); | |
| 72 | }); | |
| 73 | ||
| 74 | test("SIGSTOP the tree, SIGKILL the parent: nothing survives, nothing stays in T", async () => { | |
| 75 | const parent = Bun.spawn( | |
| 76 | ["bun", join(import.meta.dir, "helpers", "cell-parent.ts"), dir, String(port), `${mark} & ${mark} & wait`], | |
| 77 | { stdin: "ignore", stdout: "pipe", stderr: "inherit" }, | |
| 78 | ); | |
| 79 | parentPid = parent.pid; | |
| 80 | try { | |
| 81 | await until(() => tree(mark, parentPid).length >= 3, 15_000, "the cell's tree"); | |
| 82 | const before = tree(mark, parentPid); | |
| 83 | // bwrap, the shell, and two sleeps. | |
| 84 | expect(before.length).toBeGreaterThanOrEqual(3); | |
| 85 | for (const { pid } of before) process.kill(pid, "SIGSTOP"); | |
| 86 | // Every process stopped, and none gone: an empty tree is not "stopped". | |
| 87 | const stopped = (): boolean => { | |
| 88 | const now = tree(mark, parentPid); | |
outdated
|
||
| 89 | return now.length >= before.length && now.every((p) => p.state === "T"); | |
| 90 | }; | |
| 91 | await until(stopped, 5_000, "the tree stopped"); | |
| 92 | expect(stopped()).toBe(true); | |
| 93 | ||
| 94 | parent.kill("SIGKILL"); | |
| 95 | await parent.exited; | |
| 96 | await until(() => tree(mark, parentPid).length === 0, 10_000, "the tree gone"); | |
| 97 | expect(tree(mark, parentPid)).toEqual([]); | |
| 98 | } finally { | |
| 99 | parent.kill("SIGKILL"); | |
| 100 | } | |
| 101 | }, 40_000); | |
| 102 | ||
| 103 | const step = (command: string, lines: string[], listen: number) => | |
| 104 | runStep("fresh", command, { | |
| 105 | cwd: dir, | |
| 106 | env: { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, | |
| 107 | log: (l) => lines.push(l), | |
| 108 | signal: new AbortController().signal, | |
| 109 | timeoutMs: 20_000, | |
| 110 | cell: { | |
| 111 | dir, | |
| 112 | writable: [], | |
| 113 | home: join(dir, "home"), | |
| 114 | hide: [], | |
| 115 | egress: [], | |
| 116 | reach: [], | |
| 117 | limits: { memoryMiB: 256, cpus: 1, diskMiB: 64 }, | |
| 118 | listen, | |
| 119 | } satisfies CellSpec, | |
outdated
Vacuous. In |
||
| 120 | }); | |
| 121 | ||
| 122 | test("a fresh cell after that starts unpaused: its shell is not in T", async () => { | |
| 123 | const lines: string[] = []; | |
| 124 | // The shell reads its own stat (`$$`), not a child's. | |
| 125 | const result = await step("sleep 0.2; read -r _ _ s _ < /proc/$$/stat; echo state=$s", lines, freePort()); | |
| 126 | expect(result.ok).toBe(true); | |
| 127 | const state = lines.join("\n").match(/state=(\w)/)?.[1]; | |
| 128 | expect(state).toBeDefined(); | |
| 129 | expect(state).not.toBe("T"); | |
| 130 | }, 30_000); | |
| 131 | ||
| 132 | test("a cell stopped and continued runs on to the end", async () => { | |
| 133 | const run = `sleep 4.${length}`; | |
| 134 | const lines: string[] = []; | |
| 135 | const done = step(`${run}; echo finished`, lines, freePort()); | |
| 136 | await until(() => tree(run).length >= 2, 15_000, "the second cell's tree"); | |
| 137 | const held = tree(run); | |
| 138 | for (const { pid } of held) process.kill(pid, "SIGSTOP"); | |
| 139 | const stopped = (): boolean => { | |
| 140 | const now = tree(run); | |
| 141 | return now.length >= held.length && now.every((p) => p.state === "T"); | |
| 142 | }; | |
| 143 | await until(stopped, 5_000, "the second tree stopped"); | |
| 144 | for (const { pid } of held) process.kill(pid, "SIGCONT"); | |
| 145 | const result = await done; | |
| 146 | expect(result.ok).toBe(true); | |
| 147 | expect(lines.some((l) => l.includes("finished"))).toBe(true); | |
| 148 | }, 40_000); | |
| 149 | }); |
test/helpers/cell-parent.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | /** | |
| 2 | * A stand-in for the forge in test/cell-stopped-dies-with-forge.test.ts: it | |
| 3 | * opens a real cell, runs `argv[3]` in it, and waits to be killed. Arguments: | |
| 4 | * a scratch directory, a loopback port, the command. | |
| 5 | */ | |
| 6 | import { join } from "node:path"; | |
| 7 | import { openCell } from "../../src/apps/contain.ts"; | |
| 8 | ||
| 9 | const [dir, port, command] = process.argv.slice(2) as [string, string, string]; | |
| 10 | const cell = openCell({ | |
| 11 | dir: dir!, | |
| 12 | writable: [], | |
| 13 | home: join(dir!, "home"), | |
| 14 | hide: [], | |
| 15 | egress: [], | |
| 16 | reach: [], | |
| 17 | limits: { memoryMiB: 256, cpus: 1, diskMiB: 64 }, | |
| 18 | listen: Number(port), | |
| 19 | }); | |
| 20 | Bun.spawn(cell.argv(command!), { | |
| 21 | cwd: dir!, | |
| 22 | env: cell.env({ PATH: process.env["PATH"] ?? "/usr/bin:/bin" }), | |
| 23 | stdin: "ignore", | |
| 24 | stdout: "ignore", | |
| 25 | stderr: "ignore", | |
| 26 | }); | |
| 27 | console.log("opened"); | |
| 28 | setInterval(() => {}, 1000); |
The port is random in 40000–60000 and shared by both tests (
port,port+1) with no collision check. Take it from an ephemeral bind instead, so the test cannot flake on a busy port.