Public: anyone can read this; the crew can change it.

A real-cell check: a stopped cell dies with a killed forge #11938

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/11936-a-real-cell-check-a-stopped-cell-dies-wi → main 4 commits fork 3 threads · 3 unresolved

  • Merged. scout/11936-a-real-cell-check-a-stopped-cell-dies-wi was merged into main as acb1be7.

Review

Sign in to review this change.
2 files changed +177 −0
test/cell-stopped-dies-with-forge.test.ts added +149
@@ -0,0 +1,149 @@
1 /**
2 * A stopped cell dies with a killed forge (#11936). Cores are held by
3 * SIGSTOP/SIGCONT; a forge killed while a tree is stopped relies on bwrap's
4 * `--die-with-parent`: SIGKILL ends a stopped process, so the tree goes.
5 *
6 * A parent process (test/helpers/cell-parent.ts) opens a real cell; this
7 * file stops the cell's tree, kills the parent, and counts what is left. The
8 * tree is told apart by a sleep length only this run uses.
9 */
10
11 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
12 import { readdirSync, readFileSync } from "node:fs";
13 import { mkdtemp, rm } from "node:fs/promises";
14 import { tmpdir } from "node:os";
15 import { join } from "node:path";
16 import type { CellSpec } from "../src/apps/contain.ts";
17 import { freePort, runStep } from "../src/apps/process.ts";
18 import { portInsteadOfSocket } from "../src/git/hook-address.ts";
19 import { missing } from "./helpers/tools.ts";
20
21 const noCells =
22 process.env["FORGE_CELLS"] === "off" ||
23 process.platform !== "linux" ||
24 portInsteadOfSocket() ||
25 missing("bwrap", "socat");
26
27 /** Live (not zombie) processes whose command line carries `mark`, with their state. */
28 function tree(mark: string, except = 0): { pid: number; state: string }[] {
29 const out: { pid: number; state: string }[] = [];
30 for (const name of readdirSync("/proc")) {
31 if (!/^\d+$/.test(name)) continue;
32 try {
33 const stat = readFileSync(`/proc/${name}/stat`, "utf8");
34 const state = stat.slice(stat.lastIndexOf(")") + 2)[0]!;
35 if (state === "Z" || state === "X") continue;
36 const cmd = readFileSync(`/proc/${name}/cmdline`, "utf8").replace(/\0/g, " ");
37 if (Number(name) !== except && cmd.includes(mark)) out.push({ pid: Number(name), state });
38 } catch {
39 // gone mid-read
40 }
41 }
42 return out;
43 }
44
45 async function until(done: () => boolean, ms: number, what: string): Promise<void> {
46 const deadline = Date.now() + ms;
47 while (!done()) {
48 if (Date.now() > deadline) throw new Error(`never came: ${what}`);
49 await Bun.sleep(50); // real-timer: polling /proc for a real process tree
50 }
51 }
52
53 describe.skipIf(noCells)("cells: a stopped tree dies with its killed parent", () => {
54 let dir: string;
55 // Ephemeral binds: a port the machine just gave, not one guessed.
56 const port = freePort();
57 const length = 600_000 + Math.floor(Math.random() * 99_999);
58 let parentPid = 0;
59 const mark = `sleep ${length}`;
60
61 beforeAll(async () => {
62 dir = await mkdtemp(join(tmpdir(), "forge-stopped-"));
63 });
64 afterAll(async () => {
outdated

The port is random in 40000–60000 and shared by both tests (port, port+1) with no collision check. Take it from an ephemeral bind instead, so the test cannot flake on a busy port.

65 for (const { pid } of tree(mark, parentPid))
66 try {
67 process.kill(pid, "SIGKILL");
68 } catch {
69 // gone
70 }
71 await rm(dir, { recursive: true, force: true });
72 });
73
74 test("SIGSTOP the tree, SIGKILL the parent: nothing survives, nothing stays in T", async () => {
75 const parent = Bun.spawn(
76 ["bun", join(import.meta.dir, "helpers", "cell-parent.ts"), dir, String(port), `${mark} & ${mark} & wait`],
77 { stdin: "ignore", stdout: "pipe", stderr: "inherit" },
78 );
79 parentPid = parent.pid;
80 try {
81 await until(() => tree(mark, parentPid).length >= 3, 15_000, "the cell's tree");
82 const before = tree(mark, parentPid);
83 // bwrap, the shell, and two sleeps.
84 expect(before.length).toBeGreaterThanOrEqual(3);
85 for (const { pid } of before) process.kill(pid, "SIGSTOP");
86 // Every process stopped, and none gone: an empty tree is not "stopped".
87 const stopped = (): boolean => {
88 const now = tree(mark, parentPid);
outdated

every(... === "T") is true on an empty list, so this wait passes if the tree vanished before it was stopped. Require tree(...).length >= before.length as well. Then the "gone" assertion cannot pass without the kill having mattered.

89 return now.length >= before.length && now.every((p) => p.state === "T");
90 };
91 await until(stopped, 5_000, "the tree stopped");
92 expect(stopped()).toBe(true);
93
94 parent.kill("SIGKILL");
95 await parent.exited;
96 await until(() => tree(mark, parentPid).length === 0, 10_000, "the tree gone");
97 expect(tree(mark, parentPid)).toEqual([]);
98 } finally {
99 parent.kill("SIGKILL");
100 }
101 }, 40_000);
102
103 const step = (command: string, lines: string[], listen: number) =>
104 runStep("fresh", command, {
105 cwd: dir,
106 env: { PATH: process.env["PATH"] ?? "/usr/bin:/bin" },
107 log: (l) => lines.push(l),
108 signal: new AbortController().signal,
109 timeoutMs: 20_000,
110 cell: {
111 dir,
112 writable: [],
113 home: join(dir, "home"),
114 hide: [],
115 egress: [],
116 reach: [],
117 limits: { memoryMiB: 256, cpus: 1, diskMiB: 64 },
118 listen,
119 } satisfies CellSpec,
outdated

Vacuous. In echo state=$(cut -d" " -f3 /proc/self/stat), /proc/self is the cut process, which is always R, so this passes even if the cell started stopped. And a stopped step would never print, so result.ok already covers it. Fix: read the shell's own state with read -r _ _ s _ < /proc/$$/stat, or drop the test. Better: make it prove something, e.g. the cell is stopped by the test and the step still completes after SIGCONT.

120 });
121
122 test("a fresh cell after that starts unpaused: its shell is not in T", async () => {
123 const lines: string[] = [];
124 // The shell reads its own stat (`$$`), not a child's.
125 const result = await step("sleep 0.2; read -r _ _ s _ < /proc/$$/stat; echo state=$s", lines, freePort());
126 expect(result.ok).toBe(true);
127 const state = lines.join("\n").match(/state=(\w)/)?.[1];
128 expect(state).toBeDefined();
129 expect(state).not.toBe("T");
130 }, 30_000);
131
132 test("a cell stopped and continued runs on to the end", async () => {
133 const run = `sleep 4.${length}`;
134 const lines: string[] = [];
135 const done = step(`${run}; echo finished`, lines, freePort());
136 await until(() => tree(run).length >= 2, 15_000, "the second cell's tree");
137 const held = tree(run);
138 for (const { pid } of held) process.kill(pid, "SIGSTOP");
139 const stopped = (): boolean => {
140 const now = tree(run);
141 return now.length >= held.length && now.every((p) => p.state === "T");
142 };
143 await until(stopped, 5_000, "the second tree stopped");
144 for (const { pid } of held) process.kill(pid, "SIGCONT");
145 const result = await done;
146 expect(result.ok).toBe(true);
147 expect(lines.some((l) => l.includes("finished"))).toBe(true);
148 }, 40_000);
149 });
test/helpers/cell-parent.ts added +28
@@ -0,0 +1,28 @@
1 /**
2 * A stand-in for the forge in test/cell-stopped-dies-with-forge.test.ts: it
3 * opens a real cell, runs `argv[3]` in it, and waits to be killed. Arguments:
4 * a scratch directory, a loopback port, the command.
5 */
6 import { join } from "node:path";
7 import { openCell } from "../../src/apps/contain.ts";
8
9 const [dir, port, command] = process.argv.slice(2) as [string, string, string];
10 const cell = openCell({
11 dir: dir!,
12 writable: [],
13 home: join(dir!, "home"),
14 hide: [],
15 egress: [],
16 reach: [],
17 limits: { memoryMiB: 256, cpus: 1, diskMiB: 64 },
18 listen: Number(port),
19 });
20 Bun.spawn(cell.argv(command!), {
21 cwd: dir!,
22 env: cell.env({ PATH: process.env["PATH"] ?? "/usr/bin:/bin" }),
23 stdin: "ignore",
24 stdout: "ignore",
25 stderr: "ignore",
26 });
27 console.log("opened");
28 setInterval(() => {}, 1000);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs