Public: anyone can read this; the crew can change it.

A cancelled run leaves no process of its own alive #13151

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13123-a-cancelled-run-leaves-no-process-of-its → main 2 commits fork

  • Merged. mason/13123-a-cancelled-run-leaves-no-process-of-its was merged into main as c3a658c.

Review

Sign in to review this change.
3 files changed +252 −8
src/crew/openai_runner.ts +18 −8
@@ -19,6 +19,7 @@
1919 import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
2020 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
2121 import type { AgentDetection, AgentOutcome, AgentRun, AgentRunner } from "./agent_runner.ts";
22 import { stopGroup, type GroupStop } from "./stop_group.ts";
2223
2324 export const MAX_TURNS = 60;
2425 const OUTPUT_CAP = 30_000;
@@ -118,25 +119,34 @@ export async function readCapped(stream: ReadableStream<Uint8Array> | null, max:
118119 return { text: new TextDecoder().decode(bytes), more };
119120 }
120121
121 /** Run `argv` in `root`: killed at `ms`, on abort, or once its output passes the cap. */
122 export async function runShell(argv: string[], root: string, env: Record<string, string>, signal: AbortSignal, ms = BASH_MS): Promise<string> {
123 const proc = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
122 /**
123 * Run `argv` in `root`: stopped at `ms`, on abort, or once its output passes
124 * the cap. It runs in a process group of its own, and stopping it ends the
125 * whole group (`stopGroup`: SIGTERM, SIGKILL after the grace), so a child
126 * it forked, and that child's own, go with it; the call returns only once
127 * they have.
128 */
129 export async function runShell(argv: string[], root: string, env: Record<string, string>, signal: AbortSignal, ms = BASH_MS, graceMs?: number): Promise<string> {
130 const proc = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true });
131 let stopped: Promise<GroupStop> | undefined;
132 const stop = (): void => { stopped ??= stopGroup(proc.pid, graceMs); };
124133 let timedOut = false;
125 const timer = setTimeout(() => { timedOut = true; proc.kill(9); }, ms);
126 const onAbort = () => proc.kill(9);
127 signal.addEventListener("abort", onAbort);
134 const timer = setTimeout(() => { timedOut = true; stop(); }, ms);
135 signal.addEventListener("abort", stop);
136 if (signal.aborted) stop();
128137 try {
129138 // A grandchild can hold the pipes open after the shell is gone:
130139 // give them a moment past its exit, then stop reading.
131140 const gone = proc.exited.then(() => new Promise<void>((r) => setTimeout(r, 200)));
132141 const [out, err] = await Promise.all([readCapped(proc.stdout, OUTPUT_CAP, gone), readCapped(proc.stderr, OUTPUT_CAP, gone)]);
133 if (out.more || err.more) proc.kill(9);
142 if (out.more || err.more) stop();
134143 const code = await proc.exited;
144 await stopped;
135145 const tail = timedOut ? `\n[timed out after ${ms} ms]` : out.more || err.more ? "\n[truncated]" : code === 0 ? "" : `\n[exit ${code}]`;
136146 return cap(`${out.text}${err.text}`) + tail;
137147 } finally {
138148 clearTimeout(timer);
139 signal.removeEventListener("abort", onAbort);
149 signal.removeEventListener("abort", stop);
140150 }
141151 }
142152
src/crew/stop_group.ts added +56
@@ -0,0 +1,56 @@
1 /**
2 * Ends a process group a crew run started (#13123): SIGTERM to the whole
3 * group, so a command can flush and clean up; then, once `graceMs` has run
4 * out with any member still alive, SIGKILL to the group. Resolves once the
5 * group is empty (or a short while after the kill, should the kernel be
6 * slow to clear it), so the caller knows nothing it started is still
7 * working or spending.
8 *
9 * Only ever a negative pid: the group the leader was started in with
10 * `detached: true`. Never a pid outside it, and never this process's own.
11 */
12 import { CANCEL_GRACE_MS } from "../runs/grace.ts";
13
14 /** True while any process is still in group `pgid`. */
15 export function groupAlive(pgid: number): boolean {
16 try {
17 process.kill(-pgid, 0);
18 return true;
19 } catch {
20 return false;
21 }
22 }
23
24 function signal(pgid: number, sig: "SIGTERM" | "SIGKILL"): void {
25 try {
26 process.kill(-pgid, sig);
27 } catch {
28 // the group is empty: nothing to signal
29 }
30 }
31
32 async function emptyBy(pgid: number, deadline: number): Promise<boolean> {
33 while (Date.now() < deadline) {
34 if (!groupAlive(pgid)) return true;
35 await Bun.sleep(20);
36 }
37 return !groupAlive(pgid);
38 }
39
40 /** What `stopGroup` did: `killed` when the grace ran out and SIGKILL was sent. */
41 export interface GroupStop {
42 readonly killed: boolean;
43 }
44
45 /**
46 * SIGTERM group `pgid`, wait up to `graceMs` for it to empty, then SIGKILL
47 * whatever is left and wait (at most a second) for that to go too.
48 */
49 export async function stopGroup(pgid: number, graceMs: number = CANCEL_GRACE_MS): Promise<GroupStop> {
50 if (!Number.isInteger(pgid) || pgid <= 1 || pgid === process.pid) return { killed: false };
51 signal(pgid, "SIGTERM");
52 if (await emptyBy(pgid, Date.now() + graceMs)) return { killed: false };
53 signal(pgid, "SIGKILL");
54 await emptyBy(pgid, Date.now() + 1_000);
55 return { killed: true };
56 }
test/crew-cancel-shell.test.ts added +178
@@ -0,0 +1,178 @@
1 /**
2 * A cancelled crew run leaves no process of its own alive (#13123).
3 *
4 * The OpenAI runner's Bash tool (`runShell`) starts its command in a group
5 * of its own and a cancel ends that whole group: SIGTERM, then SIGKILL
6 * once the grace runs out, and the call returns only after the group is
7 * empty. Before, a cancel SIGKILLed the shell alone, and the child it had
8 * forked, and that child's own, ran on.
9 *
10 * The agent supervisor (`runClaude`) escalates the same way: an agent whose
11 * tree ignores SIGTERM is still alive halfway through `CANCEL_GRACE_MS` and
12 * gone soon after it.
13 *
14 * Every test spawns its own tree; nothing here signals a process it did
15 * not start.
16 */
17 import { afterAll, describe, expect, test } from "bun:test";
18 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
19 import { mkdtemp, rm } from "node:fs/promises";
20 import { tmpdir } from "node:os";
21 import { join } from "node:path";
22 import { runShell } from "../src/crew/openai_runner.ts";
23 import { runClaude } from "../src/crew/claude.ts";
24 import { sandboxPolicy } from "../src/crew/sandbox.ts";
25 import { stopGroup } from "../src/crew/stop_group.ts";
26 import { CANCEL_GRACE_MS } from "../src/runs/grace.ts";
27
28 const dir = await mkdtemp(join(tmpdir(), "forge-cancel-shell-"));
29 const started: number[] = [];
30 afterAll(async () => {
31 for (const pid of started)
32 try {
33 process.kill(pid, "SIGKILL");
34 } catch {
35 // gone, as it should be
36 }
37 await rm(dir, { recursive: true, force: true });
38 });
39
40 /** Alive and not a zombie waiting for its new parent to reap it. */
41 function alive(pid: number): boolean {
42 try {
43 process.kill(pid, 0);
44 } catch {
45 return false;
46 }
47 try {
48 const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
49 const state = stat.slice(stat.lastIndexOf(")") + 2)[0];
50 return state !== "Z" && state !== "X";
51 } catch {
52 return false;
53 }
54 }
55
56 async function waitFor(file: string, ms: number): Promise<number> {
57 const deadline = Date.now() + ms;
58 while (Date.now() < deadline) {
59 if (existsSync(file)) {
60 const pid = Number(readFileSync(file, "utf8").trim());
61 if (Number.isInteger(pid) && pid > 1) return pid;
62 }
63 await Bun.sleep(20); // real-timer: polling for a pid file a real child process writes
64 }
65 throw new Error(`${file} never named a pid`);
66 }
67
68 let n = 0;
69 /** A shell that forks a child that forks a sleeping grandchild; pids land in files. */
70 function tree(ignoreTerm: boolean): { argv: string[]; files: string[] } {
71 n += 1;
72 const shell = join(dir, `shell-${n}.pid`);
73 const child = join(dir, `child-${n}.pid`);
74 const grand = join(dir, `grand-${n}.pid`);
75 const trap = ignoreTerm ? "trap '' TERM; " : "";
76 const script = `${trap}echo $$ > ${shell}; sh -c 'sleep 300 & echo $$ > ${child}; echo $! > ${grand}.tmp; mv ${grand}.tmp ${grand}; wait' & wait`;
77 return { argv: ["/bin/sh", "-c", script], files: [shell, child, grand] };
78 }
79
80 async function pidsOf(files: readonly string[]): Promise<number[]> {
81 const grand = await waitFor(files[2]!, 10_000);
82 const pids = [await waitFor(files[0]!, 1_000), await waitFor(files[1]!, 1_000), grand];
83 started.push(...pids);
84 return pids;
85 }
86
87 const GRACE = 600;
88
89 describe.skipIf(process.platform !== "linux")("a cancelled Bash tool call", () => {
90 test("ends the shell, its child and its grandchild on SIGTERM, and returns once they are gone", async () => {
91 const { argv, files } = tree(false);
92 const abort = new AbortController();
93 const call = runShell(argv, dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE);
94 const pids = await pidsOf(files);
95 expect(pids.every(alive)).toBe(true);
96 const at = Date.now();
97 abort.abort();
98 await call;
99 // Honoured SIGTERM: ended well inside the grace, every one of them.
100 expect(Date.now() - at).toBeLessThan(GRACE);
101 expect(pids.filter(alive)).toEqual([]);
102 }, 20_000);
103
104 test("escalates to SIGKILL for a tree that ignores SIGTERM once the grace runs out", async () => {
105 const { argv, files } = tree(true);
106 const abort = new AbortController();
107 let returned = false;
108 const call = runShell(argv, dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE).then((out) => {
109 returned = true;
110 return out;
111 });
112 const pids = await pidsOf(files);
113 abort.abort();
114 await Bun.sleep(GRACE / 2); // real-timer: the real process group must still be alive halfway through runShell's real grace
115 // Asked, not yet told: alive halfway through the grace, and the call waits.
116 expect(pids.every(alive)).toBe(true);
117 expect(returned).toBe(false);
118 await call;
119 expect(pids.filter(alive)).toEqual([]);
120 }, 20_000);
121
122 test("a signal aborted before the call starts stops the command at once", async () => {
123 const abort = new AbortController();
124 abort.abort();
125 const at = Date.now();
126 await runShell(["/bin/sh", "-c", "sleep 30"], dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE);
127 expect(Date.now() - at).toBeLessThan(GRACE);
128 }, 20_000);
129 });
130
131 describe("stopGroup", () => {
132 test("never signals this process's group, init's, or a pid that is not one", async () => {
133 for (const pgid of [process.pid, 1, 0, -5, 1.5]) expect(await stopGroup(pgid, 10)).toEqual({ killed: false });
134 });
135 });
136
137 describe.skipIf(process.platform !== "linux")("a cancelled agent whose tree ignores SIGTERM", () => {
138 test("is alive halfway through the grace and gone, child and grandchild too, soon after it", async () => {
139 const binary = join(dir, "agent");
140 const childFile = join(dir, "agent-child.pid");
141 const grandFile = join(dir, "agent-grand.pid");
142 const agentFile = join(dir, "agent-self.pid");
143 writeFileSync(
144 binary,
145 `#!/bin/sh
146 trap '' TERM
147 IFS= read -r first
148 echo $$ > ${agentFile}
149 sh -c 'sleep 300 & echo $$ > ${childFile}; echo $! > ${grandFile}.tmp; mv ${grandFile}.tmp ${grandFile}; wait' &
150 echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}'
151 sleep 300
152 `,
153 );
154 chmodSync(binary, 0o755);
155 const abort = new AbortController();
156 const run = runClaude({
157 binary,
158 cwd: dir,
159 prompt: "Do the thing.",
160 appendSystemPrompt: "",
161 budgetUsd: 1,
162 env: {},
163 sandbox: sandboxPolicy({ dataDir: dir, agentName: "bot", urls: [] }),
164 timeoutMs: 60_000,
165 log: () => {},
166 signal: abort.signal,
167 settleMs: 1_000,
168 });
169 const pids = await pidsOf([agentFile, childFile, grandFile]);
170 abort.abort();
171 await Bun.sleep(CANCEL_GRACE_MS / 2); // real-timer: the real agent tree must still be alive halfway through the supervisor's real grace
172 expect(pids.every(alive)).toBe(true);
173 const deadline = Date.now() + CANCEL_GRACE_MS / 2 + 1_500;
174 while (pids.some(alive) && Date.now() < deadline) await Bun.sleep(20); // real-timer: polling until the kernel has ended the real processes after SIGKILL
175 expect(pids.filter(alive)).toEqual([]);
176 expect((await run).reason).toBe("cancelled");
177 }, 20_000);
178 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs