Public: anyone can read this; the crew can change it.
A cancelled run leaves no process of its own alive #13151
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13123-a-cancelled-run-leaves-no-process-of-its → main 2 commits fork
3 files
changed +252
−8
src/crew/openai_runner.ts +18 −8
| @@ -19,6 +19,7 @@ | ||
| 19 | 19 | import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; |
| 20 | 20 | import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; |
| 21 | 21 | import type { AgentDetection, AgentOutcome, AgentRun, AgentRunner } from "./agent_runner.ts"; |
| 22 | import { stopGroup, type GroupStop } from "./stop_group.ts"; | |
| 22 | 23 | |
| 23 | 24 | export const MAX_TURNS = 60; |
| 24 | 25 | const OUTPUT_CAP = 30_000; |
| @@ -118,25 +119,34 @@ export async function readCapped(stream: ReadableStream<Uint8Array> | null, max: | ||
| 118 | 119 | return { text: new TextDecoder().decode(bytes), more }; |
| 119 | 120 | } |
| 120 | 121 | |
| 121 | /** Run `argv` in `root`: killed at `ms`, on abort, or once its output passes the cap. */ | |
| 122 | export async function runShell(argv: string[], root: string, env: Record<string, string>, signal: AbortSignal, ms = BASH_MS): Promise<string> { | |
| 123 | const proc = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }); | |
| 122 | /** | |
| 123 | * Run `argv` in `root`: stopped at `ms`, on abort, or once its output passes | |
| 124 | * the cap. It runs in a process group of its own, and stopping it ends the | |
| 125 | * whole group (`stopGroup`: SIGTERM, SIGKILL after the grace), so a child | |
| 126 | * it forked, and that child's own, go with it; the call returns only once | |
| 127 | * they have. | |
| 128 | */ | |
| 129 | export async function runShell(argv: string[], root: string, env: Record<string, string>, signal: AbortSignal, ms = BASH_MS, graceMs?: number): Promise<string> { | |
| 130 | const proc = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe", detached: true }); | |
| 131 | let stopped: Promise<GroupStop> | undefined; | |
| 132 | const stop = (): void => { stopped ??= stopGroup(proc.pid, graceMs); }; | |
| 124 | 133 | let timedOut = false; |
| 125 | const timer = setTimeout(() => { timedOut = true; proc.kill(9); }, ms); | |
| 126 | const onAbort = () => proc.kill(9); | |
| 127 | signal.addEventListener("abort", onAbort); | |
| 134 | const timer = setTimeout(() => { timedOut = true; stop(); }, ms); | |
| 135 | signal.addEventListener("abort", stop); | |
| 136 | if (signal.aborted) stop(); | |
| 128 | 137 | try { |
| 129 | 138 | // A grandchild can hold the pipes open after the shell is gone: |
| 130 | 139 | // give them a moment past its exit, then stop reading. |
| 131 | 140 | const gone = proc.exited.then(() => new Promise<void>((r) => setTimeout(r, 200))); |
| 132 | 141 | const [out, err] = await Promise.all([readCapped(proc.stdout, OUTPUT_CAP, gone), readCapped(proc.stderr, OUTPUT_CAP, gone)]); |
| 133 | if (out.more || err.more) proc.kill(9); | |
| 142 | if (out.more || err.more) stop(); | |
| 134 | 143 | const code = await proc.exited; |
| 144 | await stopped; | |
| 135 | 145 | const tail = timedOut ? `\n[timed out after ${ms} ms]` : out.more || err.more ? "\n[truncated]" : code === 0 ? "" : `\n[exit ${code}]`; |
| 136 | 146 | return cap(`${out.text}${err.text}`) + tail; |
| 137 | 147 | } finally { |
| 138 | 148 | clearTimeout(timer); |
| 139 | signal.removeEventListener("abort", onAbort); | |
| 149 | signal.removeEventListener("abort", stop); | |
| 140 | 150 | } |
| 141 | 151 | } |
| 142 | 152 |
src/crew/stop_group.ts added +56
| @@ -0,0 +1,56 @@ | ||
| 1 | /** | |
| 2 | * Ends a process group a crew run started (#13123): SIGTERM to the whole | |
| 3 | * group, so a command can flush and clean up; then, once `graceMs` has run | |
| 4 | * out with any member still alive, SIGKILL to the group. Resolves once the | |
| 5 | * group is empty (or a short while after the kill, should the kernel be | |
| 6 | * slow to clear it), so the caller knows nothing it started is still | |
| 7 | * working or spending. | |
| 8 | * | |
| 9 | * Only ever a negative pid: the group the leader was started in with | |
| 10 | * `detached: true`. Never a pid outside it, and never this process's own. | |
| 11 | */ | |
| 12 | import { CANCEL_GRACE_MS } from "../runs/grace.ts"; | |
| 13 | ||
| 14 | /** True while any process is still in group `pgid`. */ | |
| 15 | export function groupAlive(pgid: number): boolean { | |
| 16 | try { | |
| 17 | process.kill(-pgid, 0); | |
| 18 | return true; | |
| 19 | } catch { | |
| 20 | return false; | |
| 21 | } | |
| 22 | } | |
| 23 | ||
| 24 | function signal(pgid: number, sig: "SIGTERM" | "SIGKILL"): void { | |
| 25 | try { | |
| 26 | process.kill(-pgid, sig); | |
| 27 | } catch { | |
| 28 | // the group is empty: nothing to signal | |
| 29 | } | |
| 30 | } | |
| 31 | ||
| 32 | async function emptyBy(pgid: number, deadline: number): Promise<boolean> { | |
| 33 | while (Date.now() < deadline) { | |
| 34 | if (!groupAlive(pgid)) return true; | |
| 35 | await Bun.sleep(20); | |
| 36 | } | |
| 37 | return !groupAlive(pgid); | |
| 38 | } | |
| 39 | ||
| 40 | /** What `stopGroup` did: `killed` when the grace ran out and SIGKILL was sent. */ | |
| 41 | export interface GroupStop { | |
| 42 | readonly killed: boolean; | |
| 43 | } | |
| 44 | ||
| 45 | /** | |
| 46 | * SIGTERM group `pgid`, wait up to `graceMs` for it to empty, then SIGKILL | |
| 47 | * whatever is left and wait (at most a second) for that to go too. | |
| 48 | */ | |
| 49 | export async function stopGroup(pgid: number, graceMs: number = CANCEL_GRACE_MS): Promise<GroupStop> { | |
| 50 | if (!Number.isInteger(pgid) || pgid <= 1 || pgid === process.pid) return { killed: false }; | |
| 51 | signal(pgid, "SIGTERM"); | |
| 52 | if (await emptyBy(pgid, Date.now() + graceMs)) return { killed: false }; | |
| 53 | signal(pgid, "SIGKILL"); | |
| 54 | await emptyBy(pgid, Date.now() + 1_000); | |
| 55 | return { killed: true }; | |
| 56 | } |
test/crew-cancel-shell.test.ts added +178
| @@ -0,0 +1,178 @@ | ||
| 1 | /** | |
| 2 | * A cancelled crew run leaves no process of its own alive (#13123). | |
| 3 | * | |
| 4 | * The OpenAI runner's Bash tool (`runShell`) starts its command in a group | |
| 5 | * of its own and a cancel ends that whole group: SIGTERM, then SIGKILL | |
| 6 | * once the grace runs out, and the call returns only after the group is | |
| 7 | * empty. Before, a cancel SIGKILLed the shell alone, and the child it had | |
| 8 | * forked, and that child's own, ran on. | |
| 9 | * | |
| 10 | * The agent supervisor (`runClaude`) escalates the same way: an agent whose | |
| 11 | * tree ignores SIGTERM is still alive halfway through `CANCEL_GRACE_MS` and | |
| 12 | * gone soon after it. | |
| 13 | * | |
| 14 | * Every test spawns its own tree; nothing here signals a process it did | |
| 15 | * not start. | |
| 16 | */ | |
| 17 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 18 | import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; | |
| 19 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 20 | import { tmpdir } from "node:os"; | |
| 21 | import { join } from "node:path"; | |
| 22 | import { runShell } from "../src/crew/openai_runner.ts"; | |
| 23 | import { runClaude } from "../src/crew/claude.ts"; | |
| 24 | import { sandboxPolicy } from "../src/crew/sandbox.ts"; | |
| 25 | import { stopGroup } from "../src/crew/stop_group.ts"; | |
| 26 | import { CANCEL_GRACE_MS } from "../src/runs/grace.ts"; | |
| 27 | ||
| 28 | const dir = await mkdtemp(join(tmpdir(), "forge-cancel-shell-")); | |
| 29 | const started: number[] = []; | |
| 30 | afterAll(async () => { | |
| 31 | for (const pid of started) | |
| 32 | try { | |
| 33 | process.kill(pid, "SIGKILL"); | |
| 34 | } catch { | |
| 35 | // gone, as it should be | |
| 36 | } | |
| 37 | await rm(dir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | /** Alive and not a zombie waiting for its new parent to reap it. */ | |
| 41 | function alive(pid: number): boolean { | |
| 42 | try { | |
| 43 | process.kill(pid, 0); | |
| 44 | } catch { | |
| 45 | return false; | |
| 46 | } | |
| 47 | try { | |
| 48 | const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); | |
| 49 | const state = stat.slice(stat.lastIndexOf(")") + 2)[0]; | |
| 50 | return state !== "Z" && state !== "X"; | |
| 51 | } catch { | |
| 52 | return false; | |
| 53 | } | |
| 54 | } | |
| 55 | ||
| 56 | async function waitFor(file: string, ms: number): Promise<number> { | |
| 57 | const deadline = Date.now() + ms; | |
| 58 | while (Date.now() < deadline) { | |
| 59 | if (existsSync(file)) { | |
| 60 | const pid = Number(readFileSync(file, "utf8").trim()); | |
| 61 | if (Number.isInteger(pid) && pid > 1) return pid; | |
| 62 | } | |
| 63 | await Bun.sleep(20); // real-timer: polling for a pid file a real child process writes | |
| 64 | } | |
| 65 | throw new Error(`${file} never named a pid`); | |
| 66 | } | |
| 67 | ||
| 68 | let n = 0; | |
| 69 | /** A shell that forks a child that forks a sleeping grandchild; pids land in files. */ | |
| 70 | function tree(ignoreTerm: boolean): { argv: string[]; files: string[] } { | |
| 71 | n += 1; | |
| 72 | const shell = join(dir, `shell-${n}.pid`); | |
| 73 | const child = join(dir, `child-${n}.pid`); | |
| 74 | const grand = join(dir, `grand-${n}.pid`); | |
| 75 | const trap = ignoreTerm ? "trap '' TERM; " : ""; | |
| 76 | const script = `${trap}echo $$ > ${shell}; sh -c 'sleep 300 & echo $$ > ${child}; echo $! > ${grand}.tmp; mv ${grand}.tmp ${grand}; wait' & wait`; | |
| 77 | return { argv: ["/bin/sh", "-c", script], files: [shell, child, grand] }; | |
| 78 | } | |
| 79 | ||
| 80 | async function pidsOf(files: readonly string[]): Promise<number[]> { | |
| 81 | const grand = await waitFor(files[2]!, 10_000); | |
| 82 | const pids = [await waitFor(files[0]!, 1_000), await waitFor(files[1]!, 1_000), grand]; | |
| 83 | started.push(...pids); | |
| 84 | return pids; | |
| 85 | } | |
| 86 | ||
| 87 | const GRACE = 600; | |
| 88 | ||
| 89 | describe.skipIf(process.platform !== "linux")("a cancelled Bash tool call", () => { | |
| 90 | test("ends the shell, its child and its grandchild on SIGTERM, and returns once they are gone", async () => { | |
| 91 | const { argv, files } = tree(false); | |
| 92 | const abort = new AbortController(); | |
| 93 | const call = runShell(argv, dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE); | |
| 94 | const pids = await pidsOf(files); | |
| 95 | expect(pids.every(alive)).toBe(true); | |
| 96 | const at = Date.now(); | |
| 97 | abort.abort(); | |
| 98 | await call; | |
| 99 | // Honoured SIGTERM: ended well inside the grace, every one of them. | |
| 100 | expect(Date.now() - at).toBeLessThan(GRACE); | |
| 101 | expect(pids.filter(alive)).toEqual([]); | |
| 102 | }, 20_000); | |
| 103 | ||
| 104 | test("escalates to SIGKILL for a tree that ignores SIGTERM once the grace runs out", async () => { | |
| 105 | const { argv, files } = tree(true); | |
| 106 | const abort = new AbortController(); | |
| 107 | let returned = false; | |
| 108 | const call = runShell(argv, dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE).then((out) => { | |
| 109 | returned = true; | |
| 110 | return out; | |
| 111 | }); | |
| 112 | const pids = await pidsOf(files); | |
| 113 | abort.abort(); | |
| 114 | await Bun.sleep(GRACE / 2); // real-timer: the real process group must still be alive halfway through runShell's real grace | |
| 115 | // Asked, not yet told: alive halfway through the grace, and the call waits. | |
| 116 | expect(pids.every(alive)).toBe(true); | |
| 117 | expect(returned).toBe(false); | |
| 118 | await call; | |
| 119 | expect(pids.filter(alive)).toEqual([]); | |
| 120 | }, 20_000); | |
| 121 | ||
| 122 | test("a signal aborted before the call starts stops the command at once", async () => { | |
| 123 | const abort = new AbortController(); | |
| 124 | abort.abort(); | |
| 125 | const at = Date.now(); | |
| 126 | await runShell(["/bin/sh", "-c", "sleep 30"], dir, { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, abort.signal, 60_000, GRACE); | |
| 127 | expect(Date.now() - at).toBeLessThan(GRACE); | |
| 128 | }, 20_000); | |
| 129 | }); | |
| 130 | ||
| 131 | describe("stopGroup", () => { | |
| 132 | test("never signals this process's group, init's, or a pid that is not one", async () => { | |
| 133 | for (const pgid of [process.pid, 1, 0, -5, 1.5]) expect(await stopGroup(pgid, 10)).toEqual({ killed: false }); | |
| 134 | }); | |
| 135 | }); | |
| 136 | ||
| 137 | describe.skipIf(process.platform !== "linux")("a cancelled agent whose tree ignores SIGTERM", () => { | |
| 138 | test("is alive halfway through the grace and gone, child and grandchild too, soon after it", async () => { | |
| 139 | const binary = join(dir, "agent"); | |
| 140 | const childFile = join(dir, "agent-child.pid"); | |
| 141 | const grandFile = join(dir, "agent-grand.pid"); | |
| 142 | const agentFile = join(dir, "agent-self.pid"); | |
| 143 | writeFileSync( | |
| 144 | binary, | |
| 145 | `#!/bin/sh | |
| 146 | trap '' TERM | |
| 147 | IFS= read -r first | |
| 148 | echo $$ > ${agentFile} | |
| 149 | sh -c 'sleep 300 & echo $$ > ${childFile}; echo $! > ${grandFile}.tmp; mv ${grandFile}.tmp ${grandFile}; wait' & | |
| 150 | echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}' | |
| 151 | sleep 300 | |
| 152 | `, | |
| 153 | ); | |
| 154 | chmodSync(binary, 0o755); | |
| 155 | const abort = new AbortController(); | |
| 156 | const run = runClaude({ | |
| 157 | binary, | |
| 158 | cwd: dir, | |
| 159 | prompt: "Do the thing.", | |
| 160 | appendSystemPrompt: "", | |
| 161 | budgetUsd: 1, | |
| 162 | env: {}, | |
| 163 | sandbox: sandboxPolicy({ dataDir: dir, agentName: "bot", urls: [] }), | |
| 164 | timeoutMs: 60_000, | |
| 165 | log: () => {}, | |
| 166 | signal: abort.signal, | |
| 167 | settleMs: 1_000, | |
| 168 | }); | |
| 169 | const pids = await pidsOf([agentFile, childFile, grandFile]); | |
| 170 | abort.abort(); | |
| 171 | await Bun.sleep(CANCEL_GRACE_MS / 2); // real-timer: the real agent tree must still be alive halfway through the supervisor's real grace | |
| 172 | expect(pids.every(alive)).toBe(true); | |
| 173 | const deadline = Date.now() + CANCEL_GRACE_MS / 2 + 1_500; | |
| 174 | while (pids.some(alive) && Date.now() < deadline) await Bun.sleep(20); // real-timer: polling until the kernel has ended the real processes after SIGKILL | |
| 175 | expect(pids.filter(alive)).toEqual([]); | |
| 176 | expect((await run).reason).toBe("cancelled"); | |
| 177 | }, 20_000); | |
| 178 | }); |
new above
offline