Public: anyone can read this; the crew can change it.

A cancelled crew run leaves no grandchild process behind, proven with a real tree #14875

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14852-a-cancelled-crew-run-leaves-no-grandchil → main 2 commits fork

  • Merged. mason/14852-a-cancelled-crew-run-leaves-no-grandchil was merged into main as 75c6839.

Review

Sign in to review this change.
3 files changed +93 −6
src/crew/claude.ts +8 −3
@@ -22,7 +22,7 @@ import { isNetworkError } from "./network.ts";
2222 import type { LimitWindow } from "../core/standing.ts";
2323 import { childEnv, pump } from "../apps/process.ts";
2424 import { endedLine, liveTree } from "../runs/ended.ts";
25 import { CANCEL_GRACE_MS } from "../runs/grace.ts";
25 import { CANCEL_GRACE_MS, graceKillLine } from "../runs/grace.ts";
2626 import { spellings, stripForgeDirs } from "../runs/paths.ts";
2727 import type { SandboxPolicy } from "./sandbox.ts";
2828 import { quietEnv } from "./runner_env.ts";
@@ -410,13 +410,18 @@ async function runClaudeRaw(input: ClaudeInput): Promise<ClaudeOutcome> {
410410 proc.kill(sig);
411411 }
412412 };
413 // Still running once the grace is out: SIGTERM was ignored, and a
414 // cancelled run's log says the group was killed, as a check's does (#14852).
415 const killGroup = (): void => {
416 if (why === "cancelled") input.log(graceKillLine());
417 signalGroup("SIGKILL");
418 };
413419 let why: "timeout" | "cancelled" | undefined;
414420 const stopChild = (): void => {
415421 closeInput();
416422 signalGroup("SIGTERM");
417423 setTimeout(() => {
418 if (proc.exitCode === null && proc.signalCode === null)
419 signalGroup("SIGKILL");
424 if (proc.exitCode === null && proc.signalCode === null) killGroup();
420425 stopReading.abort();
421426 }, CANCEL_GRACE_MS).unref();
422427 };
test/run-cancel-tree.test.ts +84 −2
@@ -12,9 +12,14 @@
1212 * once `CANCEL_GRACE_MS` has run out. A step that honours SIGTERM has its
1313 * last words logged. A step that ignores it is still alive halfway through
1414 * the grace and gone soon after the grace ends.
15 *
16 * A crew run is cancelled the same way (#14852): an agent whose tree
17 * honours SIGTERM ends, grandchild included, within 2 s and with no kill;
18 * one whose tree ignores it is killed once the grace runs out, and its log
19 * says so in the same line a check's does.
1520 */
1621 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
17 import { existsSync, readFileSync } from "node:fs";
22 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
1823 import { mkdtemp, rm } from "node:fs/promises";
1924 import { tmpdir } from "node:os";
2025 import { join } from "node:path";
@@ -28,6 +33,8 @@ import { createUser } from "../src/core/users.ts";
2833 import { childEnv, runStep } from "../src/apps/process.ts";
2934 import { CANCEL_GRACE_MS, graceKillLine } from "../src/runs/grace.ts";
3035 import { runningRun } from "../src/runs/stoppable.ts";
36 import { runClaude } from "../src/crew/claude.ts";
37 import { sandboxPolicy } from "../src/crew/sandbox.ts";
3138 import { quietLogs } from "./helpers/globals.ts";
3239
3340 quietLogs();
@@ -223,7 +230,7 @@ describe.skipIf(process.platform !== "linux")("a cancelled step's grace", () =>
223230 const at = Date.now();
224231 controller.abort();
225232 // SIGTERM came first and was ignored: a third into the grace, both still run.
226 await Bun.sleep(CANCEL_GRACE_MS / 3);
233 await Bun.sleep(CANCEL_GRACE_MS / 3); // real-timer: real processes must still be running a real third into the grace
227234 expect(pids.every(alive)).toBe(true);
228235 // Then the grace runs out and the group is killed.
229236 expect(await allGoneWithin(pids, CANCEL_GRACE_MS + 5_000)).toBe(true);
@@ -234,3 +241,78 @@ describe.skipIf(process.platform !== "linux")("a cancelled step's grace", () =>
234241 expect(lines.filter((l) => l === graceKillLine())).toHaveLength(1);
235242 }, 30_000);
236243 });
244
245 /** A stand-in agent: forks a shell that forks a sleeper, says init, and sleeps. */
246 function agentTree(name: string, ignoreTerm: boolean): { binary: string; files: string[] } {
247 const binary = join(pidDir, `${name}-agent`);
248 const files = ["self", "child", "grand"].map((f) => join(pidDir, `${name}-${f}.pid`));
249 const [self, child, grand] = files as [string, string, string];
250 writeFileSync(
251 binary,
252 `#!/bin/sh
253 ${ignoreTerm ? "trap '' TERM" : ""}
254 IFS= read -r first
255 echo $$ > ${self}
256 sh -c 'sleep 300 & echo $$ > ${child}; echo $! > ${grand}.tmp; mv ${grand}.tmp ${grand}; wait' &
257 echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}'
258 sleep 300
259 `,
260 );
261 chmodSync(binary, 0o755);
262 return { binary, files };
263 }
264
265 function startAgent(binary: string, lines: string[], signal: AbortSignal): ReturnType<typeof runClaude> {
266 return runClaude({
267 binary,
268 cwd: pidDir,
269 prompt: "Do the thing.",
270 appendSystemPrompt: "",
271 budgetUsd: 1,
272 env: {},
273 sandbox: sandboxPolicy({ dataDir: pidDir, agentName: "bot", urls: [] }),
274 timeoutMs: 60_000,
275 log: (line) => lines.push(line),
276 signal,
277 settleMs: 1_000,
278 });
279 }
280
281 async function agentPids(files: readonly string[]): Promise<number[]> {
282 const grand = await pidIn(files[2]!, 10_000);
283 const pids = [await pidIn(files[0]!, 1_000), await pidIn(files[1]!, 1_000), grand];
284 started.push(...pids);
285 return pids;
286 }
287
288 describe.skipIf(process.platform !== "linux")("a cancelled crew run", () => {
289 test("that honours SIGTERM ends the agent, its child and its grandchild within 2 s, with no kill", async () => {
290 const { binary, files } = agentTree("crew-honour", false);
291 const lines: string[] = [];
292 const abort = new AbortController();
293 const run = startAgent(binary, lines, abort.signal);
294 const pids = await agentPids(files);
295 expect(pids.every(alive)).toBe(true);
296 abort.abort();
297 expect(await allGoneWithin(pids, 2_000)).toBe(true);
298 expect((await run).reason).toBe("cancelled");
299 expect(lines).not.toContain(graceKillLine());
300 }, 20_000);
301
302 test("that ignores SIGTERM is killed once the grace runs out, grandchild too, and the log says so", async () => {
303 const { binary, files } = agentTree("crew-deaf", true);
304 const lines: string[] = [];
305 const abort = new AbortController();
306 const run = startAgent(binary, lines, abort.signal);
307 const pids = await agentPids(files);
308 abort.abort();
309 // Asked first: a third into the grace, the whole tree still runs and nothing was killed.
310 await Bun.sleep(CANCEL_GRACE_MS / 3); // real-timer: real processes must still be running a real third into the grace
311 expect(pids.map(alive)).toEqual([true, true, true]);
312 expect(lines).not.toContain(graceKillLine());
313 // Then told: within 2 s of the cancel, all three are gone.
314 expect(await allGoneWithin(pids, 2_000 - CANCEL_GRACE_MS / 3)).toBe(true);
315 expect((await run).reason).toBe("cancelled");
316 expect(lines.filter((l) => l === graceKillLine())).toHaveLength(1);
317 }, 20_000);
318 });
test/timers-named.test.ts +1 −1
@@ -180,7 +180,7 @@ const ALLOWED: readonly Allowed[] = [
180180 ["src/crew/plan.ts", "() => { try { proc.kill(\"SIGKILL\"); }"],
181181 ["src/crew/runner.ts", "() => proc.kill(\"SIGKILL\")"],
182182 ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) { input"],
183 ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) signal"],
183 ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) killGroup"],
184184 ["src/crew/claude.ts", "() => proc.kill(\"SIGKILL\")"],
185185 ["src/git/env.ts", "() => proc.kill(\"SIGKILL\")"],
186186 ["src/git/own_refs.ts", "() => proc.kill(\"SIGKILL\")"],

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs