Public: anyone can read this; the crew can change it.
A cancelled crew run leaves no grandchild process behind, proven with a real tree #14875
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14852-a-cancelled-crew-run-leaves-no-grandchil → main 2 commits fork
3 files
changed +93
−6
src/crew/claude.ts +8 −3
| @@ -22,7 +22,7 @@ import { isNetworkError } from "./network.ts"; | ||
| 22 | 22 | import type { LimitWindow } from "../core/standing.ts"; |
| 23 | 23 | import { childEnv, pump } from "../apps/process.ts"; |
| 24 | 24 | import { endedLine, liveTree } from "../runs/ended.ts"; |
| 25 | import { CANCEL_GRACE_MS } from "../runs/grace.ts"; | |
| 25 | import { CANCEL_GRACE_MS, graceKillLine } from "../runs/grace.ts"; | |
| 26 | 26 | import { spellings, stripForgeDirs } from "../runs/paths.ts"; |
| 27 | 27 | import type { SandboxPolicy } from "./sandbox.ts"; |
| 28 | 28 | import { quietEnv } from "./runner_env.ts"; |
| @@ -410,13 +410,18 @@ async function runClaudeRaw(input: ClaudeInput): Promise<ClaudeOutcome> { | ||
| 410 | 410 | proc.kill(sig); |
| 411 | 411 | } |
| 412 | 412 | }; |
| 413 | // Still running once the grace is out: SIGTERM was ignored, and a | |
| 414 | // cancelled run's log says the group was killed, as a check's does (#14852). | |
| 415 | const killGroup = (): void => { | |
| 416 | if (why === "cancelled") input.log(graceKillLine()); | |
| 417 | signalGroup("SIGKILL"); | |
| 418 | }; | |
| 413 | 419 | let why: "timeout" | "cancelled" | undefined; |
| 414 | 420 | const stopChild = (): void => { |
| 415 | 421 | closeInput(); |
| 416 | 422 | signalGroup("SIGTERM"); |
| 417 | 423 | setTimeout(() => { |
| 418 | if (proc.exitCode === null && proc.signalCode === null) | |
| 419 | signalGroup("SIGKILL"); | |
| 424 | if (proc.exitCode === null && proc.signalCode === null) killGroup(); | |
| 420 | 425 | stopReading.abort(); |
| 421 | 426 | }, CANCEL_GRACE_MS).unref(); |
| 422 | 427 | }; |
test/run-cancel-tree.test.ts +84 −2
| @@ -12,9 +12,14 @@ | ||
| 12 | 12 | * once `CANCEL_GRACE_MS` has run out. A step that honours SIGTERM has its |
| 13 | 13 | * last words logged. A step that ignores it is still alive halfway through |
| 14 | 14 | * the grace and gone soon after the grace ends. |
| 15 | * | |
| 16 | * A crew run is cancelled the same way (#14852): an agent whose tree | |
| 17 | * honours SIGTERM ends, grandchild included, within 2 s and with no kill; | |
| 18 | * one whose tree ignores it is killed once the grace runs out, and its log | |
| 19 | * says so in the same line a check's does. | |
| 15 | 20 | */ |
| 16 | 21 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 17 | import { existsSync, readFileSync } from "node:fs"; | |
| 22 | import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; | |
| 18 | 23 | import { mkdtemp, rm } from "node:fs/promises"; |
| 19 | 24 | import { tmpdir } from "node:os"; |
| 20 | 25 | import { join } from "node:path"; |
| @@ -28,6 +33,8 @@ import { createUser } from "../src/core/users.ts"; | ||
| 28 | 33 | import { childEnv, runStep } from "../src/apps/process.ts"; |
| 29 | 34 | import { CANCEL_GRACE_MS, graceKillLine } from "../src/runs/grace.ts"; |
| 30 | 35 | import { runningRun } from "../src/runs/stoppable.ts"; |
| 36 | import { runClaude } from "../src/crew/claude.ts"; | |
| 37 | import { sandboxPolicy } from "../src/crew/sandbox.ts"; | |
| 31 | 38 | import { quietLogs } from "./helpers/globals.ts"; |
| 32 | 39 | |
| 33 | 40 | quietLogs(); |
| @@ -223,7 +230,7 @@ describe.skipIf(process.platform !== "linux")("a cancelled step's grace", () => | ||
| 223 | 230 | const at = Date.now(); |
| 224 | 231 | controller.abort(); |
| 225 | 232 | // SIGTERM came first and was ignored: a third into the grace, both still run. |
| 226 | await Bun.sleep(CANCEL_GRACE_MS / 3); | |
| 233 | await Bun.sleep(CANCEL_GRACE_MS / 3); // real-timer: real processes must still be running a real third into the grace | |
| 227 | 234 | expect(pids.every(alive)).toBe(true); |
| 228 | 235 | // Then the grace runs out and the group is killed. |
| 229 | 236 | expect(await allGoneWithin(pids, CANCEL_GRACE_MS + 5_000)).toBe(true); |
| @@ -234,3 +241,78 @@ describe.skipIf(process.platform !== "linux")("a cancelled step's grace", () => | ||
| 234 | 241 | expect(lines.filter((l) => l === graceKillLine())).toHaveLength(1); |
| 235 | 242 | }, 30_000); |
| 236 | 243 | }); |
| 244 | ||
| 245 | /** A stand-in agent: forks a shell that forks a sleeper, says init, and sleeps. */ | |
| 246 | function agentTree(name: string, ignoreTerm: boolean): { binary: string; files: string[] } { | |
| 247 | const binary = join(pidDir, `${name}-agent`); | |
| 248 | const files = ["self", "child", "grand"].map((f) => join(pidDir, `${name}-${f}.pid`)); | |
| 249 | const [self, child, grand] = files as [string, string, string]; | |
| 250 | writeFileSync( | |
| 251 | binary, | |
| 252 | `#!/bin/sh | |
| 253 | ${ignoreTerm ? "trap '' TERM" : ""} | |
| 254 | IFS= read -r first | |
| 255 | echo $$ > ${self} | |
| 256 | sh -c 'sleep 300 & echo $$ > ${child}; echo $! > ${grand}.tmp; mv ${grand}.tmp ${grand}; wait' & | |
| 257 | echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}' | |
| 258 | sleep 300 | |
| 259 | `, | |
| 260 | ); | |
| 261 | chmodSync(binary, 0o755); | |
| 262 | return { binary, files }; | |
| 263 | } | |
| 264 | ||
| 265 | function startAgent(binary: string, lines: string[], signal: AbortSignal): ReturnType<typeof runClaude> { | |
| 266 | return runClaude({ | |
| 267 | binary, | |
| 268 | cwd: pidDir, | |
| 269 | prompt: "Do the thing.", | |
| 270 | appendSystemPrompt: "", | |
| 271 | budgetUsd: 1, | |
| 272 | env: {}, | |
| 273 | sandbox: sandboxPolicy({ dataDir: pidDir, agentName: "bot", urls: [] }), | |
| 274 | timeoutMs: 60_000, | |
| 275 | log: (line) => lines.push(line), | |
| 276 | signal, | |
| 277 | settleMs: 1_000, | |
| 278 | }); | |
| 279 | } | |
| 280 | ||
| 281 | async function agentPids(files: readonly string[]): Promise<number[]> { | |
| 282 | const grand = await pidIn(files[2]!, 10_000); | |
| 283 | const pids = [await pidIn(files[0]!, 1_000), await pidIn(files[1]!, 1_000), grand]; | |
| 284 | started.push(...pids); | |
| 285 | return pids; | |
| 286 | } | |
| 287 | ||
| 288 | describe.skipIf(process.platform !== "linux")("a cancelled crew run", () => { | |
| 289 | test("that honours SIGTERM ends the agent, its child and its grandchild within 2 s, with no kill", async () => { | |
| 290 | const { binary, files } = agentTree("crew-honour", false); | |
| 291 | const lines: string[] = []; | |
| 292 | const abort = new AbortController(); | |
| 293 | const run = startAgent(binary, lines, abort.signal); | |
| 294 | const pids = await agentPids(files); | |
| 295 | expect(pids.every(alive)).toBe(true); | |
| 296 | abort.abort(); | |
| 297 | expect(await allGoneWithin(pids, 2_000)).toBe(true); | |
| 298 | expect((await run).reason).toBe("cancelled"); | |
| 299 | expect(lines).not.toContain(graceKillLine()); | |
| 300 | }, 20_000); | |
| 301 | ||
| 302 | test("that ignores SIGTERM is killed once the grace runs out, grandchild too, and the log says so", async () => { | |
| 303 | const { binary, files } = agentTree("crew-deaf", true); | |
| 304 | const lines: string[] = []; | |
| 305 | const abort = new AbortController(); | |
| 306 | const run = startAgent(binary, lines, abort.signal); | |
| 307 | const pids = await agentPids(files); | |
| 308 | abort.abort(); | |
| 309 | // Asked first: a third into the grace, the whole tree still runs and nothing was killed. | |
| 310 | await Bun.sleep(CANCEL_GRACE_MS / 3); // real-timer: real processes must still be running a real third into the grace | |
| 311 | expect(pids.map(alive)).toEqual([true, true, true]); | |
| 312 | expect(lines).not.toContain(graceKillLine()); | |
| 313 | // Then told: within 2 s of the cancel, all three are gone. | |
| 314 | expect(await allGoneWithin(pids, 2_000 - CANCEL_GRACE_MS / 3)).toBe(true); | |
| 315 | expect((await run).reason).toBe("cancelled"); | |
| 316 | expect(lines.filter((l) => l === graceKillLine())).toHaveLength(1); | |
| 317 | }, 20_000); | |
| 318 | }); |
test/timers-named.test.ts +1 −1
| @@ -180,7 +180,7 @@ const ALLOWED: readonly Allowed[] = [ | ||
| 180 | 180 | ["src/crew/plan.ts", "() => { try { proc.kill(\"SIGKILL\"); }"], |
| 181 | 181 | ["src/crew/runner.ts", "() => proc.kill(\"SIGKILL\")"], |
| 182 | 182 | ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) { input"], |
| 183 | ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) signal"], | |
| 183 | ["src/crew/claude.ts", "() => { if (proc.exitCode === null && proc.signalCode === null) killGroup"], | |
| 184 | 184 | ["src/crew/claude.ts", "() => proc.kill(\"SIGKILL\")"], |
| 185 | 185 | ["src/git/env.ts", "() => proc.kill(\"SIGKILL\")"], |
| 186 | 186 | ["src/git/own_refs.ts", "() => proc.kill(\"SIGKILL\")"], |
new above
offline