Public: anyone can read this; the crew can change it.
A cancelled run ends every process it started, including grandchildren, proven by a test #10930
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10919-a-cancelled-run-ends-every-process-it-st → main 2 commits fork
4 files
changed +109
−3
src/apps/process.ts +17 −2
| @@ -19,6 +19,7 @@ import { isReservedAppEnvName } from "../core/app_env.ts"; | ||
| 19 | 19 | import { isFeatureVarName } from "./feature_env.ts"; |
| 20 | 20 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 21 | 21 | import { endedLine, liveTree } from "../runs/ended.ts"; |
| 22 | import { CANCEL_GRACE_MS, graceKillLine } from "../runs/grace.ts"; | |
| 22 | 23 | import { spellings, stripForgeDirs } from "../runs/paths.ts"; |
| 23 | 24 | import { seconds } from "./timing.ts"; |
| 24 | 25 | import { toolchainUse, type ToolchainUse } from "./toolchain.ts"; |
| @@ -381,11 +382,24 @@ export async function runStep( | ||
| 381 | 382 | kill(); |
| 382 | 383 | stopReading.abort(); |
| 383 | 384 | }, timeoutMs); |
| 385 | // A cancel asks first: SIGTERM to the group (or to the tree inside a | |
| 386 | // cell), then SIGKILL once the grace has run out, if the leader is still | |
| 387 | // up. While the grace lasts, the pipes stay open, so the step's last | |
| 388 | // words reach the log. | |
| 389 | let graceTimer: ReturnType<typeof setTimeout> | undefined; | |
| 384 | 390 | const onAbort = (): void => { |
| 385 | 391 | why = `${name} cancelled`; |
| 386 | 392 | options.log(endedLine(liveTree(proc.pid).length)); |
| 387 | kill(); | |
| 388 | stopReading.abort(); | |
| 393 | if (cell === undefined) signalGroup(proc.pid, "SIGTERM"); | |
| 394 | else signalTree(proc.pid, "SIGTERM"); | |
| 395 | graceTimer = setTimeout(() => { | |
| 396 | if (proc.exitCode === null && proc.signalCode === null) { | |
| 397 | options.log(graceKillLine()); | |
| 398 | if (cell === undefined) signalGroup(proc.pid, "SIGKILL"); | |
| 399 | else killTree(proc.pid); | |
| 400 | } | |
| 401 | stopReading.abort(); | |
| 402 | }, CANCEL_GRACE_MS); | |
| 389 | 403 | }; |
| 390 | 404 | if (options.signal.aborted) onAbort(); |
| 391 | 405 | else options.signal.addEventListener("abort", onAbort, { once: true }); |
| @@ -415,6 +429,7 @@ export async function runStep( | ||
| 415 | 429 | return ok(); |
| 416 | 430 | } finally { |
| 417 | 431 | clearTimeout(timer); |
| 432 | clearTimeout(graceTimer); | |
| 418 | 433 | options.signal.removeEventListener("abort", onAbort); |
| 419 | 434 | cell?.close(); |
| 420 | 435 | } |
src/crew/claude.ts +2 −1
| @@ -19,6 +19,7 @@ import { openRunCell, type RunLimits } from "./run-cell.ts"; | ||
| 19 | 19 | import { watchLabelled } from "../apps/contain.ts"; |
| 20 | 20 | import { childEnv, pump } from "../apps/process.ts"; |
| 21 | 21 | import { endedLine, liveTree } from "../runs/ended.ts"; |
| 22 | import { CANCEL_GRACE_MS } from "../runs/grace.ts"; | |
| 22 | 23 | import { spellings, stripForgeDirs } from "../runs/paths.ts"; |
| 23 | 24 | import type { SandboxPolicy } from "./sandbox.ts"; |
| 24 | 25 | |
| @@ -473,7 +474,7 @@ export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> { | ||
| 473 | 474 | if (proc.exitCode === null && proc.signalCode === null) |
| 474 | 475 | signalGroup("SIGKILL"); |
| 475 | 476 | stopReading.abort(); |
| 476 | }, 5_000).unref(); | |
| 477 | }, CANCEL_GRACE_MS).unref(); | |
| 477 | 478 | }; |
| 478 | 479 | const timer = setTimeout(() => { |
| 479 | 480 | why = "timeout"; |
src/runs/grace.ts added +18
| @@ -0,0 +1,18 @@ | ||
| 1 | /** | |
| 2 | * How long a cancelled run has to end on its own (#10919). A cancel asks | |
| 3 | * the run's process group to stop (SIGTERM), so a step can flush its last | |
| 4 | * output and remove its temporary files. Whatever is still alive once the | |
| 5 | * grace runs out gets SIGKILL, sent to the group and never to a pid outside it. | |
| 6 | * | |
| 7 | * Why 1.5 s: a cancel promises the person who pressed it that the run's | |
| 8 | * whole tree is gone within 2 s (`test/run-cancel-tree.test.ts`), so a run | |
| 9 | * that ignores SIGTERM has to be killed early enough to keep that promise. | |
| 10 | * A process that honours SIGTERM ends in milliseconds, so 1.5 s is far more | |
| 11 | * than it needs. | |
| 12 | */ | |
| 13 | export const CANCEL_GRACE_MS = 1_500; | |
| 14 | ||
| 15 | /** The run log line written when the grace ran out and the group was killed. */ | |
| 16 | export function graceKillLine(graceMs: number = CANCEL_GRACE_MS): string { | |
| 17 | return `cancel: still running after ${(graceMs / 1000).toFixed(1)} s grace, killed`; | |
| 18 | } |
test/run-cancel-tree.test.ts +72
| @@ -7,6 +7,11 @@ | ||
| 7 | 7 | * |
| 8 | 8 | * Before this, the cancel marked the row and nothing else: an agent run was |
| 9 | 9 | * aborted, but a check ran on to its end, its tree with it. |
| 10 | * | |
| 11 | * A cancel asks before it kills (#10919): SIGTERM to the group, then SIGKILL | |
| 12 | * once `CANCEL_GRACE_MS` has run out. A step that honours SIGTERM has its | |
| 13 | * last words logged. A step that ignores it is still alive halfway through | |
| 14 | * the grace and gone soon after the grace ends. | |
| 10 | 15 | */ |
| 11 | 16 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 12 | 17 | import { existsSync, readFileSync } from "node:fs"; |
| @@ -20,6 +25,8 @@ import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | ||
| 20 | 25 | import { getRun, listRuns } from "../src/core/runs.ts"; |
| 21 | 26 | import { readRunLog } from "../src/runs/log.ts"; |
| 22 | 27 | import { createUser } from "../src/core/users.ts"; |
| 28 | import { childEnv, runStep } from "../src/apps/process.ts"; | |
| 29 | import { CANCEL_GRACE_MS, graceKillLine } from "../src/runs/grace.ts"; | |
| 23 | 30 | import { runningRun } from "../src/runs/stoppable.ts"; |
| 24 | 31 | import { quietLogs } from "./helpers/globals.ts"; |
| 25 | 32 | |
| @@ -87,6 +94,13 @@ async function pidIn(file: string, ms: number): Promise<number> { | ||
| 87 | 94 | throw new Error(`${file} never named a pid`); |
| 88 | 95 | } |
| 89 | 96 | |
| 97 | /** Polls: true once the file exists, false at the deadline. */ | |
| 98 | async function pathWithin(file: string, ms: number): Promise<boolean> { | |
| 99 | const deadline = Date.now() + ms; | |
| 100 | while (!existsSync(file) && Date.now() < deadline) await Bun.sleep(20); | |
| 101 | return existsSync(file); | |
| 102 | } | |
| 103 | ||
| 90 | 104 | describe.skipIf(process.platform !== "linux")("a cancelled check", () => { |
| 91 | 105 | test("ends its command, the child it forked and that child's child within 2 s, and says cancelled", async () => { |
| 92 | 106 | const made = await createUser(app.db, { |
| @@ -155,3 +169,61 @@ describe.skipIf(process.platform !== "linux")("a cancelled check", () => { | ||
| 155 | 169 | expect(Number(/^cancel ended (\d+) process/.exec(ended[0]!)?.[1])).toBeGreaterThanOrEqual(pids.length); |
| 156 | 170 | }, 45_000); |
| 157 | 171 | }); |
| 172 | ||
| 173 | describe.skipIf(process.platform !== "linux")("a cancelled step's grace", () => { | |
| 174 | test("is short enough to keep the 2 s promise", () => { | |
| 175 | expect(CANCEL_GRACE_MS).toBeGreaterThan(0); | |
| 176 | expect(CANCEL_GRACE_MS).toBeLessThan(2_000); | |
| 177 | }); | |
| 178 | ||
| 179 | test("a step that honours SIGTERM ends on it, its last words logged, with no kill", async () => { | |
| 180 | const lines: string[] = []; | |
| 181 | const ready = join(pidDir, "honour.ready"); | |
| 182 | const controller = new AbortController(); | |
| 183 | const step = runStep( | |
| 184 | "check", | |
| 185 | `trap 'echo got term; exit 0' TERM; sleep 300 & echo ok > ${ready}; wait`, | |
| 186 | { cwd: pidDir, env: childEnv({}), log: (l) => lines.push(l), signal: controller.signal }, | |
| 187 | ); | |
| 188 | // The trap is set before the ready file is written: only then is a | |
| 189 | // SIGTERM the step's to honour. | |
| 190 | expect(await pathWithin(ready, 10_000)).toBe(true); | |
| 191 | controller.abort(); | |
| 192 | const result = await step; | |
| 193 | expect(result.ok).toBe(false); | |
| 194 | if (!result.ok) expect(result.error).toBe("check cancelled"); | |
| 195 | // It ended on SIGTERM: its trap spoke, and the grace never ran out. | |
| 196 | // (No wall-clock bound: a loaded check machine makes one a flake.) | |
| 197 | expect(lines).toContain("got term"); | |
| 198 | expect(lines).not.toContain(graceKillLine()); | |
| 199 | }, 20_000); | |
| 200 | ||
| 201 | test("a step and the sleeper it forked that both ignore SIGTERM are killed once the grace runs out", async () => { | |
| 202 | const lines: string[] = []; | |
| 203 | const shell = join(pidDir, "deaf-shell.pid"); | |
| 204 | const grand = join(pidDir, "deaf-grand.pid"); | |
| 205 | const controller = new AbortController(); | |
| 206 | // `trap '' TERM` is inherited: the sleeper ignores SIGTERM too. | |
| 207 | const step = runStep( | |
| 208 | "check", | |
| 209 | `trap '' TERM; echo $$ > ${shell}; sleep 300 & echo $! > ${grand}.tmp; mv ${grand}.tmp ${grand}; wait`, | |
| 210 | { cwd: pidDir, env: childEnv({}), log: (l) => lines.push(l), signal: controller.signal }, | |
| 211 | ); | |
| 212 | const pids = [await pidIn(grand, 10_000), await pidIn(shell, 5_000)]; | |
| 213 | started.push(...pids); | |
| 214 | expect(pids.every(alive)).toBe(true); | |
| 215 | ||
| 216 | const at = Date.now(); | |
| 217 | controller.abort(); | |
| 218 | // SIGTERM came first and was ignored: a third into the grace, both still run. | |
| 219 | await Bun.sleep(CANCEL_GRACE_MS / 3); | |
| 220 | expect(pids.every(alive)).toBe(true); | |
| 221 | // Then the grace runs out and the group is killed. | |
| 222 | expect(await allGoneWithin(pids, CANCEL_GRACE_MS + 5_000)).toBe(true); | |
| 223 | const result = await step; | |
| 224 | expect(Date.now() - at).toBeGreaterThanOrEqual(CANCEL_GRACE_MS - 50); | |
| 225 | expect(result.ok).toBe(false); | |
| 226 | if (!result.ok) expect(result.error).toBe("check cancelled"); | |
| 227 | expect(lines.filter((l) => l === graceKillLine())).toHaveLength(1); | |
| 228 | }, 30_000); | |
| 229 | }); |