Public: anyone can read this; the crew can change it.
A cancelled run ends every process it started, and a test proves none is left #13341
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13291-a-cancelled-run-ends-every-process-it-st → main 9 commits fork 5 threads · 5 unresolved
AGENTS.md +1 −1
Diff not shown — a line 2,989 characters wide (limit 2,000). +1 −1
View filedocs/features/run-state.md +1
| @@ -8,3 +8,4 @@ | ||
| 8 | 8 | - `GET …/runs?state=queued|running|ok|failed|cancelled` filters; an unknown state is a 422. |
| 9 | 9 | - **In the terminal**: the issue screen (`src/tui/screens/issue.ts`, rows from `src/tui/screens/runplan.ts`) shows a *Plan* card above the thread — `[x]`/`[ ]` items and the question as `needs input: …` — read through the same `readPlan` (so gated as the page is), and its title says *silent* or *needs input* when the issue's runs read that way, from `shownStates`. |
| 10 | 10 | - The runs API works out `shown` for a page of runs with one plan read (`shownStates`), not one per row. |
| 11 | - **Stopping**: a cancel marks the row at once, but the run's page says *stopping* (a moving word, filed under *working*) until its worker here has ended every process the run started — SIGTERM to the group, SIGKILL once the grace runs out, an agent's orphans reaped — and only then *cancelled*; the cancel's one `run.finished` is announced at that moment, not before (`src/runs/ending.ts`, #13291; `test/run-cancel-stopping.test.ts` cancels a check whose stand-in step holds on after the abort, and finds the word *stopping* and no `run.finished` until the step lets go, then *cancelled* and the one `run.finished` after). That none of the processes is left is `test/run-cancel-tree.test.ts`'s: a cancelled check's command, the child it forked and that child's child are all gone within 2 s, and a step and the sleeper it forked that both ignore SIGTERM are killed once the grace runs out. A worker that never lets go is announced at the 30 s deadline (`ENDING_DEADLINE_MS`) while the page still says *stopping*; when it lets go later, a `run.stopped` wake-up (not a second `run.finished`) turns the live head to *cancelled*. | |
src/actions/runs.ts +29 −2
| @@ -36,6 +36,8 @@ import { | ||
| 36 | 36 | import { getUsersByIds, type User } from "../core/users.ts"; |
| 37 | 37 | import { abortAgentRun } from "../crew/running.ts"; |
| 38 | 38 | import { stopRun } from "../runs/stoppable.ts"; |
| 39 | import { whenEnded } from "../runs/ending.ts"; | |
| 40 | import { createLog } from "../kernel/log.ts"; | |
| 39 | 41 | import { queueAgentRun, taskOf } from "../crew/dispatch.ts"; |
| 40 | 42 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 41 | 43 | import { readRunLog, readRunLogTail, runLogPath } from "../runs/log.ts"; |
| @@ -45,6 +47,8 @@ import { cancelledBy } from "../runs/cancelled.ts"; | ||
| 45 | 47 | import { requireReader, requireWriter, type AccessError } from "./access.ts"; |
| 46 | 48 | import { deskForRun } from "../crew/desk_posts.ts"; |
| 47 | 49 | |
| 50 | const log = createLog("runs"); | |
| 51 | ||
| 48 | 52 | /** What a run event carries: enough for a surface to render the card. */ |
| 49 | 53 | export function runPayload(run: Run): Record<string, unknown> { |
| 50 | 54 | return { |
| @@ -374,10 +378,33 @@ export function cancelRepoRun( | ||
| 374 | 378 | ); |
| 375 | 379 | if (!cancelled) return err("finished"); |
| 376 | 380 | // The row says cancelled first, so the run's own end leaves it so; the |
| 377 | // abort then ends the process group it started. | |
| 381 | // abort then ends the process group it started. A run worked here is | |
| 382 | // announced once its worker has let go, every process ended (#13291); | |
| 383 | // one worked nowhere here is announced now. Should the worker let go only | |
| 384 | // after the deadline announced it, `run.stopped` wakes the page still | |
| 385 | // saying *stopping*: a wake-up, not a second `run.finished`. | |
| 386 | const heard = (what: string, say: () => void): void => { | |
| 387 | try { | |
| 388 | say(); | |
| 389 | } catch (error) { | |
| 390 | // The forge closed while the run wound down: nobody is left to hear. | |
| 391 | log.warn(`a cancel's ${what} could not be announced`, { run: runId, error: String(error) }); | |
| 392 | } | |
| 393 | }; | |
| 394 | const announce = (): void => | |
| 395 | heard("end", () => announceRun(app, getRun(app.db, runId) ?? cancelled, writer.value.id)); | |
| 396 | const stopped = (): void => | |
| 397 | heard("stop", () => | |
| 398 | app.bus.append({ | |
| 399 | kind: "run.stopped", | |
| 400 | repoId: cancelled.repoId, | |
| 401 | payload: { runId, kind: cancelled.kind, env: cancelled.env, commitOid: cancelled.commitOid }, | |
| 402 | }), | |
| 403 | ); | |
| 404 | const waits = whenEnded(app.db, runId, announce, stopped); | |
| 378 | 405 | if (run.kind === "agent") abortAgentRun(runId); |
| 379 | 406 | else stopRun(runId); |
| 380 | announceRun(app, cancelled, writer.value.id); | |
| 407 | if (!waits) announce(); | |
| 381 | 408 | return ok(cancelled); |
| 382 | 409 | } |
| 383 | 410 |
src/checks/service.ts +5
| @@ -106,6 +106,7 @@ import { appendRunLog } from "../runs/log.ts"; | ||
| 106 | 106 | import { memoryHold } from "../runs/memory_hold.ts"; |
| 107 | 107 | import { cancelledLine } from "../runs/cancelled.ts"; |
| 108 | 108 | import { trackRun, untrackRun } from "../runs/stoppable.ts"; |
| 109 | import { workBegins, workEnds } from "../runs/ending.ts"; | |
| 109 | 110 | import { runningLimitSentence, runningRuns } from "../settings/quotas.ts"; |
| 110 | 111 | import { exportTree, readFileAt } from "../apps/checkout.ts"; |
| 111 | 112 | import { CHECK_TIMEOUT_MAX, parseCheck } from "../apps/manifest.ts"; |
| @@ -639,6 +640,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 639 | 640 | if (low) verifying = run.id; |
| 640 | 641 | // A person's cancel (`cancelRepoRun`) reaches it through this. |
| 641 | 642 | trackRun(run.id, abort); |
| 643 | workBegins(app.db, run.id); | |
| 642 | 644 | const promise = execute(run, repo, abort.signal, !low) |
| 643 | 645 | .catch((error) => |
| 644 | 646 | log.error("check failed to execute", { |
| @@ -648,6 +650,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 648 | 650 | ) |
| 649 | 651 | .finally(() => { |
| 650 | 652 | untrackRun(run.id, abort); |
| 653 | // Every process the check started has ended: a cancel waiting on | |
| 654 | // that is heard now (#13291). | |
| 655 | workEnds(app.db, run.id); | |
| 651 | 656 | inFlight.delete(run.id); |
| 652 | 657 | if (verifying === run.id) verifying = undefined; |
| 653 | 658 | // A pull's green that ran while its base moved is of the old tip: asked again once. |
src/crew/runner.ts +5
| @@ -181,6 +181,7 @@ import { | ||
| 181 | 181 | type UsageOutcome, |
| 182 | 182 | } from "../core/usage.ts"; |
| 183 | 183 | import { trackAgentRun, untrackAgentRun } from "./running.ts"; |
| 184 | import { workBegins, workEnds } from "../runs/ending.ts"; | |
| 184 | 185 | import { installCacheEnv, sandboxPolicy } from "./sandbox.ts"; |
| 185 | 186 | import { runGitAuth } from "./git_auth.ts"; |
| 186 | 187 | import { grantSecrets, needsYou, parseDeclaration, ungrantedSentence, type Grant } from "./grant.ts"; |
| @@ -427,6 +428,9 @@ export async function executeAgentRun( | ||
| 427 | 428 | const cancelled = cancelledLine(getRun(app.db, run.id)); |
| 428 | 429 | if (cancelled !== undefined) appendRunLog(app.config.dataDir, run.id, cancelled); |
| 429 | 430 | } |
| 431 | // The agent and the orphans it left are reaped: a cancel waiting on | |
| 432 | // that is heard now (#13291). | |
| 433 | workEnds(app.db, run.id); | |
| 430 | 434 | } |
| 431 | 435 | } |
| 432 | 436 | |
| @@ -460,6 +464,7 @@ async function runClaimed( | ||
| 460 | 464 | if (startedAt !== undefined) setRunBase(app.db, run.id, startedAt); |
| 461 | 465 | const controller = new AbortController(); |
| 462 | 466 | trackAgentRun(run.id, controller); |
| 467 | workBegins(app.db, run.id); | |
| 463 | 468 | const write = (line: string): void => |
| 464 | 469 | appendRunLog(app.config.dataDir, run.id, line); |
| 465 | 470 | announceRun(app, claimed); |
src/design/activity.ts +1
| @@ -231,6 +231,7 @@ const WORDS: Record<string, string> = { | ||
| 231 | 231 | "desk.cleared": "cleared a note from the desk", |
| 232 | 232 | "run.queued": "queued a run", |
| 233 | 233 | "run.started": "started a run", |
| 234 | "run.stopped": "ended a cancelled run's last process", | |
| 234 | 235 | "schedule.steered": "steered a schedule", |
| 235 | 236 | "app.created": "made it an app", |
| 236 | 237 | "app.updated": "changed the app's settings", |
src/runs/ending.ts added +81
| @@ -0,0 +1,81 @@ | ||
| 1 | /** | |
| 2 | * The runs whose worker in this process has not yet wound down, by run id | |
| 3 | * (#13291). A cancel marks the row at once, but the processes the run | |
| 4 | * started end a moment later (SIGTERM, then SIGKILL once the grace runs | |
| 5 | * out; an agent's orphans are reaped after it). Until the worker has let | |
| 6 | * go, the run is *stopping*, not stopped: its page says so (`runWord`), and | |
| 7 | * the cancel's one `run.finished` waits here (`whenEnded`) so no surface | |
| 8 | * hears "stopped" before every process is gone. | |
| 9 | * | |
| 10 | * The checks service (`checks/service.ts` `start`) and the agent runner | |
| 11 | * (`crew/runner.ts` `executeAgentRun`) mark each run they work from claim | |
| 12 | * to their last line. At most one entry per run in flight here; gone on | |
| 13 | * restart, when nothing is in flight. The state is the forge's, keyed by its | |
| 14 | * database (#12569): two forges in one process never see each other's runs. | |
| 15 | */ | |
| 16 | ||
| 17 | import type { Database } from "bun:sqlite"; | |
| 18 | import { during } from "../perf/jobs.ts"; | |
| 19 | ||
| 20 | /** How long a cancel's announcement waits for a worker that never lets go. */ | |
| 21 | export const ENDING_DEADLINE_MS = 30_000; | |
| 22 | ||
| 23 | const forges = new WeakMap<Database, Map<string, Array<() => void>>>(); | |
| 24 | ||
| 25 | /** The runs a forge's workers here still hold, by its database. */ | |
| 26 | function endingOf(db: Database): Map<string, Array<() => void>> { | |
| 27 | let held = forges.get(db); | |
| 28 | if (held === undefined) forges.set(db, (held = new Map())); | |
| 29 | return held; | |
| 30 | } | |
| 31 | ||
| 32 | /** A worker in this process starts on `runId`. */ | |
| 33 | export function workBegins(db: Database, runId: string): void { | |
| 34 | const held = endingOf(db); | |
| 35 | if (!held.has(runId)) held.set(runId, []); | |
| 36 | } | |
| 37 | ||
| 38 | /** The worker has let `runId` go: every process it started has ended. */ | |
| 39 | export function workEnds(db: Database, runId: string): void { | |
| 40 | const held = endingOf(db); | |
| 41 | const waiting = held.get(runId); | |
| 42 | held.delete(runId); | |
| 43 | for (const fn of waiting ?? []) fn(); | |
| 44 | } | |
| 45 | ||
| 46 | /** Whether a worker in this process still holds `runId`. */ | |
| 47 | export const stillEnding = (db: Database, runId: string): boolean => forges.get(db)?.has(runId) ?? false; | |
| 48 | ||
| 49 | /** | |
| 50 | * Call `fn` once the worker holding `runId` has wound down, or at | |
| 51 | * `ENDING_DEADLINE_MS` should it never, whichever is first. Should the | |
| 52 | * deadline come first and the worker let go after, `late` is called then, | |
| 53 | * once, so a page still saying *stopping* hears it has stopped. False, and | |
| 54 | * neither kept, when no worker here holds the run: the caller acts now. | |
| 55 | */ | |
| 56 | export function whenEnded( | |
| 57 | db: Database, | |
| 58 | runId: string, | |
| 59 | fn: () => void, | |
| 60 | late?: () => void, | |
| 61 | deadlineMs = ENDING_DEADLINE_MS, | |
| 62 | ): boolean { | |
| 63 | const waiting = forges.get(db)?.get(runId); | |
| 64 | if (waiting === undefined) return false; | |
| 65 | let timedOut = false; | |
| 66 | const timer = setTimeout( | |
| 67 | () => | |
| 68 | during("run-ending", () => { | |
| 69 | timedOut = true; | |
| 70 | fn(); | |
outdated
Once the deadline fires, |
||
| 71 | }), | |
| 72 | deadlineMs, | |
| 73 | ); | |
| 74 | timer.unref?.(); | |
| 75 | waiting.push(() => { | |
| 76 | if (timedOut) return late?.(); | |
| 77 | clearTimeout(timer); | |
| 78 | fn(); | |
| 79 | }); | |
| 80 | return true; | |
| 81 | } |
src/tui/screens/runlist.ts +12 −2
| @@ -18,6 +18,7 @@ import { getIssuesByIds, type Issue } from "../../core/issues.ts"; | ||
| 18 | 18 | import { canWrite, type Repo } from "../../core/repos.ts"; |
| 19 | 19 | import type { Run, RunState } from "../../core/runs.ts"; |
| 20 | 20 | import { getUsersByIds } from "../../core/users.ts"; |
| 21 | import { stillEnding } from "../../runs/ending.ts"; | |
| 21 | 22 | import type { Ctx, Screen } from "../app/app.ts"; |
| 22 | 23 | import type { Rect, Screen as Surface } from "../term/screen.ts"; |
| 23 | 24 | import type { Tokens } from "../ui/theme.ts"; |
| @@ -39,6 +40,14 @@ import { costText, envLabel, runTone } from "./runwords.ts"; | ||
| 39 | 40 | export const runHollow = (state: RunState): boolean => |
| 40 | 41 | state === "queued" || state === "cancelled"; |
| 41 | 42 | |
| 43 | /** | |
| 44 | * The word a row leads with: the run's state, except a cancelled run whose | |
| 45 | * worker here is still ending the processes it started, which is | |
| 46 | * `stopping` until they have (#13291), as the web's `runWord` says. | |
| 47 | */ | |
| 48 | export const runStateWord = (run: Run, ending: (runId: string) => boolean): string => | |
| 49 | run.state === "cancelled" && ending(run.id) ? "stopping" : run.state; | |
| 50 | ||
| 42 | 51 | /** `deploy production`, `deploy pull #4`, `agent robo`. */ |
| 43 | 52 | export function runTitle( |
| 44 | 53 | run: Run, |
| @@ -75,6 +84,7 @@ export function runRows( | ||
| 75 | 84 | names: ReadonlyMap<string, { username: string }>, |
| 76 | 85 | set: IconSet = "plain", |
| 77 | 86 | issues?: ReadonlyMap<string, Issue>, |
| 87 | ending: (runId: string) => boolean = () => false, | |
| 78 | 88 | ): ListRow[] { |
| 79 | 89 | return runs.map((run) => { |
| 80 | 90 | const tone = runTone(run.state); |
| @@ -83,7 +93,7 @@ export function runRows( | ||
| 83 | 93 | ? undefined |
| 84 | 94 | : issues.get(run.issueId); |
| 85 | 95 | return { |
| 86 | title: `${run.state.padEnd(9)} ${runTitle(run, names, set)}${about === undefined ? "" : ` · #${about.number} ${about.title}`}`, | |
| 96 | title: `${runStateWord(run, ending).padEnd(9)} ${runTitle(run, names, set)}${about === undefined ? "" : ` · #${about.number} ${about.title}`}`, | |
| 87 | 97 | ...(tone === undefined ? {} : { tone }), |
| 88 | 98 | ...(runHollow(run.state) ? { hollow: true } : {}), |
| 89 | 99 | // A queued run held back by a limit says so, and when it resumes — |
| @@ -165,7 +175,7 @@ export class RunList { | ||
| 165 | 175 | r, |
| 166 | 176 | c, |
| 167 | 177 | this.state, |
| 168 | runRows(this.runs, this.names, ctx.icons, this.issues), | |
| 178 | runRows(this.runs, this.names, ctx.icons, this.issues, (id) => stillEnding(this.f.db, id)), | |
| 169 | 179 | focused, |
| 170 | 180 | ); |
| 171 | 181 | hitList(ctx, r, this.state, this.runs.length, { |
src/web/studio/lists.ts +2 −1
| @@ -90,7 +90,8 @@ const WORD_FILTERS: readonly WordFilter[] = [ | ||
| 90 | 90 | ]; |
| 91 | 91 | |
| 92 | 92 | function listBucket(word: Word): WordFilter { |
| 93 | if (word === "waiting" || word === "planned" || word === "silent" || word === "needs input") | |
| 93 | // `stopping` is a cancel still ending its processes: not stopped yet (#13291). | |
New behaviour: |
||
| 94 | if (word === "waiting" || word === "planned" || word === "silent" || word === "needs input" || word === "stopping") | |
| 94 | 95 | return "working"; |
| 95 | 96 | if (word === "done") return "shipped"; |
| 96 | 97 | if (word === "cancelled") return "stopped"; |
src/web/studio/run.ts +4 −1
| @@ -125,6 +125,9 @@ export async function runView( | ||
| 125 | 125 | }; |
| 126 | 126 | const { word } = await runWord(wordCtx, run); |
| 127 | 127 | const unfinished = !isTerminal(run.state); |
| 128 | // A cancelled run still ending its processes keeps its head live, so the | |
| 129 | // word turns from *stopping* to *cancelled* when they have (#13291). | |
| 130 | const live = unfinished || word === "stopping"; | |
New behaviour: the head stays live while the word is |
||
| 128 | 131 | const duration = durationText(run); |
| 129 | 132 | const all = ctx.url.searchParams.get("all") === "1"; |
| 130 | 133 | const held = listHeldSecrets(db, run.id); |
| @@ -150,7 +153,7 @@ export async function runView( | ||
| 150 | 153 | const head = html`<div |
| 151 | 154 | class="page-head" |
| 152 | 155 | id="run-head" |
| 153 | ${unfinished | |
| 156 | ${live | |
| 154 | 157 | ? html`data-live="run.*" data-live-repo="${repo.ownerName}/${repo.name}"` |
| 155 | 158 | : null} |
| 156 | 159 | > |
src/web/studio/words.ts +7
| @@ -44,6 +44,7 @@ import { | ||
| 44 | 44 | import { html, type Html } from "../html.ts"; |
| 45 | 45 | import { icon } from "../ui.ts"; |
| 46 | 46 | import type { Paused } from "../../crew/autonomy.ts"; |
| 47 | import { stillEnding } from "../../runs/ending.ts"; | |
| 47 | 48 | |
| 48 | 49 | /** |
| 49 | 50 | * The empty crew, said once: the shelf's stuck tell, the app page's autonomy |
| @@ -69,6 +70,7 @@ export type Word = | ||
| 69 | 70 | | "told" |
| 70 | 71 | | "done" |
| 71 | 72 | | "cancelled" |
| 73 | | "stopping" | |
| 72 | 74 | | "silent" |
| 73 | 75 | | "needs input"; |
| 74 | 76 | |
| @@ -428,6 +430,10 @@ export async function runWord( | ||
| 428 | 430 | } |
| 429 | 431 | if (run.state === "failed") |
| 430 | 432 | return { word: "stopped", note: run.summary || run.state }; |
| 433 | // Cancelled, but its worker here has not let go yet: the processes it | |
| 434 | // started are still ending, so it is not stopped until they have (#13291). | |
| 435 | if (run.state === "cancelled" && stillEnding(ctx.db, run.id)) | |
| 436 | return { word: "stopping", note: "ending the processes it started" }; | |
| 431 | 437 | // A cancelled run is not a broken one — superseded by a newer push, its |
| 432 | 438 | // environment torn down, a person's own Cancel, or the runtime itself |
| 433 | 439 | // stopping (`STOPPED_SUMMARY`, the one summary that earns a friendlier |
| @@ -558,6 +564,7 @@ const WORD_VARIANT: Record<Word, WordVariant> = { | ||
| 558 | 564 | told: "secondary", |
| 559 | 565 | done: "success", |
| 560 | 566 | cancelled: "secondary", |
| 567 | stopping: "secondary", | |
| 561 | 568 | silent: "danger", |
| 562 | 569 | "needs input": "secondary", |
| 563 | 570 | }; |
test/job-slices.test.ts +1
| @@ -266,6 +266,7 @@ const UNTIMED: Record<string, string> = { | ||
| 266 | 266 | ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger", |
| 267 | 267 | "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly", |
| 268 | 268 | "reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host", |
| 269 | "run-ending": "a one-shot timer in runs/ending.ts, armed only while a cancel waits on a worker here: at its deadline it announces the run once (a row read and one outbox write); run-cancel-stopping.test.ts drives whenEnded directly", | |
| 269 | 270 | lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up", |
| 270 | 271 | "lane-catch-up": "a closure in startChecks, once on boot; same as lane", |
| 271 | 272 | "verify-sweep": "a closure in startChecks, off unless verifyEveryMs is set; same as lane", |
test/run-cancel-stopping.test.ts added +182
| @@ -0,0 +1,182 @@ | ||
| 1 | /** | |
| 2 | * A cancelled run says *stopping* until every process it started has ended, | |
| 3 | * and only then *cancelled* (#13291). The check's step is a stand-in that | |
| 4 | * hears the cancel and holds on until the test lets it go, as a process | |
| 5 | * group that ignores SIGTERM holds on until the grace's SIGKILL (that kill | |
| 6 | * is `test/run-cancel-tree.test.ts`'s). Until it lets go the run's word is | |
| 7 | * `stopping` and no `run.finished` is heard; once it has, the word is | |
| 8 | * `cancelled` and the one `run.finished` is heard, after, not before. | |
| 9 | */ | |
| 10 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 11 | import { Database } from "bun:sqlite"; | |
| 12 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 13 | import { tmpdir } from "node:os"; | |
| 14 | import { join } from "node:path"; | |
| 15 | import { createApp, type App } from "../src/app.ts"; | |
| 16 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 17 | import { cancelRepoRun } from "../src/actions/runs.ts"; | |
| 18 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 19 | import { getRun, listRuns } from "../src/core/runs.ts"; | |
| 20 | import { createUser } from "../src/core/users.ts"; | |
| 21 | import type { runStep } from "../src/apps/process.ts"; | |
| 22 | import { err } from "../src/kernel/result.ts"; | |
| 23 | import { stillEnding, whenEnded, workBegins, workEnds } from "../src/runs/ending.ts"; | |
| 24 | import { runWord, type WordCtx } from "../src/web/studio/words.ts"; | |
| 25 | import { eventVerb } from "../src/design/activity.ts"; | |
| 26 | import type { ForgeEvent } from "../src/core/events.ts"; | |
| 27 | import { quietLogs } from "./helpers/globals.ts"; | |
| 28 | ||
| 29 | quietLogs(); | |
| 30 | process.env["FORGE_CELLS"] = "off"; | |
| 31 | ||
| 32 | let dataDir: string; | |
| 33 | let app: App; | |
| 34 | let checks: ChecksHandle; | |
| 35 | ||
| 36 | // The check's step stands in for a process group that ignores SIGTERM: it | |
| 37 | // hears the cancel, then holds on until the test lets it go, as the group | |
| 38 | // does until the grace's SIGKILL lands (`apps/process.ts` `runStep`). | |
| 39 | let holding = true; | |
| 40 | let letGo!: () => void; | |
| 41 | const released = new Promise<void>((resolve) => (letGo = resolve)); | |
| 42 | let started!: () => void; | |
| 43 | const stepStarted = new Promise<void>((resolve) => (started = resolve)); | |
| 44 | let aborted!: () => void; | |
| 45 | const heardAbort = new Promise<void>((resolve) => (aborted = resolve)); | |
| 46 | const checkStep: typeof runStep = async (_name, _command, options) => { | |
| 47 | started(); | |
| 48 | await new Promise<void>((resolve) => options.signal.addEventListener("abort", () => resolve(), { once: true })); | |
| 49 | aborted(); | |
| 50 | await released; | |
| 51 | return err("check cancelled"); | |
| 52 | }; | |
| 53 | ||
| 54 | beforeAll(async () => { | |
| 55 | dataDir = await mkdtemp(join(tmpdir(), "forge-run-stopping-")); | |
| 56 | app = createApp({ dataDir, port: 0 }); | |
| 57 | checks = startChecks(app, { forgeUrl: "http://forge.test", concurrency: 1, checkStep }); | |
| 58 | }); | |
| 59 | ||
| 60 | afterAll(async () => { | |
| 61 | letGo(); | |
| 62 | await checks.stop(); | |
| 63 | app.close(); | |
| 64 | await rm(dataDir, { recursive: true, force: true }); | |
| 65 | }); | |
| 66 | ||
| 67 | /** A cancelled run's word needs only the database. */ | |
| 68 | const wordOf = async (id: string): Promise<string> => | |
| 69 | (await runWord({ db: app.db } as WordCtx, getRun(app.db, id)!)).word; | |
| 70 | ||
| 71 | describe("the ending registry", () => { | |
| 72 | test("holds a run from its worker's start to its end, and calls a waiter once at the end", () => { | |
| 73 | let heard = 0; | |
| 74 | expect(whenEnded(app.db, "r-none", () => heard++)).toBe(false); | |
| 75 | workBegins(app.db, "r-1"); | |
| 76 | expect(stillEnding(app.db, "r-1")).toBe(true); | |
| 77 | expect(whenEnded(app.db, "r-1", () => heard++)).toBe(true); | |
| 78 | expect(heard).toBe(0); | |
| 79 | workEnds(app.db, "r-1"); | |
| 80 | expect(stillEnding(app.db, "r-1")).toBe(false); | |
| 81 | expect(heard).toBe(1); | |
| 82 | workEnds(app.db, "r-1"); | |
| 83 | expect(heard).toBe(1); | |
| 84 | }); | |
| 85 | ||
| 86 | test("calls a waiter at the deadline when the worker never lets go, and its late call once it does", async () => { | |
| 87 | let heard = 0; | |
| 88 | let late = 0; | |
| 89 | workBegins(app.db, "r-2"); | |
| 90 | whenEnded(app.db, "r-2", () => heard++, () => late++, 20); | |
| 91 | await Bun.sleep(60); // real-timer: the deadline is a real 20 ms timer | |
| 92 | expect([heard, late]).toEqual([1, 0]); | |
| 93 | workEnds(app.db, "r-2"); | |
| 94 | expect([heard, late]).toEqual([1, 1]); | |
| 95 | workEnds(app.db, "r-2"); | |
| 96 | expect([heard, late]).toEqual([1, 1]); | |
| 97 | }); | |
| 98 | ||
| 99 | test("does not make the late call when the worker lets go before the deadline", () => { | |
| 100 | let heard = 0; | |
| 101 | let late = 0; | |
| 102 | workBegins(app.db, "r-4"); | |
| 103 | whenEnded(app.db, "r-4", () => heard++, () => late++, 20); | |
| 104 | workEnds(app.db, "r-4"); | |
| 105 | expect([heard, late]).toEqual([1, 0]); | |
| 106 | }); | |
| 107 | ||
| 108 | test("is one forge's: another forge in the process does not see its runs", () => { | |
| 109 | const other = new Database(":memory:"); | |
| 110 | try { | |
| 111 | workBegins(app.db, "r-3"); | |
| 112 | expect(stillEnding(other, "r-3")).toBe(false); | |
| 113 | expect(whenEnded(other, "r-3", () => {})).toBe(false); | |
| 114 | workEnds(other, "r-3"); | |
| 115 | expect(stillEnding(app.db, "r-3")).toBe(true); | |
| 116 | } finally { | |
| 117 | workEnds(app.db, "r-3"); | |
| 118 | other.close(); | |
| 119 | } | |
| 120 | }); | |
| 121 | }); | |
| 122 | ||
| 123 | describe("a cancelled check whose step has not yet let go", () => { | |
| 124 | test("says stopping until it has, then cancelled, announced once after", async () => { | |
| 125 | const made = await createUser(app.db, { | |
| 126 | origin: "seed", | |
| 127 | trusted: true, | |
| 128 | username: "alice", | |
| 129 | email: "[email protected]", | |
| 130 | password: "a-good-passphrase", | |
| 131 | adminIfFirst: true, | |
| 132 | }); | |
| 133 | if (!made.ok) throw new Error(made.error); | |
| 134 | const alice = made.value; | |
| 135 | const created = await createRepo(app, alice, { name: "deaf", visibility: "public" }); | |
| 136 | if (!created.ok) throw new Error(String(created.error)); | |
| 137 | const repo = created.value; | |
| 138 | ||
| 139 | const pushed = await commitToBranch(app, alice, repo, { | |
| 140 | message: "A check whose step outlives its cancel", | |
| 141 | files: [{ path: "forge.json", content: `${JSON.stringify({ check: { run: "true", install: "" } })}\n` }], | |
| 142 | }); | |
| 143 | if (!pushed.ok) throw new Error(JSON.stringify(pushed.error)); | |
| 144 | await stepStarted; | |
| 145 | const run = listRuns(app.db, repo.id, { kind: "check", limit: 5 }).find( | |
| 146 | (r) => r.commitOid === pushed.value.commitOid, | |
| 147 | ); | |
| 148 | if (run === undefined) throw new Error("no check run"); | |
| 149 | ||
| 150 | // Each `run.finished` for this run, and whether the step still held on when heard. | |
| 151 | const finished: boolean[] = []; | |
| 152 | const off = app.bus.subscribe((e) => { | |
| 153 | if (e.kind === "run.finished" && (e.payload as { runId?: string }).runId === run.id) | |
| 154 | finished.push(holding); | |
| 155 | }); | |
| 156 | try { | |
| 157 | const cancelled = cancelRepoRun(app, alice, repo, run.id); | |
| 158 | expect(cancelled.ok).toBe(true); | |
| 159 | // The row is cancelled at once, but the step has not let go yet. | |
| 160 | expect(getRun(app.db, run.id)?.state).toBe("cancelled"); | |
| 161 | await heardAbort; | |
| 162 | expect(await wordOf(run.id)).toBe("stopping"); | |
| 163 | expect(finished).toEqual([]); | |
| 164 | ||
| 165 | // The step lets go, as a process group does once the kill lands. | |
| 166 | holding = false; | |
| 167 | letGo(); | |
| 168 | while (stillEnding(app.db, run.id)) await Bun.sleep(1); // real-timer: the worker's finally runs on its own turn | |
| 169 | expect(await wordOf(run.id)).toBe("cancelled"); | |
| 170 | // Heard once, and only after the step had let go. | |
| 171 | expect(finished).toEqual([false]); | |
| 172 | } finally { | |
| 173 | off(); | |
| 174 | } | |
| 175 | }); | |
| 176 | }); | |
| 177 | ||
| 178 | test("the late wake-up reads as what it is in the activity feed", () => { | |
| 179 | expect(eventVerb({ kind: "run.stopped", payload: {} } as unknown as ForgeEvent)).toBe( | |
| 180 | "ended a cancelled run's last process", | |
| 181 | ); | |
| 182 | }); |
test/studio-runs.test.ts +41
| @@ -42,6 +42,7 @@ import type { Repo } from "../src/core/repos.ts"; | ||
| 42 | 42 | import { |
| 43 | 43 | claimRun, |
| 44 | 44 | createRun, |
| 45 | cancelRun, | |
| 45 | 46 | finishRun, |
| 46 | 47 | getRun, |
| 47 | 48 | markServing, |
| @@ -58,6 +59,7 @@ import { | ||
| 58 | 59 | } from "../src/web/studio/words.ts"; |
| 59 | 60 | import { repoDir } from "../src/git/repos.ts"; |
| 60 | 61 | import { appendRunLog } from "../src/runs/log.ts"; |
| 62 | import { workBegins, workEnds } from "../src/runs/ending.ts"; | |
| 61 | 63 | import { withScripts } from "./helpers/page-script.ts"; |
| 62 | 64 | |
| 63 | 65 | describe("the studio's slot count", () => { |
| @@ -753,6 +755,45 @@ describe("working: a run's own page in the studio", () => { | ||
| 753 | 755 | expect(over).not.toContain('id="run-stop"'); |
| 754 | 756 | }); |
| 755 | 757 | |
| 758 | test("a cancelled run still ending its processes keeps its head live and says stopping, then cancelled (#13291)", async () => { | |
| 759 | const run = createRun(app.db, { repoId: repo.id, kind: "agent", actorId: scoutId, reason: "assigned" }); | |
| 760 | claimRun(app.db, run.id); | |
| 761 | workBegins(app.db, run.id); | |
| 762 | cancelRun(app.db, run.id); | |
| 763 | const path = `/alice/engine/runs/${run.id}`; | |
| 764 | const head = (html: string): string => | |
| 765 | html.slice(html.indexOf('id="run-head"'), html.indexOf('id="run-live"')); | |
| 766 | try { | |
| 767 | const ending = await (await get(path, writerCookie)).text(); | |
| 768 | expect(ending).toMatch(/<div\s+class="page-head"\s+id="run-head"\s+data-live="run\.\*"/); | |
| 769 | expect(head(ending)).toContain('data-word="stopping"'); | |
| 770 | } finally { | |
| 771 | workEnds(app.db, run.id); | |
| 772 | } | |
| 773 | const ended = await (await get(path, writerCookie)).text(); | |
| 774 | expect(ended).not.toMatch(/id="run-head"[^>]*data-live=/); | |
| 775 | expect(head(ended)).toContain('data-word="cancelled"'); | |
| 776 | }); | |
| 777 | ||
| 778 | test("the runs list files a stopping run under working, not stopped, until its processes have ended (#13291)", async () => { | |
| 779 | const run = createRun(app.db, { repoId: repo.id, kind: "agent", actorId: scoutId, reason: "assigned" }); | |
| 780 | claimRun(app.db, run.id); | |
| 781 | workBegins(app.db, run.id); | |
| 782 | cancelRun(app.db, run.id); | |
| 783 | const link = `/alice/engine/runs/${run.id}`; | |
| 784 | const list = async (query: string): Promise<string> => | |
| 785 | (await get(`/alice/engine/runs?${query}`, writerCookie)).text(); | |
| 786 | try { | |
| 787 | // The default agent list shows what is open and moving: a stopping run is. | |
| 788 | expect(await list("kind=agent")).toContain(link); | |
| 789 | expect(await list("kind=agent&word=stopped")).not.toContain(link); | |
| 790 | } finally { | |
| 791 | workEnds(app.db, run.id); | |
| 792 | } | |
| 793 | expect(await list("kind=agent")).not.toContain(link); | |
| 794 | expect(await list("kind=agent&word=stopped")).toContain(link); | |
| 795 | }); | |
| 796 | ||
| 756 | 797 | test("a long transcript holds its last window, and the whole of it is one step deeper at ?all=1", async () => { |
| 757 | 798 | const run = createRun(app.db, { |
| 758 | 799 | repoId: repo.id, |
test/tui-run-cancel.test.ts +22
| @@ -8,6 +8,7 @@ import { createApp, type App as Forge } from "../src/app.ts"; | ||
| 8 | 8 | import { claimRun, createRun, finishRun, getRun, type RunKind } from "../src/core/runs.ts"; |
| 9 | 9 | import type { Repo } from "../src/core/repos.ts"; |
| 10 | 10 | import { createUser, type User } from "../src/core/users.ts"; |
| 11 | import { workBegins, workEnds } from "../src/runs/ending.ts"; | |
| 11 | 12 | import { createForgeApp } from "../src/tui/forge.ts"; |
| 12 | 13 | import { runsScreen } from "../src/tui/screens/runs.ts"; |
| 13 | 14 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| @@ -109,6 +110,27 @@ test("a running run offers x cancel too", async () => { | ||
| 109 | 110 | h.app.stop(); |
| 110 | 111 | }); |
| 111 | 112 | |
| 113 | test("a cancelled run still ending its processes reads stopping, and cancelled once they have (#13291)", async () => { | |
| 114 | const h = await open(alice, (repo) => { | |
| 115 | const run = createRun(forge.db, { repoId: repo.id, kind: "deploy", env: "production" }); | |
| 116 | claimRun(forge.db, run.id); | |
| 117 | workBegins(forge.db, run.id); | |
| 118 | return run.id; | |
| 119 | }); | |
| 120 | try { | |
| 121 | await h.press("x"); | |
| 122 | await h.press("left"); | |
| 123 | const text = await h.press("enter"); | |
| 124 | expect(state(h.runId)).toBe("cancelled"); | |
| 125 | expect(text).toMatch(/stopping +deploy production/); | |
| 126 | expect(text).not.toMatch(/cancelled +deploy production/); | |
| 127 | } finally { | |
| 128 | workEnds(forge.db, h.runId); | |
| 129 | } | |
| 130 | expect(await h.press("r")).toMatch(/cancelled +deploy production/); | |
| 131 | h.app.stop(); | |
| 132 | }); | |
| 133 | ||
| 112 | 134 | test("x does nothing on a finished run", async () => { |
| 113 | 135 | const h = await open(alice, (repo) => { |
| 114 | 136 | const run = createRun(forge.db, { repoId: repo.id, kind: "check" }); |
This describes a grandchild that ignores SIGTERM, a real process and a check that finds none alive. The test now uses a stand-in step and starts no process, so the line is false. Reword it to what the test does and point at the test that proves the processes end.