Public: anyone can read this; the crew can change it.

A cancelled run ends every process it started, and a test proves none is left #13341

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13291-a-cancelled-run-ends-every-process-it-st → main 9 commits fork 5 threads · 5 unresolved

  • Merged. mason/13291-a-cancelled-run-ends-every-process-it-st was merged into main as 5df0d61.

Review

Sign in to review this change.
15 files changed +394 −7
AGENTS.md +1 −1

Diff not shown — a line 2,989 characters wide (limit 2,000). +1 −1

View file
docs/features/run-state.md +1
@@ -8,3 +8,4 @@
88 - `GET …/runs?state=queued|running|ok|failed|cancelled` filters; an unknown state is a 422.
99 - **In the terminal**: the issue screen (`src/tui/screens/issue.ts`, rows from `src/tui/screens/runplan.ts`) shows a *Plan* card above the thread — `[x]`/`[ ]` items and the question as `needs input: …` — read through the same `readPlan` (so gated as the page is), and its title says *silent* or *needs input* when the issue's runs read that way, from `shownStates`.
1010 - The runs API works out `shown` for a page of runs with one plan read (`shownStates`), not one per row.
11 - **Stopping**: a cancel marks the row at once, but the run's page says *stopping* (a moving word, filed under *working*) until its worker here has ended every process the run started — SIGTERM to the group, SIGKILL once the grace runs out, an agent's orphans reaped — and only then *cancelled*; the cancel's one `run.finished` is announced at that moment, not before (`src/runs/ending.ts`, #13291; `test/run-cancel-stopping.test.ts` cancels a check whose stand-in step holds on after the abort, and finds the word *stopping* and no `run.finished` until the step lets go, then *cancelled* and the one `run.finished` after). That none of the processes is left is `test/run-cancel-tree.test.ts`'s: a cancelled check's command, the child it forked and that child's child are all gone within 2 s, and a step and the sleeper it forked that both ignore SIGTERM are killed once the grace runs out. A worker that never lets go is announced at the 30 s deadline (`ENDING_DEADLINE_MS`) while the page still says *stopping*; when it lets go later, a `run.stopped` wake-up (not a second `run.finished`) turns the live head to *cancelled*.
outdated

This describes a grandchild that ignores SIGTERM, a real process and a check that finds none alive. The test now uses a stand-in step and starts no process, so the line is false. Reword it to what the test does and point at the test that proves the processes end.

src/actions/runs.ts +29 −2
@@ -36,6 +36,8 @@ import {
3636 import { getUsersByIds, type User } from "../core/users.ts";
3737 import { abortAgentRun } from "../crew/running.ts";
3838 import { stopRun } from "../runs/stoppable.ts";
39 import { whenEnded } from "../runs/ending.ts";
40 import { createLog } from "../kernel/log.ts";
3941 import { queueAgentRun, taskOf } from "../crew/dispatch.ts";
4042 import { err, ok, type Result } from "../kernel/result.ts";
4143 import { readRunLog, readRunLogTail, runLogPath } from "../runs/log.ts";
@@ -45,6 +47,8 @@ import { cancelledBy } from "../runs/cancelled.ts";
4547 import { requireReader, requireWriter, type AccessError } from "./access.ts";
4648 import { deskForRun } from "../crew/desk_posts.ts";
4749
50 const log = createLog("runs");
51
4852 /** What a run event carries: enough for a surface to render the card. */
4953 export function runPayload(run: Run): Record<string, unknown> {
5054 return {
@@ -374,10 +378,33 @@ export function cancelRepoRun(
374378 );
375379 if (!cancelled) return err("finished");
376380 // The row says cancelled first, so the run's own end leaves it so; the
377 // abort then ends the process group it started.
381 // abort then ends the process group it started. A run worked here is
382 // announced once its worker has let go, every process ended (#13291);
383 // one worked nowhere here is announced now. Should the worker let go only
384 // after the deadline announced it, `run.stopped` wakes the page still
385 // saying *stopping*: a wake-up, not a second `run.finished`.
386 const heard = (what: string, say: () => void): void => {
387 try {
388 say();
389 } catch (error) {
390 // The forge closed while the run wound down: nobody is left to hear.
391 log.warn(`a cancel's ${what} could not be announced`, { run: runId, error: String(error) });
392 }
393 };
394 const announce = (): void =>
395 heard("end", () => announceRun(app, getRun(app.db, runId) ?? cancelled, writer.value.id));
396 const stopped = (): void =>
397 heard("stop", () =>
398 app.bus.append({
399 kind: "run.stopped",
400 repoId: cancelled.repoId,
401 payload: { runId, kind: cancelled.kind, env: cancelled.env, commitOid: cancelled.commitOid },
402 }),
403 );
404 const waits = whenEnded(app.db, runId, announce, stopped);
378405 if (run.kind === "agent") abortAgentRun(runId);
379406 else stopRun(runId);
380 announceRun(app, cancelled, writer.value.id);
407 if (!waits) announce();
381408 return ok(cancelled);
382409 }
383410
src/checks/service.ts +5
@@ -106,6 +106,7 @@ import { appendRunLog } from "../runs/log.ts";
106106 import { memoryHold } from "../runs/memory_hold.ts";
107107 import { cancelledLine } from "../runs/cancelled.ts";
108108 import { trackRun, untrackRun } from "../runs/stoppable.ts";
109 import { workBegins, workEnds } from "../runs/ending.ts";
109110 import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
110111 import { exportTree, readFileAt } from "../apps/checkout.ts";
111112 import { CHECK_TIMEOUT_MAX, parseCheck } from "../apps/manifest.ts";
@@ -639,6 +640,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
639640 if (low) verifying = run.id;
640641 // A person's cancel (`cancelRepoRun`) reaches it through this.
641642 trackRun(run.id, abort);
643 workBegins(app.db, run.id);
642644 const promise = execute(run, repo, abort.signal, !low)
643645 .catch((error) =>
644646 log.error("check failed to execute", {
@@ -648,6 +650,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
648650 )
649651 .finally(() => {
650652 untrackRun(run.id, abort);
653 // Every process the check started has ended: a cancel waiting on
654 // that is heard now (#13291).
655 workEnds(app.db, run.id);
651656 inFlight.delete(run.id);
652657 if (verifying === run.id) verifying = undefined;
653658 // A pull's green that ran while its base moved is of the old tip: asked again once.
src/crew/runner.ts +5
@@ -181,6 +181,7 @@ import {
181181 type UsageOutcome,
182182 } from "../core/usage.ts";
183183 import { trackAgentRun, untrackAgentRun } from "./running.ts";
184 import { workBegins, workEnds } from "../runs/ending.ts";
184185 import { installCacheEnv, sandboxPolicy } from "./sandbox.ts";
185186 import { runGitAuth } from "./git_auth.ts";
186187 import { grantSecrets, needsYou, parseDeclaration, ungrantedSentence, type Grant } from "./grant.ts";
@@ -427,6 +428,9 @@ export async function executeAgentRun(
427428 const cancelled = cancelledLine(getRun(app.db, run.id));
428429 if (cancelled !== undefined) appendRunLog(app.config.dataDir, run.id, cancelled);
429430 }
431 // The agent and the orphans it left are reaped: a cancel waiting on
432 // that is heard now (#13291).
433 workEnds(app.db, run.id);
430434 }
431435 }
432436
@@ -460,6 +464,7 @@ async function runClaimed(
460464 if (startedAt !== undefined) setRunBase(app.db, run.id, startedAt);
461465 const controller = new AbortController();
462466 trackAgentRun(run.id, controller);
467 workBegins(app.db, run.id);
463468 const write = (line: string): void =>
464469 appendRunLog(app.config.dataDir, run.id, line);
465470 announceRun(app, claimed);
src/design/activity.ts +1
@@ -231,6 +231,7 @@ const WORDS: Record<string, string> = {
231231 "desk.cleared": "cleared a note from the desk",
232232 "run.queued": "queued a run",
233233 "run.started": "started a run",
234 "run.stopped": "ended a cancelled run's last process",
234235 "schedule.steered": "steered a schedule",
235236 "app.created": "made it an app",
236237 "app.updated": "changed the app's settings",
src/runs/ending.ts added +81
@@ -0,0 +1,81 @@
1 /**
2 * The runs whose worker in this process has not yet wound down, by run id
3 * (#13291). A cancel marks the row at once, but the processes the run
4 * started end a moment later (SIGTERM, then SIGKILL once the grace runs
5 * out; an agent's orphans are reaped after it). Until the worker has let
6 * go, the run is *stopping*, not stopped: its page says so (`runWord`), and
7 * the cancel's one `run.finished` waits here (`whenEnded`) so no surface
8 * hears "stopped" before every process is gone.
9 *
10 * The checks service (`checks/service.ts` `start`) and the agent runner
11 * (`crew/runner.ts` `executeAgentRun`) mark each run they work from claim
12 * to their last line. At most one entry per run in flight here; gone on
13 * restart, when nothing is in flight. The state is the forge's, keyed by its
14 * database (#12569): two forges in one process never see each other's runs.
15 */
16
17 import type { Database } from "bun:sqlite";
18 import { during } from "../perf/jobs.ts";
19
20 /** How long a cancel's announcement waits for a worker that never lets go. */
21 export const ENDING_DEADLINE_MS = 30_000;
22
23 const forges = new WeakMap<Database, Map<string, Array<() => void>>>();
24
25 /** The runs a forge's workers here still hold, by its database. */
26 function endingOf(db: Database): Map<string, Array<() => void>> {
27 let held = forges.get(db);
28 if (held === undefined) forges.set(db, (held = new Map()));
29 return held;
30 }
31
32 /** A worker in this process starts on `runId`. */
33 export function workBegins(db: Database, runId: string): void {
34 const held = endingOf(db);
35 if (!held.has(runId)) held.set(runId, []);
36 }
37
38 /** The worker has let `runId` go: every process it started has ended. */
39 export function workEnds(db: Database, runId: string): void {
40 const held = endingOf(db);
41 const waiting = held.get(runId);
42 held.delete(runId);
43 for (const fn of waiting ?? []) fn();
44 }
45
46 /** Whether a worker in this process still holds `runId`. */
47 export const stillEnding = (db: Database, runId: string): boolean => forges.get(db)?.has(runId) ?? false;
48
49 /**
50 * Call `fn` once the worker holding `runId` has wound down, or at
51 * `ENDING_DEADLINE_MS` should it never, whichever is first. Should the
52 * deadline come first and the worker let go after, `late` is called then,
53 * once, so a page still saying *stopping* hears it has stopped. False, and
54 * neither kept, when no worker here holds the run: the caller acts now.
55 */
56 export function whenEnded(
57 db: Database,
58 runId: string,
59 fn: () => void,
60 late?: () => void,
61 deadlineMs = ENDING_DEADLINE_MS,
62 ): boolean {
63 const waiting = forges.get(db)?.get(runId);
64 if (waiting === undefined) return false;
65 let timedOut = false;
66 const timer = setTimeout(
67 () =>
68 during("run-ending", () => {
69 timedOut = true;
70 fn();
outdated

Once the deadline fires, once is spent and the page has no later event to turn stopping into cancelled. Let workEnds send one more run.* announcement when the deadline already fired, or document that the page stays stopping.

71 }),
72 deadlineMs,
73 );
74 timer.unref?.();
75 waiting.push(() => {
76 if (timedOut) return late?.();
77 clearTimeout(timer);
78 fn();
79 });
80 return true;
81 }
src/tui/screens/runlist.ts +12 −2
@@ -18,6 +18,7 @@ import { getIssuesByIds, type Issue } from "../../core/issues.ts";
1818 import { canWrite, type Repo } from "../../core/repos.ts";
1919 import type { Run, RunState } from "../../core/runs.ts";
2020 import { getUsersByIds } from "../../core/users.ts";
21 import { stillEnding } from "../../runs/ending.ts";
2122 import type { Ctx, Screen } from "../app/app.ts";
2223 import type { Rect, Screen as Surface } from "../term/screen.ts";
2324 import type { Tokens } from "../ui/theme.ts";
@@ -39,6 +40,14 @@ import { costText, envLabel, runTone } from "./runwords.ts";
3940 export const runHollow = (state: RunState): boolean =>
4041 state === "queued" || state === "cancelled";
4142
43 /**
44 * The word a row leads with: the run's state, except a cancelled run whose
45 * worker here is still ending the processes it started, which is
46 * `stopping` until they have (#13291), as the web's `runWord` says.
47 */
48 export const runStateWord = (run: Run, ending: (runId: string) => boolean): string =>
49 run.state === "cancelled" && ending(run.id) ? "stopping" : run.state;
50
4251 /** `deploy production`, `deploy pull #4`, `agent robo`. */
4352 export function runTitle(
4453 run: Run,
@@ -75,6 +84,7 @@ export function runRows(
7584 names: ReadonlyMap<string, { username: string }>,
7685 set: IconSet = "plain",
7786 issues?: ReadonlyMap<string, Issue>,
87 ending: (runId: string) => boolean = () => false,
7888 ): ListRow[] {
7989 return runs.map((run) => {
8090 const tone = runTone(run.state);
@@ -83,7 +93,7 @@ export function runRows(
8393 ? undefined
8494 : issues.get(run.issueId);
8595 return {
86 title: `${run.state.padEnd(9)} ${runTitle(run, names, set)}${about === undefined ? "" : ` · #${about.number} ${about.title}`}`,
96 title: `${runStateWord(run, ending).padEnd(9)} ${runTitle(run, names, set)}${about === undefined ? "" : ` · #${about.number} ${about.title}`}`,
8797 ...(tone === undefined ? {} : { tone }),
8898 ...(runHollow(run.state) ? { hollow: true } : {}),
8999 // A queued run held back by a limit says so, and when it resumes —
@@ -165,7 +175,7 @@ export class RunList {
165175 r,
166176 c,
167177 this.state,
168 runRows(this.runs, this.names, ctx.icons, this.issues),
178 runRows(this.runs, this.names, ctx.icons, this.issues, (id) => stillEnding(this.f.db, id)),
169179 focused,
170180 );
171181 hitList(ctx, r, this.state, this.runs.length, {
src/web/studio/lists.ts +2 −1
@@ -90,7 +90,8 @@ const WORD_FILTERS: readonly WordFilter[] = [
9090 ];
9191
9292 function listBucket(word: Word): WordFilter {
93 if (word === "waiting" || word === "planned" || word === "silent" || word === "needs input")
93 // `stopping` is a cancel still ending its processes: not stopped yet (#13291).

New behaviour: stopping is filed under working. No test would fail without this line. Add one.

94 if (word === "waiting" || word === "planned" || word === "silent" || word === "needs input" || word === "stopping")
9495 return "working";
9596 if (word === "done") return "shipped";
9697 if (word === "cancelled") return "stopped";
src/web/studio/run.ts +4 −1
@@ -125,6 +125,9 @@ export async function runView(
125125 };
126126 const { word } = await runWord(wordCtx, run);
127127 const unfinished = !isTerminal(run.state);
128 // A cancelled run still ending its processes keeps its head live, so the
129 // word turns from *stopping* to *cancelled* when they have (#13291).
130 const live = unfinished || word === "stopping";

New behaviour: the head stays live while the word is stopping. No test covers it, such as a page render with data-live for a cancelled run that is still ending. Add one.

128131 const duration = durationText(run);
129132 const all = ctx.url.searchParams.get("all") === "1";
130133 const held = listHeldSecrets(db, run.id);
@@ -150,7 +153,7 @@ export async function runView(
150153 const head = html`<div
151154 class="page-head"
152155 id="run-head"
153 ${unfinished
156 ${live
154157 ? html`data-live="run.*" data-live-repo="${repo.ownerName}/${repo.name}"`
155158 : null}
156159 >
src/web/studio/words.ts +7
@@ -44,6 +44,7 @@ import {
4444 import { html, type Html } from "../html.ts";
4545 import { icon } from "../ui.ts";
4646 import type { Paused } from "../../crew/autonomy.ts";
47 import { stillEnding } from "../../runs/ending.ts";
4748
4849 /**
4950 * The empty crew, said once: the shelf's stuck tell, the app page's autonomy
@@ -69,6 +70,7 @@ export type Word =
6970 | "told"
7071 | "done"
7172 | "cancelled"
73 | "stopping"
7274 | "silent"
7375 | "needs input";
7476
@@ -428,6 +430,10 @@ export async function runWord(
428430 }
429431 if (run.state === "failed")
430432 return { word: "stopped", note: run.summary || run.state };
433 // Cancelled, but its worker here has not let go yet: the processes it
434 // started are still ending, so it is not stopped until they have (#13291).
435 if (run.state === "cancelled" && stillEnding(ctx.db, run.id))
436 return { word: "stopping", note: "ending the processes it started" };
431437 // A cancelled run is not a broken one — superseded by a newer push, its
432438 // environment torn down, a person's own Cancel, or the runtime itself
433439 // stopping (`STOPPED_SUMMARY`, the one summary that earns a friendlier
@@ -558,6 +564,7 @@ const WORD_VARIANT: Record<Word, WordVariant> = {
558564 told: "secondary",
559565 done: "success",
560566 cancelled: "secondary",
567 stopping: "secondary",
561568 silent: "danger",
562569 "needs input": "secondary",
563570 };
test/job-slices.test.ts +1
@@ -266,6 +266,7 @@ const UNTIMED: Record<string, string> = {
266266 ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger",
267267 "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly",
268268 "reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host",
269 "run-ending": "a one-shot timer in runs/ending.ts, armed only while a cancel waits on a worker here: at its deadline it announces the run once (a row read and one outbox write); run-cancel-stopping.test.ts drives whenEnded directly",
269270 lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up",
270271 "lane-catch-up": "a closure in startChecks, once on boot; same as lane",
271272 "verify-sweep": "a closure in startChecks, off unless verifyEveryMs is set; same as lane",
test/run-cancel-stopping.test.ts added +182
@@ -0,0 +1,182 @@
1 /**
2 * A cancelled run says *stopping* until every process it started has ended,
3 * and only then *cancelled* (#13291). The check's step is a stand-in that
4 * hears the cancel and holds on until the test lets it go, as a process
5 * group that ignores SIGTERM holds on until the grace's SIGKILL (that kill
6 * is `test/run-cancel-tree.test.ts`'s). Until it lets go the run's word is
7 * `stopping` and no `run.finished` is heard; once it has, the word is
8 * `cancelled` and the one `run.finished` is heard, after, not before.
9 */
10 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
11 import { Database } from "bun:sqlite";
12 import { mkdtemp, rm } from "node:fs/promises";
13 import { tmpdir } from "node:os";
14 import { join } from "node:path";
15 import { createApp, type App } from "../src/app.ts";
16 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
17 import { cancelRepoRun } from "../src/actions/runs.ts";
18 import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
19 import { getRun, listRuns } from "../src/core/runs.ts";
20 import { createUser } from "../src/core/users.ts";
21 import type { runStep } from "../src/apps/process.ts";
22 import { err } from "../src/kernel/result.ts";
23 import { stillEnding, whenEnded, workBegins, workEnds } from "../src/runs/ending.ts";
24 import { runWord, type WordCtx } from "../src/web/studio/words.ts";
25 import { eventVerb } from "../src/design/activity.ts";
26 import type { ForgeEvent } from "../src/core/events.ts";
27 import { quietLogs } from "./helpers/globals.ts";
28
29 quietLogs();
30 process.env["FORGE_CELLS"] = "off";
31
32 let dataDir: string;
33 let app: App;
34 let checks: ChecksHandle;
35
36 // The check's step stands in for a process group that ignores SIGTERM: it
37 // hears the cancel, then holds on until the test lets it go, as the group
38 // does until the grace's SIGKILL lands (`apps/process.ts` `runStep`).
39 let holding = true;
40 let letGo!: () => void;
41 const released = new Promise<void>((resolve) => (letGo = resolve));
42 let started!: () => void;
43 const stepStarted = new Promise<void>((resolve) => (started = resolve));
44 let aborted!: () => void;
45 const heardAbort = new Promise<void>((resolve) => (aborted = resolve));
46 const checkStep: typeof runStep = async (_name, _command, options) => {
47 started();
48 await new Promise<void>((resolve) => options.signal.addEventListener("abort", () => resolve(), { once: true }));
49 aborted();
50 await released;
51 return err("check cancelled");
52 };
53
54 beforeAll(async () => {
55 dataDir = await mkdtemp(join(tmpdir(), "forge-run-stopping-"));
56 app = createApp({ dataDir, port: 0 });
57 checks = startChecks(app, { forgeUrl: "http://forge.test", concurrency: 1, checkStep });
58 });
59
60 afterAll(async () => {
61 letGo();
62 await checks.stop();
63 app.close();
64 await rm(dataDir, { recursive: true, force: true });
65 });
66
67 /** A cancelled run's word needs only the database. */
68 const wordOf = async (id: string): Promise<string> =>
69 (await runWord({ db: app.db } as WordCtx, getRun(app.db, id)!)).word;
70
71 describe("the ending registry", () => {
72 test("holds a run from its worker's start to its end, and calls a waiter once at the end", () => {
73 let heard = 0;
74 expect(whenEnded(app.db, "r-none", () => heard++)).toBe(false);
75 workBegins(app.db, "r-1");
76 expect(stillEnding(app.db, "r-1")).toBe(true);
77 expect(whenEnded(app.db, "r-1", () => heard++)).toBe(true);
78 expect(heard).toBe(0);
79 workEnds(app.db, "r-1");
80 expect(stillEnding(app.db, "r-1")).toBe(false);
81 expect(heard).toBe(1);
82 workEnds(app.db, "r-1");
83 expect(heard).toBe(1);
84 });
85
86 test("calls a waiter at the deadline when the worker never lets go, and its late call once it does", async () => {
87 let heard = 0;
88 let late = 0;
89 workBegins(app.db, "r-2");
90 whenEnded(app.db, "r-2", () => heard++, () => late++, 20);
91 await Bun.sleep(60); // real-timer: the deadline is a real 20 ms timer
92 expect([heard, late]).toEqual([1, 0]);
93 workEnds(app.db, "r-2");
94 expect([heard, late]).toEqual([1, 1]);
95 workEnds(app.db, "r-2");
96 expect([heard, late]).toEqual([1, 1]);
97 });
98
99 test("does not make the late call when the worker lets go before the deadline", () => {
100 let heard = 0;
101 let late = 0;
102 workBegins(app.db, "r-4");
103 whenEnded(app.db, "r-4", () => heard++, () => late++, 20);
104 workEnds(app.db, "r-4");
105 expect([heard, late]).toEqual([1, 0]);
106 });
107
108 test("is one forge's: another forge in the process does not see its runs", () => {
109 const other = new Database(":memory:");
110 try {
111 workBegins(app.db, "r-3");
112 expect(stillEnding(other, "r-3")).toBe(false);
113 expect(whenEnded(other, "r-3", () => {})).toBe(false);
114 workEnds(other, "r-3");
115 expect(stillEnding(app.db, "r-3")).toBe(true);
116 } finally {
117 workEnds(app.db, "r-3");
118 other.close();
119 }
120 });
121 });
122
123 describe("a cancelled check whose step has not yet let go", () => {
124 test("says stopping until it has, then cancelled, announced once after", async () => {
125 const made = await createUser(app.db, {
126 origin: "seed",
127 trusted: true,
128 username: "alice",
129 email: "[email protected]",
130 password: "a-good-passphrase",
131 adminIfFirst: true,
132 });
133 if (!made.ok) throw new Error(made.error);
134 const alice = made.value;
135 const created = await createRepo(app, alice, { name: "deaf", visibility: "public" });
136 if (!created.ok) throw new Error(String(created.error));
137 const repo = created.value;
138
139 const pushed = await commitToBranch(app, alice, repo, {
140 message: "A check whose step outlives its cancel",
141 files: [{ path: "forge.json", content: `${JSON.stringify({ check: { run: "true", install: "" } })}\n` }],
142 });
143 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
144 await stepStarted;
145 const run = listRuns(app.db, repo.id, { kind: "check", limit: 5 }).find(
146 (r) => r.commitOid === pushed.value.commitOid,
147 );
148 if (run === undefined) throw new Error("no check run");
149
150 // Each `run.finished` for this run, and whether the step still held on when heard.
151 const finished: boolean[] = [];
152 const off = app.bus.subscribe((e) => {
153 if (e.kind === "run.finished" && (e.payload as { runId?: string }).runId === run.id)
154 finished.push(holding);
155 });
156 try {
157 const cancelled = cancelRepoRun(app, alice, repo, run.id);
158 expect(cancelled.ok).toBe(true);
159 // The row is cancelled at once, but the step has not let go yet.
160 expect(getRun(app.db, run.id)?.state).toBe("cancelled");
161 await heardAbort;
162 expect(await wordOf(run.id)).toBe("stopping");
163 expect(finished).toEqual([]);
164
165 // The step lets go, as a process group does once the kill lands.
166 holding = false;
167 letGo();
168 while (stillEnding(app.db, run.id)) await Bun.sleep(1); // real-timer: the worker's finally runs on its own turn
169 expect(await wordOf(run.id)).toBe("cancelled");
170 // Heard once, and only after the step had let go.
171 expect(finished).toEqual([false]);
172 } finally {
173 off();
174 }
175 });
176 });
177
178 test("the late wake-up reads as what it is in the activity feed", () => {
179 expect(eventVerb({ kind: "run.stopped", payload: {} } as unknown as ForgeEvent)).toBe(
180 "ended a cancelled run's last process",
181 );
182 });
test/studio-runs.test.ts +41
@@ -42,6 +42,7 @@ import type { Repo } from "../src/core/repos.ts";
4242 import {
4343 claimRun,
4444 createRun,
45 cancelRun,
4546 finishRun,
4647 getRun,
4748 markServing,
@@ -58,6 +59,7 @@ import {
5859 } from "../src/web/studio/words.ts";
5960 import { repoDir } from "../src/git/repos.ts";
6061 import { appendRunLog } from "../src/runs/log.ts";
62 import { workBegins, workEnds } from "../src/runs/ending.ts";
6163 import { withScripts } from "./helpers/page-script.ts";
6264
6365 describe("the studio's slot count", () => {
@@ -753,6 +755,45 @@ describe("working: a run's own page in the studio", () => {
753755 expect(over).not.toContain('id="run-stop"');
754756 });
755757
758 test("a cancelled run still ending its processes keeps its head live and says stopping, then cancelled (#13291)", async () => {
759 const run = createRun(app.db, { repoId: repo.id, kind: "agent", actorId: scoutId, reason: "assigned" });
760 claimRun(app.db, run.id);
761 workBegins(app.db, run.id);
762 cancelRun(app.db, run.id);
763 const path = `/alice/engine/runs/${run.id}`;
764 const head = (html: string): string =>
765 html.slice(html.indexOf('id="run-head"'), html.indexOf('id="run-live"'));
766 try {
767 const ending = await (await get(path, writerCookie)).text();
768 expect(ending).toMatch(/<div\s+class="page-head"\s+id="run-head"\s+data-live="run\.\*"/);
769 expect(head(ending)).toContain('data-word="stopping"');
770 } finally {
771 workEnds(app.db, run.id);
772 }
773 const ended = await (await get(path, writerCookie)).text();
774 expect(ended).not.toMatch(/id="run-head"[^>]*data-live=/);
775 expect(head(ended)).toContain('data-word="cancelled"');
776 });
777
778 test("the runs list files a stopping run under working, not stopped, until its processes have ended (#13291)", async () => {
779 const run = createRun(app.db, { repoId: repo.id, kind: "agent", actorId: scoutId, reason: "assigned" });
780 claimRun(app.db, run.id);
781 workBegins(app.db, run.id);
782 cancelRun(app.db, run.id);
783 const link = `/alice/engine/runs/${run.id}`;
784 const list = async (query: string): Promise<string> =>
785 (await get(`/alice/engine/runs?${query}`, writerCookie)).text();
786 try {
787 // The default agent list shows what is open and moving: a stopping run is.
788 expect(await list("kind=agent")).toContain(link);
789 expect(await list("kind=agent&word=stopped")).not.toContain(link);
790 } finally {
791 workEnds(app.db, run.id);
792 }
793 expect(await list("kind=agent")).not.toContain(link);
794 expect(await list("kind=agent&word=stopped")).toContain(link);
795 });
796
756797 test("a long transcript holds its last window, and the whole of it is one step deeper at ?all=1", async () => {
757798 const run = createRun(app.db, {
758799 repoId: repo.id,
test/tui-run-cancel.test.ts +22
@@ -8,6 +8,7 @@ import { createApp, type App as Forge } from "../src/app.ts";
88 import { claimRun, createRun, finishRun, getRun, type RunKind } from "../src/core/runs.ts";
99 import type { Repo } from "../src/core/repos.ts";
1010 import { createUser, type User } from "../src/core/users.ts";
11 import { workBegins, workEnds } from "../src/runs/ending.ts";
1112 import { createForgeApp } from "../src/tui/forge.ts";
1213 import { runsScreen } from "../src/tui/screens/runs.ts";
1314 import { headlessTerminal } from "../src/tui/term/terminal.ts";
@@ -109,6 +110,27 @@ test("a running run offers x cancel too", async () => {
109110 h.app.stop();
110111 });
111112
113 test("a cancelled run still ending its processes reads stopping, and cancelled once they have (#13291)", async () => {
114 const h = await open(alice, (repo) => {
115 const run = createRun(forge.db, { repoId: repo.id, kind: "deploy", env: "production" });
116 claimRun(forge.db, run.id);
117 workBegins(forge.db, run.id);
118 return run.id;
119 });
120 try {
121 await h.press("x");
122 await h.press("left");
123 const text = await h.press("enter");
124 expect(state(h.runId)).toBe("cancelled");
125 expect(text).toMatch(/stopping +deploy production/);
126 expect(text).not.toMatch(/cancelled +deploy production/);
127 } finally {
128 workEnds(forge.db, h.runId);
129 }
130 expect(await h.press("r")).toMatch(/cancelled +deploy production/);
131 h.app.stop();
132 });
133
112134 test("x does nothing on a finished run", async () => {
113135 const h = await open(alice, (repo) => {
114136 const run = createRun(forge.db, { repoId: repo.id, kind: "check" });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs