Public: anyone can read this; the crew can change it.
A crew run interrupted by a restart twice in a row is failed with its reason, not queued a third time #11585
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/11581-a-crew-run-interrupted-by-a-restart-twic → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +99
−2
src/crew/dispatch.ts +14 −2
| @@ -64,8 +64,10 @@ import { | ||
| 64 | 64 | automaticRunsSince, |
| 65 | 65 | cancelRun, |
| 66 | 66 | createRun, |
| 67 | finishRun, | |
| 67 | 68 | getRun, |
| 68 | 69 | leaseLapsed, |
| 70 | REQUEUED_SUMMARY, | |
| 69 | 71 | listActive, |
| 70 | 72 | listQueued, |
| 71 | 73 | PULL_STOPPED, |
| @@ -1363,6 +1365,9 @@ function forgetWorkdir(app: App, repo: Repo, pull: Pull): void { | ||
| 1363 | 1365 | }); |
| 1364 | 1366 | } |
| 1365 | 1367 | |
| 1368 | /** What a run says when a restart cut it off a second time in a row. */ | |
| 1369 | export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried again"; | |
| 1370 | ||
| 1366 | 1371 | /** |
| 1367 | 1372 | * A run a dead forge left `running` was cut off with it: its lease has |
| 1368 | 1373 | * lapsed. Each ends here as cancelled, and the same task goes back on the |
| @@ -1375,8 +1380,15 @@ function forgetWorkdir(app: App, repo: Repo, pull: Pull): void { | ||
| 1375 | 1380 | */ |
| 1376 | 1381 | export function requeueInterrupted(app: App): void { |
| 1377 | 1382 | for (const run of listActive(app.db, "agent")) { |
| 1378 | if (run.state === "running" && leaseLapsed(run, STALE_AFTER_MS)) | |
| 1379 | requeue(app, run, "interrupted by a restart; queued again"); | |
| 1383 | if (run.state !== "running" || !leaseLapsed(run, STALE_AFTER_MS)) continue; | |
| 1384 | // Queued by an interruption and cut off again: a forge stopped every | |
| 1385 | // 90 s would loop it until its credit is gone. Fail it, saying why. | |
| 1386 | if ((run.summary ?? "").startsWith(REQUEUED_SUMMARY)) { | |
| 1387 | finishRun(app.db, run.id, { state: "failed", summary: STOPPED_TWICE }); | |
| 1388 | log.info("run failed, stopped twice", { run: run.id }); | |
| 1389 | continue; | |
| 1390 | } | |
| 1391 | requeue(app, run, REQUEUED_SUMMARY); | |
| 1380 | 1392 | } |
| 1381 | 1393 | } |
| 1382 | 1394 |
test/crew-stopped-twice.test.ts added +85
| @@ -0,0 +1,85 @@ | ||
| 1 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 6 | import { openIssue } from "../src/actions/issues.ts"; | |
| 7 | import { createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { createApp, type App } from "../src/app.ts"; | |
| 9 | import { claimRun, createRun, getRun, listQueued } from "../src/core/runs.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { requeueInterrupted } from "../src/crew/dispatch.ts"; | |
| 12 | import { STALE_AFTER_MS } from "../src/runs/lease.ts"; | |
| 13 | import type { User } from "../src/core/users.ts"; | |
| 14 | ||
| 15 | describe("a run a restart cuts off", () => { | |
| 16 | let dataDir: string; | |
| 17 | let app: App; | |
| 18 | let alice: User; | |
| 19 | let bot: User; | |
| 20 | let n = 0; | |
| 21 | ||
| 22 | beforeAll(async () => { | |
| 23 | dataDir = await mkdtemp(join(tmpdir(), "forge-stopped-twice-")); | |
| 24 | app = createApp({ dataDir, port: 0 }); | |
| 25 | const a = await createUser(app.db, { | |
| 26 | origin: "seed", trusted: true, username: "alice", | |
| 27 | email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true, | |
| 28 | }); | |
| 29 | if (!a.ok) throw new Error(a.error); | |
| 30 | alice = a.value; | |
| 31 | const made = await createAgent(app, alice, { username: "bot" }); | |
| 32 | if (!made.ok) throw new Error(String(made.error)); | |
| 33 | bot = made.value.user; | |
| 34 | }); | |
| 35 | ||
| 36 | afterAll(async () => { | |
| 37 | app.close(); | |
| 38 | await rm(dataDir, { recursive: true, force: true }); | |
| 39 | }); | |
| 40 | ||
| 41 | /** A queued run on a fresh issue, claimed and its lease lapsed. */ | |
| 42 | const setup = async () => { | |
| 43 | const created = await createRepo(app, alice, { name: `r${n++}`, visibility: "private" }); | |
| 44 | if (!created.ok) throw new Error(String(created.error)); | |
| 45 | expect(addToCrew(app, alice, created.value, "bot").ok).toBe(true); | |
| 46 | const issued = openIssue(app, alice, created.value, { title: "Plant", body: "" }); | |
| 47 | if (!issued.ok) throw new Error(String(issued.error)); | |
| 48 | const run = createRun(app.db, { | |
| 49 | repoId: created.value.id, kind: "agent", issueId: issued.value.id, | |
| 50 | actorId: bot.id, reason: "feedback", | |
| 51 | }); | |
| 52 | return { issue: issued.value, run }; | |
| 53 | }; | |
| 54 | const lapse = (id: string) => { | |
| 55 | claimRun(app.db, id); | |
| 56 | app.db | |
| 57 | .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1") | |
| 58 | .run(id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString()); | |
| 59 | }; | |
| 60 | const forIssue = (issueId: string) => | |
| 61 | listQueued(app.db, "agent").filter((r) => r.issueId === issueId); | |
| 62 | ||
| 63 | test("once: cancelled and queued again", async () => { | |
| 64 | const { issue, run } = await setup(); | |
| 65 | lapse(run.id); | |
| 66 | requeueInterrupted(app); | |
| 67 | expect(getRun(app.db, run.id)!.state).toBe("cancelled"); | |
| 68 | const again = forIssue(issue.id); | |
| 69 | expect(again.length).toBe(1); | |
| 70 | expect(again[0]!.summary).toStartWith("interrupted by a restart; queued again"); | |
| 71 | }); | |
| 72 | ||
| 73 | test("twice in a row: failed with its reason, nothing queued", async () => { | |
| 74 | const { issue, run } = await setup(); | |
| 75 | lapse(run.id); | |
| 76 | requeueInterrupted(app); | |
| 77 | const second = forIssue(issue.id)[0]!; | |
| 78 | lapse(second.id); | |
| 79 | requeueInterrupted(app); | |
| 80 | const after = getRun(app.db, second.id)!; | |
| 81 | expect(after.state).toBe("failed"); | |
| 82 | expect(after.summary).toBe("stopped twice by its forge restarting; not tried again"); | |
| 83 | expect(forIssue(issue.id)).toEqual([]); | |
| 84 | }); | |
| 85 | }); |
Nit/next: this guard relies on requeue writing a summary that starts with REQUEUED_SUMMARY; the test covers it, keep it that way if requeue changes.