Public: anyone can read this; the crew can change it.
A crew run interrupted by a forge stop twice in a row is not queued a third time, and the forge's home says why #11576
scout/forge:scout/11560-a-crew-run-interrupted-by-a-forge-stop-t → main 7 commits fork
9 files
changed +170
−6
forge.json +1 −1
| @@ -17,7 +17,7 @@ | ||
| 17 | 17 | } |
| 18 | 18 | }, |
| 19 | 19 | "check": { |
| 20 | "fast": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"", | |
| 20 | "fast": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && FORGE_CHECK_SHARDS=4 bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"", | |
| 21 | 21 | "run": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo", |
| 22 | 22 | "verify": "bun scripts/gate.ts && bun scripts/test-shards.ts --timeout 60000 --gate --impact --keep-going --nice", |
| 23 | 23 | "timeout": 7200, |
src/actions/runs.ts +4
| @@ -397,6 +397,10 @@ export function requeueRepoRun( | ||
| 397 | 397 | task.agent, |
| 398 | 398 | task.reason, |
| 399 | 399 | writer.value.id, |
| 400 | undefined, | |
| 401 | undefined, | |
| 402 | // A person asked: a follow-up with nothing new is still run. | |
| 403 | true, | |
| 400 | 404 | ); |
| 401 | 405 | return queued ? ok(queued) : err("not_queued"); |
| 402 | 406 | } |
src/core/runs.ts +41
| @@ -592,6 +592,47 @@ export function infraBefore(db: Database, run: Run): boolean { | ||
| 592 | 592 | return cutOffBefore(db, run, INFRA_SUMMARY); |
| 593 | 593 | } |
| 594 | 594 | |
| 595 | /** What a run says when its forge restarting stopped it a second time in a row: it is not queued a third time. */ | |
| 596 | export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried again"; | |
| 597 | ||
| 598 | /** Whether the attempt just before this one was also stopped by its forge restarting. */ | |
| 599 | export function restartedBefore(db: Database, run: Run): boolean { | |
| 600 | return cutOffBefore(db, run, "interrupted by a restart") || cutOffBefore(db, run, "stopped for a restart"); | |
| 601 | } | |
| 602 | ||
| 603 | /** | |
| 604 | * End a run its forge's restart stopped for the second time in a row as | |
| 605 | * failed, with `STOPPED_TWICE`: unlike a cancel, it is a verdict a person | |
| 606 | * sees. A run the stop had already cancelled is moved too. | |
| 607 | */ | |
| 608 | export function failStoppedTwice(db: Database, runId: string): Run | undefined { | |
| 609 | const changes = db | |
| 610 | .query( | |
| 611 | `UPDATE run SET state = 'failed', summary = ?2, | |
| 612 | finished_at = COALESCE(finished_at, strftime('%Y-%m-%dT%H:%M:%fZ','now')) | |
| 613 | WHERE id = ?1 AND kind = 'agent' AND state IN ('queued','running','cancelled')`, | |
| 614 | ) | |
| 615 | .run(runId, STOPPED_TWICE).changes; | |
| 616 | return changes > 0 ? getRun(db, runId) : undefined; | |
| 617 | } | |
| 618 | ||
| 619 | /** | |
| 620 | * Agent runs stopped twice by their forge's restart that nothing has been | |
| 621 | * queued after since, newest first: what home offers a person to try again. | |
| 622 | */ | |
| 623 | export function stoppedTwice(db: Database, repoIds?: readonly string[], limit = 20): Run[] { | |
| 624 | return db | |
| 625 | .query<RunRow, [string, string | null, number]>( | |
| 626 | `SELECT * FROM run r WHERE r.kind = 'agent' AND r.state = 'failed' AND r.summary = ?1 | |
| 627 | AND (?2 IS NULL OR r.repo_id IN (SELECT value FROM json_each(?2))) | |
| 628 | AND NOT EXISTS (SELECT 1 FROM run n WHERE n.kind = 'agent' AND n.repo_id = r.repo_id | |
| 629 | AND COALESCE(n.issue_id, '') = COALESCE(r.issue_id, '') AND n.id > r.id) | |
| 630 | ORDER BY r.id DESC LIMIT ?3`, | |
| 631 | ) | |
| 632 | .all(STOPPED_TWICE, repoIds === undefined ? null : JSON.stringify(repoIds), limit) | |
| 633 | .map(toRun); | |
| 634 | } | |
| 635 | ||
| 595 | 636 | function cutOffBefore(db: Database, run: Run, prefix: string): boolean { |
| 596 | 637 | return ( |
| 597 | 638 | db |
src/crew/dispatch.ts +10
| @@ -64,6 +64,9 @@ import { | ||
| 64 | 64 | automaticRunsSince, |
| 65 | 65 | cancelRun, |
| 66 | 66 | createRun, |
| 67 | failStoppedTwice, | |
| 68 | restartedBefore, | |
| 69 | STOPPED_TWICE, | |
| 67 | 70 | getRun, |
| 68 | 71 | leaseLapsed, |
| 69 | 72 | listActive, |
| @@ -1383,6 +1386,13 @@ export function requeueInterrupted(app: App): void { | ||
| 1383 | 1386 | /** End a run that cannot go on, and put its task back on the queue. */ |
| 1384 | 1387 | function requeue(app: App, run: Run, why: string): void { |
| 1385 | 1388 | cancelRun(app.db, run.id, why); |
| 1389 | // Stopped by a restart twice running: a forge restarted every few | |
| 1390 | // seconds would loop until its credit is gone. Fail it, saying why. | |
| 1391 | if (restartedBefore(app.db, run)) { | |
| 1392 | failStoppedTwice(app.db, run.id); | |
| 1393 | log.info("run not queued again", { run: run.id, why: STOPPED_TWICE }); | |
| 1394 | return; | |
| 1395 | } | |
| 1386 | 1396 | const task = taskOf(app, run); |
| 1387 | 1397 | if (task === undefined) return; |
| 1388 | 1398 | // The run that starts carries on in the clone this one left, in the |
src/http/server.ts +2
| @@ -358,6 +358,7 @@ import { | ||
| 358 | 358 | } from "../web/pages/migrate.ts"; |
| 359 | 359 | import { |
| 360 | 360 | runCancel, |
| 361 | runRetry, | |
| 361 | 362 | runPage, |
| 362 | 363 | runRedirect, |
| 363 | 364 | runSteer, |
| @@ -1404,6 +1405,7 @@ export function buildRouter(): Router<Handler> { | ||
| 1404 | 1405 | router.add("GET", "/:owner/:name/runs", runsList); |
| 1405 | 1406 | router.add("GET", "/:owner/:name/runs/:id", runPage); |
| 1406 | 1407 | router.add("POST", "/:owner/:name/runs/:id/cancel", runCancel); |
| 1408 | router.add("POST", "/:owner/:name/runs/:id/retry", runRetry); | |
| 1407 | 1409 | router.add("POST", "/:owner/:name/runs/:id/steer", runSteer); |
| 1408 | 1410 | // The public social cards: a repo as owner~name, an app as its slug. |
| 1409 | 1411 | router.add("GET", "/og/repo/:ref.svg", ogCard("repo")); |
src/web/pages/runs.ts +17
| @@ -18,6 +18,7 @@ import type { Database } from "bun:sqlite"; | ||
| 18 | 18 | import { |
| 19 | 19 | cancelRepoRun, |
| 20 | 20 | listRepoRuns, |
| 21 | requeueRepoRun, | |
| 21 | 22 | runLog, |
| 22 | 23 | steerRepoRun, |
| 23 | 24 | } from "../../actions/runs.ts"; |
| @@ -387,6 +388,22 @@ export function runRedirect(ctx: Ctx): Response { | ||
| 387 | 388 | return redirect(`${repoBase(repo)}/runs/${run.id}`); |
| 388 | 389 | } |
| 389 | 390 | |
| 391 | // POST /:owner/:name/runs/:id/retry | |
| 392 | export async function runRetry(ctx: Ctx): Promise<Response> { | |
| 393 | const repo = loadRepo(ctx); | |
| 394 | if (!repo) return notFoundPage(ctx); | |
| 395 | if (!ctx.auth.user) return signInFirst(ctx); | |
| 396 | const form = await readForm(ctx, "rw"); | |
| 397 | if (form instanceof Response) return form; | |
| 398 | const id = ctx.params.id ?? ""; | |
| 399 | const queued = requeueRepoRun(ctx.app, ctx.auth.user, repo, id); | |
| 400 | if (!queued.ok && queued.error === "not_found") return notFoundPage(ctx); | |
| 401 | return flashTo(ctx, "/", { | |
| 402 | ...(queued.ok ? {} : { tone: "danger" as const }), | |
| 403 | message: queued.ok ? "Queued again." : `Not queued: ${queued.error.replaceAll("_", " ")}.`, | |
| 404 | }); | |
| 405 | } | |
| 406 | ||
| 390 | 407 | // POST /:owner/:name/runs/:id/cancel |
| 391 | 408 | export async function runCancel(ctx: Ctx): Promise<Response> { |
| 392 | 409 | const repo = loadRepo(ctx); |
src/web/studio/held.ts +31 −1
| @@ -4,6 +4,8 @@ | ||
| 4 | 4 | * gate, when — and a link that opens the list. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { getIssuesByIds } from "../../core/issues.ts"; | |
| 8 | import { stoppedTwice } from "../../core/runs.ts"; | |
| 7 | 9 | import { gateHold } from "../../core/gate_hold.ts"; |
| 8 | 10 | import { reposWithUmbrellas } from "../../core/pillars.ts"; |
| 9 | 11 | import { canWrite, listReposOwnedBy, type Repo } from "../../core/repos.ts"; |
| @@ -11,7 +13,7 @@ import type { User } from "../../core/users.ts"; | ||
| 11 | 13 | import type { Ctx } from "../../http/context.ts"; |
| 12 | 14 | import { repoBase } from "../chrome.ts"; |
| 13 | 15 | import { html, type Html } from "../html.ts"; |
| 14 | import { time } from "../ui.ts"; | |
| 16 | import { btn, time } from "../ui.ts"; | |
| 15 | 17 | import { writableAppsOf } from "./asks.ts"; |
| 16 | 18 | |
| 17 | 19 | /** The repositories whose gate holds work back from this person, with the hold. */ |
| @@ -48,3 +50,31 @@ export function heldLine(ctx: Ctx, user: User): Html | null { | ||
| 48 | 50 | )} |
| 49 | 51 | </section>`; |
| 50 | 52 | } |
| 53 | ||
| 54 | /** | |
| 55 | * One line for each crew run its forge's restart stopped twice and the crew | |
| 56 | * did not queue a third time: the run, why, and one button to try again. | |
| 57 | * Shown to those who may write to the repository. | |
| 58 | */ | |
| 59 | export function stoppedLine(ctx: Ctx, user: User): Html | null { | |
| 60 | const db = ctx.app.db; | |
| 61 | // One cheap read first: nearly always nothing, and home pays no more. | |
| 62 | const stopped = stoppedTwice(db); | |
| 63 | if (stopped.length === 0) return null; | |
| 64 | const byId = new Map<string, Repo>(listReposOwnedBy(db, user.id).map((r) => [r.id, r])); | |
| 65 | for (const r of writableAppsOf(ctx, user).repos) byId.set(r.id, r); | |
| 66 | const runs = stopped.filter((run) => byId.has(run.repoId) && canWrite(db, byId.get(run.repoId)!, user)).slice(0, 3); | |
| 67 | if (runs.length === 0) return null; | |
| 68 | const issues = getIssuesByIds(db, runs.flatMap((r) => (r.issueId === null ? [] : [r.issueId]))); | |
| 69 | return html`<section class="stack stack-tight" id="stopped-line" aria-label="Stopped"> | |
| 70 | ${runs.map((run) => { | |
| 71 | const repo = byId.get(run.repoId)!; | |
| 72 | const issue = run.issueId === null ? undefined : issues.get(run.issueId); | |
| 73 | return html`<form class="small" method="post" action="${repoBase(repo)}/runs/${run.id}/retry"> | |
| 74 | <a href="${repoBase(repo)}/runs/${run.id}">The run${issue === undefined ? "" : html` on #${issue.number} ${issue.title}`}</a> | |
| 75 | in ${repo.name}: ${run.summary}. | |
| 76 | ${btn("Try again", { type: "submit", variant: "outline" })} | |
| 77 | </form>`; | |
| 78 | })} | |
| 79 | </section>`; | |
| 80 | } |
src/web/studio/home.ts +5 −4
| @@ -71,7 +71,7 @@ import { writableAppsOf } from "./asks.ts"; | ||
| 71 | 71 | import { needsYouSection } from "./asks_page.ts"; |
| 72 | 72 | import { flowCards } from "./flow_card.ts"; |
| 73 | 73 | import { composer, feedJs, feedNote, feedRegion, registerWorkerJs } from "./feed.ts"; |
| 74 | import { heldLine } from "./held.ts"; | |
| 74 | import { heldLine, stoppedLine } from "./held.ts"; | |
| 75 | 75 | import { appChips, movingRegion, tellBox } from "./living.ts"; |
| 76 | 76 | import { alarmLines, cachedAlarms } from "./alarm_lines.ts"; |
| 77 | 77 | import { kpiRow, waitingLine } from "./kpi_row.ts"; |
| @@ -326,7 +326,7 @@ export async function studioHome( | ||
| 326 | 326 | // Before the first tell the card is the one box and carries the key line and the composer. |
| 327 | 327 | const firstTell = steps.some((s) => s.key === "tell" && !s.done); |
| 328 | 328 | const lean = html`<h1 class="sr-only">Home</h1> |
| 329 | ${firstSteps} ${address} ${firstTell ? null : crewKeyCard(ctx)} ${heldLine(ctx, user)} ${apps} | |
| 329 | ${firstSteps} ${address} ${firstTell ? null : crewKeyCard(ctx)} ${heldLine(ctx, user)} ${stoppedLine(ctx, user)} ${apps} | |
| 330 | 330 | <div class="stack"> |
| 331 | 331 | ${firstTell ? null : composer(ctx, user)} |
| 332 | 332 | <p class="small muted"> |
| @@ -355,6 +355,7 @@ export async function studioHome( | ||
| 355 | 355 | const draws = homeDraws(ctx, user); |
| 356 | 356 | const waiting = await homeRegion(ctx.app.db, ctx.request, user.id, "waiting", homeVariant(ctx.url), access, draws.waiting); |
| 357 | 357 | const held = timePhase(ctx.request, "held", () => heldLine(ctx, user)); |
| 358 | const stopped = timePhase(ctx.request, "stopped", () => stoppedLine(ctx, user)); | |
| 358 | 359 | const alarms = user.isAdmin |
| 359 | 360 | ? alarmLines(cachedAlarms(ctx.app.db, new Date(), forgeCeiling(ctx.app.config), ctx.app.config.dataDir)) |
| 360 | 361 | : null; |
| @@ -368,7 +369,7 @@ export async function studioHome( | ||
| 368 | 369 | // The first page after Allow says so, once. |
| 369 | 370 | const allowed = untoldAnswer(ctx.app.db, user.id) === "allowed"; |
| 370 | 371 | if (allowed) markTold(ctx.app.db, user.id, new Date().toISOString()); |
| 371 | const needsYou = html`${requests}${waiting}${held}${alarms}`; | |
| 372 | const needsYou = html`${requests}${waiting}${held}${stopped}${alarms}`; | |
| 372 | 373 | const mine = writableAppsOf(ctx, user).repos; |
| 373 | 374 | const body = html`<h1 class="sr-only">Home</h1> |
| 374 | 375 | ${allowed ? html`<div class="toast" data-tone="success" role="status" id="youre-in"><strong>${YOURE_IN_MESSAGE}</strong> Your account is open: tell the crew what you want to build.</div>` : null} |
| @@ -392,7 +393,7 @@ export async function studioHome( | ||
| 392 | 393 | body, |
| 393 | 394 | scriptJs: `${liveClientJs}\n${liveRegionsJs}\n${refreshJs}\n${registerWorkerJs}${ask ? `\n${passkeyJs}` : ""}`, |
| 394 | 395 | }); |
| 395 | notePhaseRest(ctx.request, "render", rendering, ["kpis", "moving", "access", "waiting", "held", "visit", "since"]); | |
| 396 | notePhaseRest(ctx.request, "render", rendering, ["kpis", "moving", "access", "waiting", "held", "stopped", "visit", "since"]); | |
| 396 | 397 | return res; |
| 397 | 398 | } |
| 398 | 399 |
test/crew-interrupted.test.ts +59
| @@ -23,6 +23,10 @@ import { claimWorkdir, isClaimed, workdirFor } from "../src/crew/workdir.ts"; | ||
| 23 | 23 | import { STALE_AFTER_MS } from "../src/runs/lease.ts"; |
| 24 | 24 | import { createUser, type User } from "../src/core/users.ts"; |
| 25 | 25 | import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts"; |
| 26 | import { recordUsage } from "../src/core/usage.ts"; | |
| 27 | import { requeueRepoRun } from "../src/actions/runs.ts"; | |
| 28 | import { requeueInterrupted } from "../src/crew/dispatch.ts"; | |
| 29 | import { stoppedLine } from "../src/web/studio/held.ts"; | |
| 26 | 30 | import { serve, type Servers } from "../src/http/server.ts"; |
| 27 | 31 | |
| 28 | 32 | /** The crew's tick, injected short: every wait below is for the next tick (#11053). */ |
| @@ -213,6 +217,61 @@ describe("a run the last forge left running", () => { | ||
| 213 | 217 | expect(queued[0]!.attempt).toBe(1); |
| 214 | 218 | }, 30_000); |
| 215 | 219 | |
| 220 | /** A run the last forge left running with a lapsed lease, as `requeueInterrupted` finds it. */ | |
| 221 | const lapse = (id: string) => { | |
| 222 | claimRun(app.db, id); | |
| 223 | app.db | |
| 224 | .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1") | |
| 225 | .run(id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString()); | |
| 226 | }; | |
| 227 | ||
| 228 | test("stopped by a restart once it is queued again; twice in a row it fails and is not queued, charged once (#11560)", async () => { | |
| 229 | await atRest(); | |
| 230 | const repo = await crewedRepo("hedge"); | |
| 231 | const issued = openIssue(app, alice, repo, { title: "Trim", body: "" }); | |
| 232 | if (!issued.ok) throw new Error(String(issued.error)); | |
| 233 | const issue = issued.value; | |
| 234 | const first = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: issue.id, actorId: bot.id, reason: "feedback" }); | |
| 235 | lapse(first.id); | |
| 236 | // The first run paid for its minutes: one line in the ledger. | |
| 237 | recordUsage(app.db, { | |
| 238 | runId: first.id, repoId: repo.id, issueId: issue.id, agentId: bot.id, subscriptionId: null, | |
| 239 | kind: "build", tokensIn: 10, tokensOut: 10, costUsd: 0.5, wallMs: 90_000, outcome: "running", | |
| 240 | at: new Date().toISOString(), | |
| 241 | }); | |
| 242 | ||
| 243 | requeueInterrupted(app); | |
| 244 | expect(getRun(app.db, first.id)?.state).toBe("cancelled"); | |
| 245 | const second = listQueued(app.db, "agent").filter((r) => r.issueId === issue.id && r.state === "queued"); | |
| 246 | expect(second).toHaveLength(1); | |
| 247 | expect(second[0]!.attempt).toBe(1); | |
| 248 | ||
| 249 | // The same run is stopped again by the next restart. | |
| 250 | lapse(second[0]!.id); | |
| 251 | requeueInterrupted(app); | |
| 252 | const stopped = getRun(app.db, second[0]!.id)!; | |
| 253 | expect(stopped.state).toBe("failed"); | |
| 254 | expect(stopped.summary).toBe("stopped twice by its forge restarting; not tried again"); | |
| 255 | expect(listQueued(app.db, "agent").some((r) => r.issueId === issue.id && r.state === "queued")).toBe(false); | |
| 256 | ||
| 257 | // Charged once: the interrupted run has one line, and the second try added none of its minutes. | |
| 258 | const lines = app.db | |
| 259 | .query<{ n: number; cost: number }, [string]>("SELECT COUNT(*) AS n, SUM(cost_usd) AS cost FROM usage WHERE issue_id = ?1") | |
| 260 | .get(issue.id)!; | |
| 261 | expect(lines.n).toBe(1); | |
| 262 | expect(lines.cost).toBe(0.5); | |
| 263 | ||
| 264 | // Home names it, with the way to try again; trying again queues it and the line goes. | |
| 265 | const ctx = { app, auth: { user: alice }, request: new Request("http://x/"), url: new URL("http://x/") } as never; | |
| 266 | const line = String(stoppedLine(ctx, alice)); | |
| 267 | expect(line).toContain("stopped twice by its forge restarting; not tried again"); | |
| 268 | expect(line).toContain(`/runs/${stopped.id}/retry`); | |
| 269 | const again = requeueRepoRun(app, alice, repo, stopped.id); | |
| 270 | expect(again.ok).toBe(true); | |
| 271 | expect(stoppedLine(ctx, alice)).toBeNull(); | |
| 272 | cancelRun(app.db, listQueued(app.db, "agent").find((r) => r.issueId === issue.id)!.id); | |
| 273 | }); | |
| 274 | ||
| 216 | 275 | test("a run in flight when the crew stops goes back on the queue", async () => { |
| 217 | 276 | await atRest(); |
| 218 | 277 | const created = await createRepo(app, alice, { |