Public: anyone can read this; the crew can change it.
A crew run paused for credit or a rate limit gives up its slot and says why and when it resumes #11580
Check · Review · Land · Live — Its check is red.
scout/forge:scout/11564-a-crew-run-paused-for-credit-or-a-rate-l → main 1 commit fork
4 files
changed +82
−1
src/crew/dispatch.ts +2
| @@ -123,6 +123,7 @@ import { | ||
| 123 | 123 | import { certain } from "./decide.ts"; |
| 124 | 124 | import { readRunLog } from "../runs/log.ts"; |
| 125 | 125 | import { whenText } from "./limits.ts"; |
| 126 | import { pauseCause } from "./pacing.ts"; | |
| 126 | 127 | import { |
| 127 | 128 | executeAgentRun, |
| 128 | 129 | workdirFor, |
| @@ -1431,6 +1432,7 @@ function resumeLater(app: App, run: Run, pause: Pause): void { | ||
| 1431 | 1432 | until: pause.resumeAt, |
| 1432 | 1433 | attempt: pause.attempt, |
| 1433 | 1434 | queued: queued?.id, |
| 1435 | cause: pauseCause(pause.summary ?? ""), | |
| 1434 | 1436 | }); |
| 1435 | 1437 | } |
| 1436 | 1438 |
src/crew/pacing.ts +13
| @@ -107,3 +107,16 @@ export function pauseReason( | ||
| 107 | 107 | const who = name === undefined ? "the" : `${name}'s`; |
| 108 | 108 | return `${who} ${LABEL[limit.window]} window is full until ${whenText(limit.resetsAt)}`; |
| 109 | 109 | } |
| 110 | ||
| 111 | /** | |
| 112 | * What a paused run's summary says caused the pause, in a word for the log: | |
| 113 | * the provider's balance ("credit"), a full window or limit ("rate limit"), | |
| 114 | * the network, or an account resting. Read from the summary the run wrote. | |
| 115 | */ | |
| 116 | export function pauseCause(summary: string): string { | |
| 117 | const said = summary.replace(/^paused: /, "").toLowerCase(); | |
| 118 | if (/credit|balance|billing|budget|quota/.test(said)) return "credit"; | |
| 119 | if (/network/.test(said)) return "network"; | |
| 120 | if (/window|limit|rate|overloaded|resting/.test(said)) return "rate limit"; | |
| 121 | return "other"; | |
| 122 | } |
src/crew/runner.ts +5 −1
| @@ -544,7 +544,11 @@ async function runClaimed( | ||
| 544 | 544 | }); |
| 545 | 545 | const paused = cancelRun(app.db, run.id, outcome.summary); |
| 546 | 546 | if (paused) announceRun(app, paused); |
| 547 | return { resumeAt: outcome.resumeAt, attempt: run.attempt + 1 }; | |
| 547 | return { | |
| 548 | resumeAt: outcome.resumeAt, | |
| 549 | attempt: run.attempt + 1, | |
| 550 | summary: outcome.summary, | |
| 551 | }; | |
| 548 | 552 | } |
| 549 | 553 | // A run the provider or a restart cut short says so, and when it goes |
| 550 | 554 | // on: interrupted, never a park (`interrupted.ts`). |
test/crew-pause-slot.test.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | /** | |
| 2 | * A run paused for a limit gives up its slot: with one slot, another queued | |
| 3 | * run starts at once, and the pause's log line names its cause (#11564). | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import type { App } from "../src/app.ts"; | |
| 7 | import type { Repo } from "../src/core/repos.ts"; | |
| 8 | import type { User } from "../src/core/users.ts"; | |
| 9 | import { assignIssue, openIssue } from "../src/actions/issues.ts"; | |
| 10 | import { queueAgentRun } from "../src/crew/dispatch.ts"; | |
| 11 | import { pauseCause } from "../src/crew/pacing.ts"; | |
| 12 | import { getAgentByName } from "../src/core/agents.ts"; | |
| 13 | import { listActive } from "../src/core/runs.ts"; | |
| 14 | import { | |
| 15 | agentRunSettled, | |
| 16 | startStandInCrew, | |
| 17 | stopStandInCrew, | |
| 18 | type StandInCrew, | |
| 19 | } from "./helpers/crew-standin.ts"; | |
| 20 | ||
| 21 | describe("a paused run does not hold the crew's only slot", () => { | |
| 22 | let standIn: StandInCrew; | |
| 23 | let app: App; | |
| 24 | let alice: User; | |
| 25 | let bot: User; | |
| 26 | let repo: Repo; | |
| 27 | ||
| 28 | beforeAll(async () => { | |
| 29 | standIn = await startStandInCrew({ tickMs: 20 }); | |
| 30 | ({ app, alice, bot, repo } = standIn); | |
| 31 | }, 30_000); | |
| 32 | afterAll(() => stopStandInCrew(standIn)); | |
| 33 | ||
| 34 | test("run A paused until later, run B queued: B starts now and A keeps its words", async () => { | |
| 35 | const a = openIssue(app, alice, repo, { title: "Paused work", body: "Waits." }); | |
| 36 | const b = openIssue(app, alice, repo, { title: "Review follow-up", body: "Now." }); | |
| 37 | if (!a.ok || !b.ok) throw new Error("issues"); | |
| 38 | const agent = getAgentByName(app.db, "bot")!; | |
| 39 | const resumeAt = new Date(Date.now() + 3_600_000).toISOString(); | |
| 40 | const summary = "paused: the 5-hour window is full; resumes later"; | |
| 41 | const paused = queueAgentRun(app, repo, a.value, agent, "assigned", undefined, { | |
| 42 | notBefore: resumeAt, | |
| 43 | attempt: 1, | |
| 44 | summary, | |
| 45 | }); | |
| 46 | expect(paused).toBeDefined(); | |
| 47 | expect(assignIssue(app, alice, repo, b.value, bot).ok).toBe(true); | |
| 48 | const ran = await agentRunSettled(app, repo, (r) => r.issueId === b.value.id && r.reason === "assigned", undefined, 20); | |
| 49 | expect(ran.state).not.toBe("queued"); | |
| 50 | const still = listActive(app.db, "agent").find((r) => r.id === paused!.id)!; | |
| 51 | expect(still.state).toBe("queued"); | |
| 52 | expect(still.notBefore).toBe(resumeAt); | |
| 53 | expect(still.summary).toBe(summary); | |
| 54 | }); | |
| 55 | ||
| 56 | test("the cause is named", () => { | |
| 57 | expect(pauseCause("paused: the 5-hour window is full until 00:52; resumes 00:52")).toBe("rate limit"); | |
| 58 | expect(pauseCause("paused: credit balance is too low; resumes 09:00")).toBe("credit"); | |
| 59 | expect(pauseCause("paused: network error, resumes 01:00")).toBe("network"); | |
| 60 | expect(pauseCause("paused: something else")).toBe("other"); | |
| 61 | }); | |
| 62 | }); |