Public: anyone can read this; the crew can change it.
Sentinel: Crew runs in trevato/quorum keep failing at the push or the sandbox #1982
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1923-sentinel-crew-runs-in-trevato-quorum-kee → main 3 commits fork 2 threads · 2 unresolved
6 files
changed +105
−4
src/crew/interrupted.ts +2 −2
| @@ -17,7 +17,7 @@ export const INTERRUPTED = "interrupted: "; | ||
| 17 | 17 | const PROVIDER = |
| 18 | 18 | /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b|\(api_error\)|may use is resting|resting —/i; |
| 19 | 19 | const NETWORK = |
| 20 | /ECONNREFUSED|ECONNRESET|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|unable to connect|connection error|could not reach|unreachable|socket hang up/i; | |
| 20 | /ECONNREFUSED|ECONNRESET|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|unable to connect|connection error|could not reach|can'?t reach the API server|unreachable|socket hang up/i; | |
| 21 | 21 | const RESTART = |
| 22 | 22 | /^cancelled$|interrupted by a restart|stopped for a restart|the forge stopped/i; |
| 23 | 23 | |
| @@ -37,8 +37,8 @@ export function interruptionOf(summary: string): Interruption | undefined { | ||
| 37 | 37 | } |
| 38 | 38 | if (/^declined\b/i.test(s)) return undefined; |
| 39 | 39 | if (RESTART.test(s)) return "restart"; |
| 40 | if (PROVIDER.test(s)) return "provider limit"; | |
| 41 | 40 | if (NETWORK.test(s)) return "network error"; |
| 41 | if (PROVIDER.test(s)) return "provider limit"; | |
| 42 | 42 | return undefined; |
| 43 | 43 | } |
| 44 | 44 |
src/crew/limits.ts +2 −1
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import type { LimitWindow } from "../core/standing.ts"; |
| 9 | import { isNetworkError } from "./network.ts"; | |
| 9 | 10 | import type { ClaudeOutcome } from "./claude.ts"; |
| 10 | 11 | |
| 11 | 12 | /** How many times one task may be paused before the forge gives up on it. */ |
| @@ -22,7 +23,7 @@ const SLACK_MS = 60_000; | ||
| 22 | 23 | * the limit, and is checked too in case the ending is named differently. |
| 23 | 24 | */ |
| 24 | 25 | export function isRateLimited(outcome: ClaudeOutcome): boolean { |
| 25 | if (outcome.ok) return false; | |
| 26 | if (outcome.ok || isNetworkError(outcome)) return false; | |
| 26 | 27 | return ( |
| 27 | 28 | outcome.reason === "api_error" || |
| 28 | 29 | /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b/i.test( |
src/crew/network.ts +3
| @@ -13,6 +13,9 @@ const NETWORK = | ||
| 13 | 13 | /** Seconds to wait before each retry; its length is the number of retries. */ |
| 14 | 14 | export const NETWORK_RETRY_SECONDS: readonly number[] = [5, 20, 60]; |
| 15 | 15 | |
| 16 | /** How long a run waits, paused, when the retries found the network still down. */ | |
| 17 | export const NETWORK_PAUSE_MS = 5 * 60_000; | |
| 18 | ||
| 16 | 19 | /** Whether the run ended on the network rather than on the work. */ |
| 17 | 20 | export function isNetworkError(outcome: ClaudeOutcome): boolean { |
| 18 | 21 | if (outcome.ok) return false; |
src/crew/runner.ts +20
| @@ -78,6 +78,7 @@ import { | ||
| 78 | 78 | import { policyBrief } from "../contract/audit.ts"; |
| 79 | 79 | import { |
| 80 | 80 | isNetworkError, |
| 81 | NETWORK_PAUSE_MS, | |
| 81 | 82 | NETWORK_RETRY_SECONDS, |
| 82 | 83 | pause, |
| 83 | 84 | } from "./network.ts"; |
| @@ -1076,6 +1077,25 @@ async function work( | ||
| 1076 | 1077 | turns: result.turns + again.turns, |
| 1077 | 1078 | }; |
| 1078 | 1079 | } |
| 1080 | // Still unreachable after the retries: an outage, not a verdict. The run | |
| 1081 | // pauses with its clone and session and asks again later, as it does for | |
| 1082 | // a provider limit, rather than ending as "no changes pushed". | |
| 1083 | if ( | |
| 1084 | isNetworkError(result) && | |
| 1085 | !signal.aborted && | |
| 1086 | Date.now() < deadline && | |
| 1087 | run.attempt < MAX_PAUSES | |
| 1088 | ) { | |
| 1089 | const at = new Date(Date.now() + NETWORK_PAUSE_MS); | |
| 1090 | write(`network still down; paused, resumes ${whenText(at)}`); | |
| 1091 | return { | |
| 1092 | state: "paused", | |
| 1093 | summary: `paused: network error, resumes ${whenText(at)}`, | |
| 1094 | resumeAt: at.toISOString(), | |
| 1095 | sessionId: result.sessionId, | |
| 1096 | costUsd: result.costUsd, | |
| 1097 | }; | |
| 1098 | } | |
| 1079 | 1099 | noteRunProgress(app.db, run.id, { |
| 1080 | 1100 | sessionId: result.sessionId, |
| 1081 | 1101 | costUsd: result.costUsd, |
test/crew-network.test.ts +73 −1
| @@ -3,12 +3,18 @@ | ||
| 3 | 3 | * not judged (#1613). |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | import { rmSync } from "node:fs"; | |
| 7 | import { join } from "node:path"; | |
| 6 | 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 7 | 9 | import type { App } from "../src/app.ts"; |
| 8 | 10 | import type { Repo } from "../src/core/repos.ts"; |
| 9 | 11 | import type { User } from "../src/core/users.ts"; |
| 10 | 12 | import { assignIssue, openIssue } from "../src/actions/issues.ts"; |
| 11 | import { isNetworkError } from "../src/crew/network.ts"; | |
| 13 | import { listRuns, type Run } from "../src/core/runs.ts"; | |
| 14 | import { MAX_PAUSES } from "../src/crew/limits.ts"; | |
| 15 | import { isNetworkError, NETWORK_PAUSE_MS } from "../src/crew/network.ts"; | |
| 16 | import { isRateLimited } from "../src/crew/limits.ts"; | |
| 17 | import { interruptionOf } from "../src/crew/interrupted.ts"; | |
| 12 | 18 | import type { ClaudeOutcome } from "../src/crew/claude.ts"; |
| 13 | 19 | import { |
| 14 | 20 | agentRunSettled, |
| @@ -77,4 +83,70 @@ describe("the crew, with a stand-in Claude Code: network errors", () => { | ||
| 77 | 83 | expect(run.state).toBe("ok"); |
| 78 | 84 | expect(run.summary).toContain("opened pull"); |
| 79 | 85 | }, 60_000); |
| 86 | ||
| 87 | test("an outage that outlasts the retries pauses the run; after MAX_PAUSES it ends", async () => { | |
| 88 | const issue = openIssue(app, alice, repo, { | |
| 89 | title: "Network down", | |
| 90 | body: "NET-DOWN", | |
| 91 | }); | |
| 92 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 93 | expect(assignIssue(app, alice, repo, issue.value, bot).ok).toBe(true); | |
| 94 | const mine = () => | |
| 95 | listRuns(app.db, repo.id, { kind: "agent" }).filter( | |
| 96 | (r) => r.issueId === issue.value.id, | |
| 97 | ); | |
| 98 | const waitFor = async (where: (r: Run) => boolean): Promise<Run> => { | |
| 99 | const until = Date.now() + 30_000; | |
| 100 | while (Date.now() < until) { | |
| 101 | const found = mine().find(where); | |
outdated
This passes because the sample contains EAI_AGAIN. The interrupted.ts regex lacks "can't reach the API server"; test that phrase alone and add it there. |
||
| 102 | if (found) return found; | |
| 103 | await Bun.sleep(50); | |
| 104 | } | |
| 105 | throw new Error("run did not settle: " + JSON.stringify(mine().map((r) => [r.state, r.attempt, r.summary]))); | |
| 106 | }; | |
| 107 | const paused = await waitFor((r) => r.state === "cancelled"); | |
| 108 | expect(paused.summary).toStartWith("paused: network error"); | |
| 109 | const next = await waitFor((r) => r.state === "queued"); | |
| 110 | expect(next.attempt).toBe(1); | |
| 111 | const wait = Date.parse(next.notBefore ?? "") - Date.now(); | |
| 112 | expect(wait).toBeGreaterThan(NETWORK_PAUSE_MS - 30_000); | |
| 113 | expect(wait).toBeLessThanOrEqual(NETWORK_PAUSE_MS); | |
| 114 | ||
| 115 | // The sixth pause was the last: the run that starts now ends instead. | |
| 116 | app.db | |
| 117 | .query("UPDATE run SET attempt = ?1, not_before = ?2 WHERE id = ?3") | |
| 118 | .run(MAX_PAUSES, new Date().toISOString(), next.id); | |
| 119 | const ended = await waitFor((r) => r.id === next.id && r.state === "failed"); | |
| 120 | expect(ended.summary).not.toStartWith("paused"); | |
| 121 | rmSync(join(standIn.dataDir, "down")); | |
| 122 | }, 60_000); | |
| 123 | }); | |
| 124 | ||
| 125 | describe("a network outage is not a provider limit", () => { | |
| 126 | const down = { | |
| 127 | result: | |
| 128 | "API Error: Can't reach the API server — check your internet or DNS (EAI_AGAIN)", | |
| 129 | reason: "api_error", | |
| 130 | }; | |
| 131 | ||
| 132 | test("an unreachable API is not rate limited", () => { | |
| 133 | expect(isRateLimited(outcome(down))).toBe(false); | |
| 134 | expect( | |
| 135 | isRateLimited( | |
| 136 | outcome({ result: "429 too many requests", reason: "api_error" }), | |
| 137 | ), | |
| 138 | ).toBe(true); | |
| 139 | }); | |
| 140 | ||
| 141 | test("the phrase alone reads as a network error", () => { | |
| 142 | expect(interruptionOf("no changes pushed: Can't reach the API server")).toBe( | |
| 143 | "network error", | |
| 144 | ); | |
| 145 | }); | |
| 146 | ||
| 147 | test("its summary reads as a network error", () => { | |
| 148 | expect( | |
| 149 | interruptionOf(`no changes pushed: ${down.result} (api_error)`), | |
| 150 | ).toBe("network error"); | |
| 151 | }); | |
| 80 | 152 | }); |
test/helpers/crew-standin.ts +5
| @@ -62,6 +62,11 @@ echo "half-done" > PAUSED.md | ||
| 62 | 62 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"You\u2019ve hit your session limit \u00b7 resets 7:20am (UTC)","total_cost_usd":0.02,"num_turns":2,"session_id":"fake-1","terminal_reason":"api_error"}' |
| 63 | 63 | exit 0 |
| 64 | 64 | fi |
| 65 | if grep -q NET-DOWN "${promptFile}"; then touch "${dataDir}/down"; fi | |
| 66 | if [ -f "${dataDir}/down" ] && grep -q "Continue where you stopped\\|NET-DOWN" "${promptFile}"; then | |
| 67 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' | |
| 68 | exit 0 | |
| 69 | fi | |
| 65 | 70 | if grep -q NET-FLAKE "${promptFile}" && [ ! -f "${dataDir}/flaked" ]; then |
| 66 | 71 | touch "${dataDir}/flaked" |
| 67 | 72 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server — check your internet or DNS (EAI_AGAIN)","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' |
new above
offline
Untested new branch. Add a runner test that forces a persistent network error and asserts state paused plus resumeAt, and the MAX_PAUSES fall-through. Consider also requiring Date.now() < deadline, as the retry loop does.