Public: anyone can read this; the crew can change it.
Sentinel: Crew runs in trevato/quorum keep failing at the push or the sandbox #1895
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1613-sentinel-crew-runs-in-trevato-quorum-kee → main 2 commits fork 1 thread · 1 unresolved
5 files
changed +147
−0
scripts/gate.ts +1
| @@ -72,6 +72,7 @@ export const GATE: readonly string[] = [ | ||
| 72 | 72 | "test/crew-check-failed.test.ts", |
| 73 | 73 | "test/crew-continues.test.ts", |
| 74 | 74 | "test/crew-declines.test.ts", |
| 75 | "test/crew-network.test.ts", | |
| 75 | 76 | "test/crew-pause.test.ts", |
| 76 | 77 | "test/crew-review.test.ts", |
| 77 | 78 | "test/crew-review-budget.test.ts", |
src/crew/network.ts added +34
| @@ -0,0 +1,34 @@ | ||
| 1 | /** | |
| 2 | * A dropped connection is not the agent's failure. DNS that answers | |
| 3 | * `EAI_AGAIN` for a moment, a reset socket: Claude Code ends such a run | |
| 4 | * with an error and no work, and the same session asked again a few seconds | |
| 5 | * later goes on. The runner retries before it judges the run. | |
| 6 | */ | |
| 7 | ||
| 8 | import type { ClaudeOutcome } from "./claude.ts"; | |
| 9 | ||
| 10 | const NETWORK = | |
| 11 | /EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EPIPE|fetch failed|socket hang up|can'?t reach the API server|unable to connect|connection error/i; | |
| 12 | ||
| 13 | /** Seconds to wait before each retry; its length is the number of retries. */ | |
| 14 | export const NETWORK_RETRY_SECONDS: readonly number[] = [5, 20, 60]; | |
| 15 | ||
| 16 | /** Whether the run ended on the network rather than on the work. */ | |
| 17 | export function isNetworkError(outcome: ClaudeOutcome): boolean { | |
| 18 | if (outcome.ok) return false; | |
| 19 | return NETWORK.test(outcome.result) || NETWORK.test(outcome.reason); | |
| 20 | } | |
| 21 | ||
| 22 | /** Resolves after `ms`, or at once when `signal` aborts. */ | |
| 23 | export function pause(ms: number, signal: AbortSignal): Promise<void> { | |
| 24 | return new Promise((resolve) => { | |
| 25 | if (signal.aborted) return resolve(); | |
| 26 | const done = () => { | |
| 27 | clearTimeout(timer); | |
| 28 | signal.removeEventListener("abort", done); | |
| 29 | resolve(); | |
| 30 | }; | |
| 31 | const timer = setTimeout(done, ms); | |
| 32 | signal.addEventListener("abort", done); | |
| 33 | }); | |
| 34 | } |
src/crew/runner.ts +26
| @@ -76,6 +76,11 @@ import { | ||
| 76 | 76 | type RunReason, |
| 77 | 77 | } from "./brief.ts"; |
| 78 | 78 | import { policyBrief } from "../contract/audit.ts"; |
| 79 | import { | |
| 80 | isNetworkError, | |
| 81 | NETWORK_RETRY_SECONDS, | |
| 82 | pause, | |
| 83 | } from "./network.ts"; | |
| 79 | 84 | import { BUDGET_REACHED, runClaude, type ClaudeOutcome } from "./claude.ts"; |
| 80 | 85 | import { |
| 81 | 86 | isRateLimited, |
| @@ -125,6 +130,8 @@ export interface CrewOptions { | ||
| 125 | 130 | /** Spend per run when the agent's own budget is 0. */ |
| 126 | 131 | readonly budgetUsd: number; |
| 127 | 132 | readonly runTimeoutMs: number; |
| 133 | /** Scales the wait before a network retry; tests pass 0. @default 1 */ | |
| 134 | readonly networkRetryScale?: number | undefined; | |
| 128 | 135 | /** How long a review run may take. @default BUDGETS.review */ |
| 129 | 136 | readonly reviewBudgetMs?: number | undefined; |
| 130 | 137 | /** A `CLAUDE_CONFIG_DIR` for runs, when the operator wants one. */ |
| @@ -1050,6 +1057,25 @@ async function work( | ||
| 1050 | 1057 | write("the session could not be resumed; starting fresh"); |
| 1051 | 1058 | result = await invoke(prompt, working, undefined); |
| 1052 | 1059 | } |
| 1060 | // A dropped connection says nothing about the work: ask again, in the | |
| 1061 | // same session when there is one, before the run is judged. | |
| 1062 | for (const seconds of NETWORK_RETRY_SECONDS) { | |
| 1063 | if (!isNetworkError(result) || signal.aborted) break; | |
| 1064 | write(`network error; retrying in ${seconds}s`); | |
| 1065 | await pause(seconds * 1000 * (options.networkRetryScale ?? 1), signal); | |
| 1066 | if (signal.aborted || Date.now() >= deadline) break; | |
| 1067 | const again = await invoke( | |
| 1068 | result.sessionId === "" ? prompt : "Continue where you stopped.", | |
| 1069 | working, | |
| 1070 | result.sessionId === "" ? undefined : result.sessionId, | |
| 1071 | ); | |
| 1072 | result = { | |
| 1073 | ...again, | |
| 1074 | sessionId: again.sessionId || result.sessionId, | |
| 1075 | costUsd: result.costUsd + again.costUsd, | |
| 1076 | turns: result.turns + again.turns, | |
| 1077 | }; | |
| 1078 | } | |
| 1053 | 1079 | noteRunProgress(app.db, run.id, { |
| 1054 | 1080 | sessionId: result.sessionId, |
| 1055 | 1081 | costUsd: result.costUsd, |
test/crew-network.test.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * The crew with a stand-in Claude Code: a dropped connection is retried, | |
| 3 | * not judged (#1613). | |
| 4 | */ | |
| 5 | ||
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import type { App } from "../src/app.ts"; | |
| 8 | import type { Repo } from "../src/core/repos.ts"; | |
| 9 | import type { User } from "../src/core/users.ts"; | |
| 10 | import { assignIssue, openIssue } from "../src/actions/issues.ts"; | |
| 11 | import { isNetworkError } from "../src/crew/network.ts"; | |
| 12 | import type { ClaudeOutcome } from "../src/crew/claude.ts"; | |
| 13 | import { | |
| 14 | agentRunSettled, | |
| 15 | startStandInCrew, | |
| 16 | stopStandInCrew, | |
| 17 | type StandInCrew, | |
| 18 | } from "./helpers/crew-standin.ts"; | |
| 19 | ||
| 20 | const outcome = (over: Partial<ClaudeOutcome>): ClaudeOutcome => ({ | |
| 21 | ok: false, | |
| 22 | sessionId: "", | |
| 23 | result: "", | |
| 24 | costUsd: 0, | |
| 25 | turns: 0, | |
| 26 | tokensIn: 0, | |
| 27 | tokensOut: 0, | |
| 28 | reason: "error", | |
| 29 | ...over, | |
| 30 | }); | |
| 31 | ||
| 32 | describe("isNetworkError", () => { | |
| 33 | test("names DNS and socket failures, not the work's", () => { | |
| 34 | expect( | |
| 35 | isNetworkError( | |
| 36 | outcome({ | |
| 37 | result: "API Error: Can't reach the API server (EAI_AGAIN)", | |
| 38 | }), | |
| 39 | ), | |
| 40 | ).toBe(true); | |
| 41 | expect(isNetworkError(outcome({ result: "read ECONNRESET" }))).toBe(true); | |
| 42 | expect( | |
| 43 | isNetworkError(outcome({ result: "You hit your session limit" })), | |
| 44 | ).toBe(false); | |
| 45 | expect(isNetworkError(outcome({ ok: true, result: "EAI_AGAIN" }))).toBe( | |
| 46 | false, | |
| 47 | ); | |
| 48 | }); | |
| 49 | }); | |
| 50 | ||
| 51 | describe("the crew, with a stand-in Claude Code: network errors", () => { | |
| 52 | let standIn: StandInCrew; | |
| 53 | let app: App; | |
| 54 | let alice: User; | |
| 55 | let bot: User; | |
| 56 | let repo: Repo; | |
| 57 | ||
| 58 | beforeAll(async () => { | |
| 59 | standIn = await startStandInCrew(); | |
| 60 | ({ app, alice, bot, repo } = standIn); | |
| 61 | }, 30_000); | |
| 62 | ||
| 63 | afterAll(() => stopStandInCrew(standIn)); | |
| 64 | ||
| 65 | test("a run that lost its connection asks again and lands its work", async () => { | |
| 66 | const issue = openIssue(app, alice, repo, { | |
| 67 | title: "Flaky network", | |
| 68 | body: "NET-FLAKE", | |
| 69 | }); | |
| 70 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 71 | expect(assignIssue(app, alice, repo, issue.value, bot).ok).toBe(true); | |
| 72 | const run = await agentRunSettled( | |
| 73 | app, | |
| 74 | repo, | |
| 75 | (r) => r.issueId === issue.value.id, | |
| 76 | ); | |
| 77 | expect(run.state).toBe("ok"); | |
| 78 | expect(run.summary).toContain("opened pull"); | |
| 79 | }, 60_000); | |
| 80 | }); |
test/helpers/crew-standin.ts +6
| @@ -62,6 +62,11 @@ echo "half-done" > PAUSED.md | ||
| 62 | 62 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"You\u2019ve hit your session limit \u00b7 resets 7:20am (UTC)","total_cost_usd":0.02,"num_turns":2,"session_id":"fake-1","terminal_reason":"api_error"}' |
| 63 | 63 | exit 0 |
| 64 | 64 | fi |
| 65 | if grep -q NET-FLAKE "${promptFile}" && [ ! -f "${dataDir}/flaked" ]; then | |
| 66 | touch "${dataDir}/flaked" | |
| 67 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server — check your internet or DNS (EAI_AGAIN)","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' | |
| 68 | exit 0 | |
| 69 | fi | |
| 65 | 70 | if grep -q "Carry on from where you stopped" "${promptFile}"; then |
| 66 | 71 | if [ -f PAUSED.md ]; then kept=yes; else kept=no; fi |
| 67 | 72 | echo '{"type":"assistant","message":{"content":[{"type":"text","text":"PAUSED.md present: '"$kept"'"}]}}' |
| @@ -105,6 +110,7 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Added CREW | ||
| 105 | 110 | concurrency: 1, |
| 106 | 111 | budgetUsd: 1, |
| 107 | 112 | runTimeoutMs: 60_000, |
| 113 | networkRetryScale: 0, | |
| 108 | 114 | reviewBudgetMs: 4_000, |
| 109 | 115 | }); |
| 110 | 116 | const a = await createUser(app.db, { |
new above
offline
Broad alternatives like ECONNREFUSED and connection error can match unrelated failure text. Prefer gating on the api_error terminal reason as well.