Public: anyone can read this; the crew can change it.
Sentinel: Crew runs in trevato/quorum keep failing at the push or the sandbox #2012
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2010-sentinel-crew-runs-in-trevato-quorum-kee → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +124
−30
src/crew/runner.ts +63 −29
| @@ -1058,44 +1058,66 @@ async function work( | ||
| 1058 | 1058 | write("the session could not be resumed; starting fresh"); |
| 1059 | 1059 | result = await invoke(prompt, working, undefined); |
| 1060 | 1060 | } |
| 1061 | // A dropped connection says nothing about the work: ask again, in the | |
| 1062 | // same session when there is one, before the run is judged. | |
| 1063 | for (const seconds of NETWORK_RETRY_SECONDS) { | |
| 1064 | if (!isNetworkError(result) || signal.aborted) break; | |
| 1065 | write(`network error; retrying in ${seconds}s`); | |
| 1066 | await pause(seconds * 1000 * (options.networkRetryScale ?? 1), signal); | |
| 1067 | if (signal.aborted || Date.now() >= deadline) break; | |
| 1068 | const again = await invoke( | |
| 1069 | result.sessionId === "" ? prompt : "Continue where you stopped.", | |
| 1070 | working, | |
| 1071 | result.sessionId === "" ? undefined : result.sessionId, | |
| 1072 | ); | |
| 1073 | result = { | |
| 1074 | ...again, | |
| 1075 | sessionId: again.sessionId || result.sessionId, | |
| 1076 | costUsd: result.costUsd + again.costUsd, | |
| 1077 | turns: result.turns + again.turns, | |
| 1078 | }; | |
| 1079 | } | |
| 1061 | // A dropped connection says nothing about the work: ask again before the | |
| 1062 | // run is judged. `next` names the prompt and session for each new ask, | |
| 1063 | // given what the last one left; the main turn and the wrap-up share this. | |
| 1064 | const retrying = async ( | |
| 1065 | first: typeof result, | |
| 1066 | budgetUsd: number, | |
| 1067 | what: string, | |
| 1068 | next: (last: typeof result) => { | |
| 1069 | prompt: string; | |
| 1070 | sessionId: string | undefined; | |
| 1071 | }, | |
| 1072 | ): Promise<typeof result> => { | |
| 1073 | let current = first; | |
| 1074 | for (const seconds of NETWORK_RETRY_SECONDS) { | |
| 1075 | if (!isNetworkError(current) || signal.aborted) break; | |
| 1076 | write(`network error; retrying${what} in ${seconds}s`); | |
| 1077 | await pause(seconds * 1000 * (options.networkRetryScale ?? 1), signal); | |
| 1078 | if (signal.aborted || Date.now() >= deadline) break; | |
| 1079 | const ask = next(current); | |
| 1080 | const again = await invoke(ask.prompt, budgetUsd, ask.sessionId); | |
| 1081 | current = { | |
| 1082 | ...again, | |
| 1083 | sessionId: again.sessionId || current.sessionId, | |
| 1084 | costUsd: current.costUsd + again.costUsd, | |
| 1085 | turns: current.turns + again.turns, | |
| 1086 | }; | |
| 1087 | } | |
| 1088 | return current; | |
| 1089 | }; | |
| 1080 | 1090 | // Still unreachable after the retries: an outage, not a verdict. The run |
| 1081 | 1091 | // pauses with its clone and session and asks again later, as it does for |
| 1082 | 1092 | // a provider limit, rather than ending as "no changes pushed". |
| 1083 | if ( | |
| 1084 | isNetworkError(result) && | |
| 1085 | !signal.aborted && | |
| 1086 | Date.now() < deadline && | |
| 1087 | run.attempt < MAX_PAUSES | |
| 1088 | ) { | |
| 1093 | const outagePause = ( | |
| 1094 | last: typeof result, | |
| 1095 | costUsd: number, | |
| 1096 | ): Outcome | undefined => { | |
| 1097 | if ( | |
| 1098 | !isNetworkError(last) || | |
| 1099 | signal.aborted || | |
| 1100 | Date.now() >= deadline || | |
| 1101 | run.attempt >= MAX_PAUSES | |
| 1102 | ) | |
| 1103 | return undefined; | |
| 1089 | 1104 | const at = new Date(Date.now() + NETWORK_PAUSE_MS); |
| 1090 | 1105 | write(`network still down; paused, resumes ${whenText(at)}`); |
| 1091 | 1106 | return { |
| 1092 | 1107 | state: "paused", |
| 1093 | 1108 | summary: `paused: network error, resumes ${whenText(at)}`, |
| 1094 | 1109 | resumeAt: at.toISOString(), |
| 1095 | sessionId: result.sessionId, | |
| 1096 | costUsd: result.costUsd, | |
| 1110 | sessionId: last.sessionId, | |
| 1111 | costUsd, | |
| 1097 | 1112 | }; |
| 1098 | } | |
| 1113 | }; | |
| 1114 | result = await retrying(result, working, "", (last) => | |
| 1115 | last.sessionId === "" | |
| 1116 | ? { prompt, sessionId: undefined } | |
| 1117 | : { prompt: "Continue where you stopped.", sessionId: last.sessionId }, | |
| 1118 | ); | |
| 1119 | const down = outagePause(result, result.costUsd); | |
| 1120 | if (down !== undefined) return down; | |
| 1099 | 1121 | noteRunProgress(app.db, run.id, { |
| 1100 | 1122 | sessionId: result.sessionId, |
| 1101 | 1123 | costUsd: result.costUsd, |
| @@ -1195,7 +1217,19 @@ async function work( | ||
| 1195 | 1217 | ) { |
| 1196 | 1218 | stoppedAt = result.costUsd; |
| 1197 | 1219 | write(`claude: wrapping up with the reserve, $${reserve.toFixed(2)}`); |
| 1198 | const wrapped = await invoke(wrapUpPrompt(), reserve, result.sessionId); | |
| 1220 | const wrapped = await retrying( | |
| 1221 | await invoke(wrapUpPrompt(), reserve, result.sessionId), | |
| 1222 | reserve, | |
| 1223 | " the wrap-up", | |
| 1224 | (last) => ({ | |
| 1225 | prompt: wrapUpPrompt(), | |
| 1226 | sessionId: last.sessionId || result.sessionId, | |
| 1227 | }), | |
| 1228 | ); | |
| 1229 | // Still unreachable after the retries: the work is not judged on a | |
| 1230 | // wrap-up that never got through; the run pauses as the main turn does. | |
| 1231 | const wrapDown = outagePause(wrapped, result.costUsd + wrapped.costUsd); | |
| 1232 | if (wrapDown !== undefined) return wrapDown; | |
| 1199 | 1233 | write( |
| 1200 | 1234 | `claude: ${wrapped.reason}, ${wrapped.turns} turns, $${wrapped.costUsd.toFixed(2)}`, |
| 1201 | 1235 | ); |
test/crew-network.test.ts +37 −1
| @@ -3,7 +3,7 @@ | ||
| 3 | 3 | * not judged (#1613). |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | import { rmSync } from "node:fs"; | |
| 6 | import { existsSync, rmSync } from "node:fs"; | |
| 7 | 7 | import { join } from "node:path"; |
| 8 | 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 9 | 9 | import type { App } from "../src/app.ts"; |
| @@ -84,6 +84,42 @@ describe("the crew, with a stand-in Claude Code: network errors", () => { | ||
| 84 | 84 | expect(run.summary).toContain("opened pull"); |
| 85 | 85 | }, 60_000); |
| 86 | 86 | |
| 87 | test("a wrap-up that lost its connection asks again and lands the work", async () => { | |
| 88 | const issue = openIssue(app, alice, repo, { | |
| 89 | title: "Flaky wrap-up", | |
| 90 | body: "WRAP-FLAKE", | |
| 91 | }); | |
| 92 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 93 | expect(assignIssue(app, alice, repo, issue.value, bot).ok).toBe(true); | |
| 94 | const run = await agentRunSettled( | |
| 95 | app, | |
| 96 | repo, | |
| 97 | (r) => r.issueId === issue.value.id, | |
| 98 | ); | |
| 99 | expect(run.summary).toContain("opened pull"); | |
| 100 | expect(run.summary).not.toContain("network"); | |
| 101 | expect(existsSync(join(standIn.dataDir, "wrapflaked"))).toBe(true); | |
| 102 | }, 60_000); | |
| 103 | ||
| 104 | test("a wrap-up that never reconnects pauses the run instead of judging it", async () => { | |
| 105 | const issue = openIssue(app, alice, repo, { | |
| 106 | title: "Wrap-up down", | |
| 107 | body: "WRAP-DOWN", | |
| 108 | }); | |
| 109 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 110 | expect(assignIssue(app, alice, repo, issue.value, bot).ok).toBe(true); | |
| 111 | const until = Date.now() + 30_000; | |
| 112 | let found: Run | undefined; | |
| 113 | while (Date.now() < until && !found) { | |
| 114 | found = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 115 | (r) => r.issueId === issue.value.id && r.state === "cancelled", | |
| 116 | ); | |
| 117 | if (!found) await Bun.sleep(50); | |
| 118 | } | |
| 119 | expect(found?.summary).toStartWith("paused: network error"); | |
| 120 | rmSync(join(standIn.dataDir, "wrapdown")); | |
| 121 | }, 60_000); | |
| 122 | ||
| 87 | 123 | test("an outage that outlasts the retries pauses the run; after MAX_PAUSES it ends", async () => { |
| 88 | 124 | const issue = openIssue(app, alice, repo, { |
| 89 | 125 | title: "Network down", |
test/helpers/crew-standin.ts +24
| @@ -67,6 +67,30 @@ if [ -f "${dataDir}/down" ] && grep -q "Continue where you stopped\\|NET-DOWN" " | ||
| 67 | 67 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' |
| 68 | 68 | exit 0 |
| 69 | 69 | fi |
| 70 | if grep -q "You are out of budget" "${promptFile}"; then | |
| 71 | if [ -f "${dataDir}/wrapdown" ]; then | |
| 72 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' | |
| 73 | exit 0 | |
| 74 | fi | |
| 75 | if [ ! -f "${dataDir}/wrapflaked" ]; then | |
| 76 | touch "${dataDir}/wrapflaked" | |
| 77 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server — check your internet or DNS (EAI_AGAIN)","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' | |
| 78 | else | |
| 79 | echo '{"type":"result","subtype":"success","is_error":false,"result":"Wrapped up after the outage.","total_cost_usd":0.01,"num_turns":1,"session_id":"fake-1","terminal_reason":"completed"}' | |
| 80 | fi | |
| 81 | exit 0 | |
| 82 | fi | |
| 83 | if grep -q WRAP-DOWN "${promptFile}"; then | |
| 84 | touch "${dataDir}/wrapdown" | |
| 85 | echo '{"type":"result","subtype":"error_max_budget_usd","is_error":true,"result":"","total_cost_usd":0.5,"num_turns":9,"session_id":"fake-1","terminal_reason":"budget_exhausted"}' | |
| 86 | exit 0 | |
| 87 | fi | |
| 88 | if grep -q WRAP-FLAKE "${promptFile}"; then | |
| 89 | echo "work by the crew $(date +%s%N)" >> CREW.md | |
| 90 | git add -A && git commit -qm "crew work" && git push -q -u origin HEAD | |
| 91 | echo '{"type":"result","subtype":"error_max_budget_usd","is_error":true,"result":"","total_cost_usd":0.5,"num_turns":9,"session_id":"fake-1","terminal_reason":"budget_exhausted"}' | |
| 92 | exit 0 | |
| 93 | fi | |
| 70 | 94 | if grep -q NET-FLAKE "${promptFile}" && [ ! -f "${dataDir}/flaked" ]; then |
| 71 | 95 | touch "${dataDir}/flaked" |
| 72 | 96 | echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"API Error: Can'"'"'t reach the API server — check your internet or DNS (EAI_AGAIN)","total_cost_usd":0,"num_turns":0,"session_id":"fake-1","terminal_reason":"api_error"}' |
new above
offline
The existing loop is wrapped in a
retriedclosure that is called once and never reused. The wrap-up below copies the loop instead of calling it. Either make this a shared helper parameterised by the next prompt, budget and session, or leave the original loop as it was. A one-use closure plus a copy is the worst of both.