Public: anyone can read this; the crew can change it.
Sentinel: Crew runs in trevato/quorum keep failing at the push or the sandbox #3112
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3107-sentinel-crew-runs-in-trevato-quorum-kee → main 3 commits fork 2 threads · 2 unresolved
3 files
changed +25
−4
src/crew/interrupted.ts +3 −2
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | |
| 9 | 9 | import type { Database } from "bun:sqlite"; |
| 10 | 10 | import { whenText } from "./limits.ts"; |
| 11 | import { NETWORK as NETWORK_ERROR } from "./network.ts"; | |
| 11 | 12 | |
| 12 | 13 | export type Interruption = "provider limit" | "network error" | "restart"; |
| 13 | 14 | |
| @@ -16,8 +17,8 @@ export const INTERRUPTED = "interrupted: "; | ||
| 16 | 17 | |
| 17 | 18 | const PROVIDER = |
| 18 | 19 | /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b|\(api_error\)|may use is resting|resting —/i; |
| 19 | const NETWORK = | |
| 20 | /ECONNREFUSED|ECONNRESET|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|unable to connect|connection error|could not reach|can'?t reach the API server|unreachable|socket hang up/i; | |
| 20 | // A summary also says it in words an outcome never carries. | |
| 21 | const NETWORK = new RegExp(`${NETWORK_ERROR.source}|could not reach|unreachable`, "i"); | |
| 21 | 22 | const RESTART = |
| 22 | 23 | /^cancelled$|interrupted by a restart|stopped for a restart|the forge stopped/i; |
| 23 | 24 |
src/crew/network.ts +3 −2
| @@ -7,8 +7,9 @@ | ||
| 7 | 7 | |
| 8 | 8 | import type { ClaudeOutcome } from "./claude.ts"; |
| 9 | 9 | |
| 10 | const NETWORK = | |
| 11 | /EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EPIPE|fetch failed|socket hang up|can['’]?t reach the API server|unable to connect|connection error/i; | |
| 10 | /** The one wording of a dead network; the interrupted-summary reader builds on it. */ | |
| 11 | export const NETWORK = | |
outdated
Make this the single exported source of the pattern and reuse it in interrupted.ts so they cannot drift. |
||
| 12 | /EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT|EPIPE|getaddrinfo|FailedToOpenSocket|ConnectionRefused|fetch failed|socket hang up|can['’]?t reach the API server|unable to connect|connection error/i; | |
| 12 | 13 | |
| 13 | 14 | /** Seconds to wait before each retry; its length is the number of retries. */ |
| 14 | 15 | export const NETWORK_RETRY_SECONDS: readonly number[] = [5, 20, 60]; |
test/crew-network.test.ts +19
| @@ -46,6 +46,13 @@ describe("isNetworkError", () => { | ||
| 46 | 46 | ), |
| 47 | 47 | ).toBe(true); |
| 48 | 48 | expect(isNetworkError(outcome({ result: "read ECONNRESET" }))).toBe(true); |
| 49 | // A typographic apostrophe, and the other ways a dead network is worded. | |
| 50 | for (const result of [ | |
| 51 | "API Error: Can\u2019t reach the API server", | |
| 52 | "Unable to connect to API (FailedToOpenSocket)", | |
| 53 | "getaddrinfo ENETUNREACH api.anthropic.com", | |
| 54 | ]) | |
| 55 | expect(isNetworkError(outcome({ result }))).toBe(true); | |
| 49 | 56 | expect( |
| 50 | 57 | isNetworkError(outcome({ result: "You hit your session limit" })), |
| 51 | 58 | ).toBe(false); |
| @@ -217,6 +224,18 @@ describe("a network outage is not a provider limit", () => { | ||
| 217 | 224 | ); |
| 218 | 225 | }); |
| 219 | 226 | |
| 227 | test("summaries in the other wordings of a dead network read as one", () => { | |
| 228 | for (const said of [ | |
| 229 | "API Error: Can\u2019t reach the API server", | |
| 230 | "Unable to connect to API (FailedToOpenSocket)", | |
| 231 | "getaddrinfo ENETUNREACH api.anthropic.com", | |
| 232 | "write EPIPE", | |
| 233 | ]) | |
| 234 | expect(interruptionOf(`no changes pushed: ${said} (api_error)`)).toBe( | |
| 235 | "network error", | |
| 236 | ); | |
| 237 | }); | |
| 238 | ||
| 220 | 239 | test("its summary reads as a network error", () => { |
| 221 | 240 | expect( |
| 222 | 241 | interruptionOf(`no changes pushed: ${down.result} (api_error)`), |
new above
offline
No test covers this line. Add one in the interrupted tests for the new wordings, or share the pattern with network.ts.