Public: anyone can read this; the crew can change it.
Sentinel: probe:probe-5893f874 was degraded 25 minutes today: The release proof's probe forge probe-5893f874 could not be removed and is still running. #12557
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12556-sentinel-probe-probe-5893f874-was-degrad → main 1 commit fork
2 files
changed +20
−1
src/release/journey.ts +1 −1
| @@ -72,7 +72,7 @@ export const SWEEP_BUDGET_MS = 5_000; | ||
| 72 | 72 | /** At most this many stale forges are removed per run; a host with more clears over several. */ |
| 73 | 73 | export const SWEEP_MAX = 5; |
| 74 | 74 | /** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */ |
| 75 | export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000]; | |
| 75 | export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000, 3000, 10_000]; | |
| 76 | 76 | |
| 77 | 77 | /** A probe forge older than this was left by a run that never finished: an hour. */ |
| 78 | 78 | export const STALE_PROBE_MS = 60 * 60 * 1000; |
test/release-journey.test.ts +19
| @@ -282,3 +282,22 @@ test("a sweep that throws comes back as its reason and never throws", async () = | ||
| 282 | 282 | const broke = (async () => { throw new Error("the list is down"); }) as unknown as typeof fetch; |
| 283 | 283 | expect(await sweepProbes(broke, new URL(parentServers.url), adminToken, Date.now())).toEqual({ gone: [], error: "the list is down" }); |
| 284 | 284 | }); |
| 285 | ||
| 286 | test("a removal that fails three times is still retried, so a slow host does not leave the probe forge running", async () => { | |
| 287 | expect(REMOVE_BACKOFF_MS.length).toBeGreaterThanOrEqual(4); | |
| 288 | const inner = routed([]); | |
| 289 | let failed = 0; | |
| 290 | const slow = (async (input: string | URL | Request, init?: RequestInit) => { | |
| 291 | if (/\/forges\/probe-[0-9a-f]{8}\/remove$/.test(new URL(String(input)).pathname) && init?.method === "POST" && failed < 4) { | |
| 292 | failed += 1; | |
| 293 | return new Response("busy", { status: 503 }); | |
| 294 | } | |
| 295 | return inner(input, init); | |
| 296 | }) as typeof fetch; | |
| 297 | let t = 0; | |
| 298 | const result = await walkJourney({ base: parentServers.url, fetch: slow, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async () => {} }); | |
| 299 | expect(failed).toBe(4); | |
| 300 | expect(result.removed).toBe(true); | |
| 301 | expect(result.failure === undefined || !result.failure.includes("could not be removed")).toBe(true); | |
| 302 | await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } }); | |
| 303 | }, 60_000); |
new above
offline