Public: anyone can read this; the crew can change it.

Sentinel: probe:probe-5893f874 was degraded 25 minutes today: The release proof's probe forge probe-5893f874 could not be removed and is still running. #12557

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12556-sentinel-probe-probe-5893f874-was-degrad → main 1 commit fork

  • Merged. scout/12556-sentinel-probe-probe-5893f874-was-degrad was merged into main as 32c41b9.

Review

Sign in to review this change.
2 files changed +20 −1
src/release/journey.ts +1 −1
@@ -72,7 +72,7 @@ export const SWEEP_BUDGET_MS = 5_000;
7272 /** At most this many stale forges are removed per run; a host with more clears over several. */
7373 export const SWEEP_MAX = 5;
7474 /** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */
75 export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000];
75 export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000, 3000, 10_000];
7676
7777 /** A probe forge older than this was left by a run that never finished: an hour. */
7878 export const STALE_PROBE_MS = 60 * 60 * 1000;
test/release-journey.test.ts +19
@@ -282,3 +282,22 @@ test("a sweep that throws comes back as its reason and never throws", async () =
282282 const broke = (async () => { throw new Error("the list is down"); }) as unknown as typeof fetch;
283283 expect(await sweepProbes(broke, new URL(parentServers.url), adminToken, Date.now())).toEqual({ gone: [], error: "the list is down" });
284284 });
285
286 test("a removal that fails three times is still retried, so a slow host does not leave the probe forge running", async () => {
287 expect(REMOVE_BACKOFF_MS.length).toBeGreaterThanOrEqual(4);
288 const inner = routed([]);
289 let failed = 0;
290 const slow = (async (input: string | URL | Request, init?: RequestInit) => {
291 if (/\/forges\/probe-[0-9a-f]{8}\/remove$/.test(new URL(String(input)).pathname) && init?.method === "POST" && failed < 4) {
292 failed += 1;
293 return new Response("busy", { status: 503 });
294 }
295 return inner(input, init);
296 }) as typeof fetch;
297 let t = 0;
298 const result = await walkJourney({ base: parentServers.url, fetch: slow, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async () => {} });
299 expect(failed).toBe(4);
300 expect(result.removed).toBe(true);
301 expect(result.failure === undefined || !result.failure.includes("could not be removed")).toBe(true);
302 await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
303 }, 60_000);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs