Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 65 minutes today: Release 0.7.293 is unproven: https://forge.open-platform.sh/healthz as desktop /healthz did not answer within 2000 ms (… #7747
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/7724-sentinel-release-was-degraded-65-minutes → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +17
−3
src/release/smoke.ts +7 −3
| @@ -141,7 +141,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 141 | 141 | const headers: Record<string, string> = { "user-agent": AGENTS["desktop"]! }; |
| 142 | 142 | const pages = HEAVY_PAGES.map((p) => (p === REPO_HOME && heavy.repo ? heavy.repo : p)); |
| 143 | 143 | // An answer between `slowMs` and twice it is slow; none by then is a hang. |
| 144 | const once = async (path: string): Promise<{ failure: Failure | null; slow: boolean }> => { | |
| 144 | const once = async (path: string): Promise<{ failure: Failure | null; slow: boolean; hung?: boolean }> => { | |
| 145 | 145 | const url = new URL(path, root).href; |
| 146 | 146 | const started = performance.now(); |
| 147 | 147 | const failureFor = (why: string, status: number | string = "error"): Failure => ({ url, agent: "desktop", status, contentType: "", why }); |
| @@ -160,7 +160,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 160 | 160 | if (!body.trim()) return gone(`${path} answered with an empty body`, res.status); |
| 161 | 161 | return { failure: null, slow: false }; |
| 162 | 162 | } catch (error) { |
| 163 | return gone(`${path} did not answer within ${slowMs * 2} ms (${error})`); | |
| 163 | return { ...gone(`${path} did not answer within ${slowMs * 2} ms (${error})`), hung: true }; | |
| 164 | 164 | } |
| 165 | 165 | }; |
| 166 | 166 | // The cause of 0.7.279's 1098 ms /boards is unconfirmed; a cold cache after |
| @@ -181,7 +181,11 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 181 | 181 | const deadline = Date.now() + (heavy.watchMs ?? WATCH_MS); |
| 182 | 182 | const pollMs = heavy.pollMs ?? WATCH_POLL_MS; |
| 183 | 183 | for (let tick = 0; Date.now() < deadline; tick++) { |
| 184 | const failed = (await once("/healthz")).failure ?? (await once(pages[tick % pages.length]!)).failure; | |
| 184 | // One lost answer in a minute of polling (#7724) is asked again at once; | |
| 185 | // two in a row is a hang and fails the proof. | |
| 186 | let health = await once("/healthz"); | |
| 187 | if (health.hung) health = await once("/healthz"); | |
| 188 | const failed = health.failure ?? (await once(pages[tick % pages.length]!)).failure; | |
| 185 | 189 | if (failed) return failed; |
| 186 | 190 | await Bun.sleep(pollMs); |
| 187 | 191 | } |
test/release-smoke.test.ts +10
| @@ -96,3 +96,13 @@ test("a hung first load is not asked again", async () => { | ||
| 96 | 96 | const failures = await smoke(forge(0, seen, () => undefined, (p, hit) => (p === "/boards" && hit === 1 ? 2_000 : 0)), "t", { heavy }); |
| 97 | 97 | expect(failures.some((f) => f.url.endsWith("/boards") && f.why.includes("did not answer"))).toBe(true); |
| 98 | 98 | }); |
| 99 | ||
| 100 | test("a /healthz that gets no answer once in the watch is asked again and passes (#7724)", async () => { | |
| 101 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && hit === 4 ? 1_000 : 0)), "t", { heavy }); | |
| 102 | expect(failures.filter((f) => f.url.endsWith("/healthz"))).toEqual([]); | |
| 103 | }); | |
| 104 | ||
| 105 | test("a /healthz that gets no answer twice in a row in the watch fails (#7724)", async () => { | |
| 106 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && (hit === 4 || hit === 5) ? 1_000 : 0)), "t", { heavy }); | |
| 107 | expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("did not answer"))).toBe(true); | |
| 108 | }); |
new above
offline
Print the first miss before asking again (
console.warn("smoke: ...; asking again")), astimeddoes for slow answers. Otherwise a flaky /healthz leaves no trace and the cause stays unfindable.