Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 622 minutes today: Release 0.7.374 is unproven: https://forge.open-platform.sh/explore as desktop /explore answered with an empty body (st… #9340
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9334-sentinel-release-was-degraded-622-minute → main 2 commits fork 1 thread · 1 unresolved
2 files
changed +34
−4
src/release/smoke.ts +6 −4
| @@ -11,6 +11,8 @@ | ||
| 11 | 11 | * must answer 200 within `SLOW_MS`, then polls `/healthz` once a second for |
| 12 | 12 | * `WATCH_MS`, rotating through those pages, and fails naming the page that |
| 13 | 13 | * was slow or failed (#6898: `/readyz` never touches the database, 0.7.277 hung). |
| 14 | * An empty 200 body is asked again once, like a slow answer: one lost body in | |
| 15 | * a release's proof says little, two in a row fail it (#9334). | |
| 14 | 16 | * An answer slow on the probe's clock whose `Server-Timing` says the forge |
| 15 | 17 | * itself spent `SLOW_MS` or less is the road to the public edge, not the |
| 16 | 18 | * release, and passes (#9022); a hang still fails. |
| @@ -157,7 +159,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 157 | 159 | const headers: Record<string, string> = { "user-agent": AGENTS["desktop"]! }; |
| 158 | 160 | const pages = HEAVY_PAGES.map((p) => (p === REPO_HOME && heavy.repo ? heavy.repo : p)); |
| 159 | 161 | // An answer between `slowMs` and twice it is slow; none by then is a hang. |
| 160 | const once = async (path: string): Promise<{ failure: Failure | null; slow: boolean; hung?: boolean }> => { | |
| 162 | const once = async (path: string): Promise<{ failure: Failure | null; slow: boolean; hung?: boolean; empty?: boolean }> => { | |
| 161 | 163 | const url = new URL(path, root).href; |
| 162 | 164 | const started = performance.now(); |
| 163 | 165 | const failureFor = (why: string, status: number | string = "error"): Failure => ({ url, agent: "desktop", status, contentType: "", why }); |
| @@ -178,7 +180,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 178 | 180 | return gone(`/healthz reported X-Forge-Version ${res.headers.get("x-forge-version") ?? "none"}, wanted ${heavy.version}`, res.status); |
| 179 | 181 | if (path === "/boards" && anonymousBoard && !token && isSignInRedirect(url, res)) return { failure: null, slow: false }; |
| 180 | 182 | if (res.status !== 200) return gone(`${path} expected 200`, res.status); |
| 181 | if (!body.trim()) return gone(`${path} answered with an empty body`, res.status); | |
| 183 | if (!body.trim()) return { ...gone(`${path} answered with an empty body`, res.status), empty: true }; | |
| 182 | 184 | return { failure: null, slow: false }; |
| 183 | 185 | } catch (error) { |
| 184 | 186 | return { ...gone(`${path} did not answer within ${slowMs * 2} ms (${error})`), hung: true }; |
| @@ -191,7 +193,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 191 | 193 | // so one slow answer there no longer fails the proof (#8917). |
| 192 | 194 | const timed = async (path: string): Promise<Failure | null> => { |
| 193 | 195 | const first = await once(path); |
| 194 | if (!first.slow) return first.failure; | |
| 196 | if (!first.slow && !first.empty) return first.failure; | |
| 195 | 197 | console.warn(`smoke: ${first.failure?.why}; asking again`); |
| 196 | 198 | return (await once(path)).failure; |
| 197 | 199 | }; |
| @@ -206,7 +208,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 206 | 208 | // answer among dozens says little. The first slow answer is printed. |
| 207 | 209 | const again = async (path: string) => { |
| 208 | 210 | const first = await once(path); |
| 209 | if (!first.hung && !first.slow) return first; | |
| 211 | if (!first.hung && !first.slow && !first.empty) return first; | |
| 210 | 212 | console.warn(`smoke: ${first.failure?.why}; asking again`); |
| 211 | 213 | return once(path); |
| 212 | 214 | }; |
test/release-smoke.test.ts +28
| @@ -152,3 +152,31 @@ test("a /healthz that gets no answer twice in a row in the watch fails (#7724)", | ||
| 152 | 152 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && (hit === 4 || hit === 5) ? 1_000 : 0)), "t", { heavy }); |
| 153 | 153 | expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("did not answer"))).toBe(true); |
| 154 | 154 | }); |
| 155 | ||
| 156 | test("a heavy page that answers an empty body once is asked again and passes (#9334)", async () => { | |
| 157 | let n = 0; | |
| 158 | const flaky = (p: string) => (p === "/explore" && ++n === 1 ? new Response("", { status: 200 }) : undefined); | |
| 159 | expect(await smoke(forge(0, [], flaky), "t", { heavy })).toEqual([]); | |
| 160 | }); | |
| 161 | ||
| 162 | test("a heavy page that answers an empty body twice in a row fails (#9334)", async () => { | |
| 163 | let n = 0; | |
| 164 | const flaky = (p: string) => (p === "/explore" && ++n <= 2 ? new Response("", { status: 200 }) : undefined); | |
| 165 | const failures = await smoke(forge(0, [], flaky), "t", { heavy }); | |
| 166 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("empty body"))).toBe(true); | |
| 167 | }); | |
| 168 | ||
| 169 | test("a polled heavy page that answers an empty body once in the watch is asked again and passes (#9334)", async () => { | |
| 170 | // call 1 is the first load; call 2 is the watch's poll of /explore | |
| 171 | let n = 0; | |
| 172 | const flaky = (p: string) => (p === "/explore" && ++n === 2 ? new Response("", { status: 200 }) : undefined); | |
| 173 | expect(await smoke(forge(0, [], flaky), "t", { heavy })).toEqual([]); | |
| 174 | expect(n).toBeGreaterThanOrEqual(3); | |
| 175 | }); | |
| 176 | ||
| 177 | test("a polled heavy page that answers an empty body twice in a row in the watch fails naming it (#9334)", async () => { | |
| 178 | let n = 0; | |
| 179 | const flaky = (p: string) => (p === "/explore" && (n += 1) >= 2 && n <= 3 ? new Response("", { status: 200 }) : undefined); | |
| 180 | const failures = await smoke(forge(0, [], flaky), "t", { heavy }); | |
| 181 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("empty body"))).toBe(true); | |
| 182 | }); |
new above
offline
Watch-path retry has no test. Add one that makes /healthz or a rotated heavy page answer empty once during the watch (passes) and twice in a row (fails naming the page).