Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 1057 minutes today: Release 0.7.345 is unproven: https://forge.open-platform.sh/healthz as desktop /healthz took 1086 ms, over 1000 ms (sta… #8918
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8917-sentinel-release-was-degraded-1057-minut → main 2 commits fork 2 threads · 2 unresolved
2 files
changed +33
−9
src/release/smoke.ts +15 −7
| @@ -166,8 +166,8 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 166 | 166 | // The cause of 0.7.279's 1098 ms /boards is unconfirmed; a cold cache after |
| 167 | 167 | // the boot is the guess (#7198). So each page's first load, and only that, |
| 168 | 168 | // is asked again after a slow answer (never a hang or an error), and the |
| 169 | // first slow answer is printed. The watch below calls `once`: a degradation | |
| 170 | // there is seen on the first slow answer. | |
| 169 | // first slow answer is printed. The watch below asks again once as well, | |
| 170 | // so one slow answer there no longer fails the proof (#8917). | |
| 171 | 171 | const timed = async (path: string): Promise<Failure | null> => { |
| 172 | 172 | const first = await once(path); |
| 173 | 173 | if (!first.slow) return first.failure; |
| @@ -178,14 +178,22 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 178 | 178 | const failed = await timed(path); |
| 179 | 179 | if (failed) return failed; |
| 180 | 180 | } |
| 181 | // In the watch one lost or slow answer in a minute of polling (#7724, #8917) | |
| 182 | // is asked again at once; two in a row is a hang or a real slowness and | |
| 183 | // fails the proof. Unlike `timed`, a hung answer is asked again too: the | |
| 184 | // first load is judged once, but the watch polls for a minute and one lost | |
| 185 | // answer among dozens says little. The first slow answer is printed. | |
| 186 | const again = async (path: string) => { | |
| 187 | const first = await once(path); | |
| 188 | if (!first.hung && !first.slow) return first; | |
| 189 | console.warn(`smoke: ${first.failure?.why}; asking again`); | |
| 190 | return once(path); | |
| 191 | }; | |
| 181 | 192 | const deadline = Date.now() + (heavy.watchMs ?? WATCH_MS); |
| 182 | 193 | const pollMs = heavy.pollMs ?? WATCH_POLL_MS; |
| 183 | 194 | for (let tick = 0; Date.now() < deadline; tick++) { |
| 184 | // One lost answer in a minute of polling (#7724) is asked again at once; | |
| 185 | // two in a row is a hang and fails the proof. | |
| 186 | let health = await once("/healthz"); | |
| 187 | if (health.hung) health = await once("/healthz"); | |
| 188 | const failed = health.failure ?? (await once(pages[tick % pages.length]!)).failure; | |
| 195 | const health = await again("/healthz"); | |
| 196 | const failed = health.failure ?? (await again(pages[tick % pages.length]!)).failure; | |
| 189 | 197 | if (failed) return failed; |
| 190 | 198 | await Bun.sleep(pollMs); |
| 191 | 199 | } |
test/release-smoke.test.ts +18 −2
| @@ -85,12 +85,28 @@ test("a page slow on both first-load attempts fails", async () => { | ||
| 85 | 85 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("took"))).toBe(true); |
| 86 | 86 | }); |
| 87 | 87 | |
| 88 | test("a /healthz that is slow once in the watch fails, with no retry", async () => { | |
| 89 | // hit 1 is the pre-watch check of /healthz by smoke; the watch's polls follow | |
| 88 | test("a /healthz that is slow once in the watch is asked again and passes (#8917)", async () => { | |
Add a test for a polled heavy page that is slow once (passes) and one that is slow twice (fails). The page half of |
||
| 90 | 89 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && hit === 4 ? 300 : 0)), "t", { heavy }); |
| 90 | expect(failures.filter((f) => f.url.endsWith("/healthz"))).toEqual([]); | |
| 91 | }); | |
| 92 | ||
| 93 | test("a /healthz that is slow twice in a row in the watch fails", async () => { | |
| 94 | // hit 1 is the pre-watch check of /healthz by smoke; the watch's polls follow | |
| 95 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && (hit === 4 || hit === 5) ? 300 : 0)), "t", { heavy }); | |
| 91 | 96 | expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("took"))).toBe(true); |
| 92 | 97 | }); |
| 93 | 98 | |
| 99 | test("a polled heavy page that is slow once in the watch is asked again and passes", async () => { | |
| 100 | // hit 1 is the first load; hit 2 is the watch's poll of /explore | |
| 101 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/explore" && hit === 2 ? 300 : 0)), "t", { heavy }); | |
| 102 | expect(failures).toEqual([]); | |
| 103 | }); | |
| 104 | ||
| 105 | test("a polled heavy page that is slow twice in a row in the watch fails", async () => { | |
| 106 | const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/explore" && (hit === 2 || hit === 3) ? 300 : 0)), "t", { heavy }); | |
| 107 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("took"))).toBe(true); | |
| 108 | }); | |
| 109 | ||
| 94 | 110 | test("a hung first load is not asked again", async () => { |
| 95 | 111 | const seen: string[] = []; |
| 96 | 112 | const failures = await smoke(forge(0, seen, () => undefined, (p, hit) => (p === "/boards" && hit === 1 ? 2_000 : 0)), "t", { heavy }); |
new above
offline
Hoist
againout of the loop, and warn on the first slow answer astimeddoes, so a recurring slowness is logged.