Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 30 minutes today: Release 0.7.522 is unproven: https://forge.open-platform.sh/search?q=a as desktop /search?q=a took 2615 ms, over 1000 m… #11016
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/10993-sentinel-release-was-degraded-30-minutes → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +18
−1
src/release/smoke.ts +5
| @@ -35,6 +35,8 @@ export const AGENTS: Record<string, string> = { | ||
| 35 | 35 | export const WATCH_MS = 60_000; |
| 36 | 36 | export const WATCH_POLL_MS = 1_000; |
| 37 | 37 | export const SLOW_MS = 1_000; |
| 38 | /** The pause before a slow or lost first load is asked again: a boot's stall is seconds, a slow release stays slow (#10993). */ | |
| 39 | export const SETTLE_MS = 3_000; | |
| 38 | 40 | /** One repository's home, for the heavy-page load. */ |
| 39 | 41 | export const REPO_HOME = "/trevato/forge"; |
| 40 | 42 | /** The heaviest pages; `/boards` is fetched with the token. */ |
| @@ -67,6 +69,8 @@ export interface Heavy { | ||
| 67 | 69 | watchMs?: number; |
| 68 | 70 | pollMs?: number; |
| 69 | 71 | slowMs?: number; |
| 72 | /** How long a slow first load waits before it is asked again, so a stall that follows the boot has passed (#10993). @default SETTLE_MS */ | |
| 73 | settleMs?: number; | |
| 70 | 74 | /** The repository whose home is loaded; unset, `REPO_HOME` is tried and a 404 there is skipped. */ |
| 71 | 75 | repo?: string; |
| 72 | 76 | /** Every `/healthz` answer in the watch must carry this `X-Forge-Version`. */ |
| @@ -233,6 +237,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 233 | 237 | const first = await once(path); |
| 234 | 238 | if (!first.slow && !first.empty && !first.hung) return first.failure; |
| 235 | 239 | console.warn(`smoke: ${first.failure?.why}; asking again`); |
| 240 | await Bun.sleep(heavy.settleMs ?? SETTLE_MS); | |
| 236 | 241 | return (await once(path)).failure; |
| 237 | 242 | }; |
| 238 | 243 | for (const path of pages) { |
test/release-smoke.test.ts +13 −1
| @@ -27,7 +27,7 @@ function forge(boardsMs: number, seen: string[] = [], over: (path: string) => Re | ||
| 27 | 27 | return `http://127.0.0.1:${server.port}`; |
| 28 | 28 | } |
| 29 | 29 | |
| 30 | const heavy = { watchMs: 300, pollMs: 50, slowMs: 200 }; | |
| 30 | const heavy = { watchMs: 300, pollMs: 50, slowMs: 200, settleMs: 0 }; | |
| 31 | 31 | |
| 32 | 32 | test("a healthy forge passes the heavy pages and the /healthz watch", async () => { |
| 33 | 33 | const seen: string[] = []; |
| @@ -242,3 +242,15 @@ test("an empty body under a content-encoding that is empty unencoded too still f | ||
| 242 | 242 | await edge.stop(true); |
| 243 | 243 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("empty body"))).toBe(true); |
| 244 | 244 | }); |
| 245 | ||
| 246 | test("a first load slow for a moment, then quick after the settle pause, passes (#10993)", async () => { | |
| 247 | // A stall right after the boot: every /explore answer within the first 500 ms is slow, so an immediate retry is slow too. | |
| 248 | const born = performance.now(); | |
| 249 | const stall = (p: string) => (p === "/explore" && performance.now() - born < 500 ? 300 : 0); | |
| 250 | const impatient = await smoke(forge(0, [], () => undefined, stall), "t", { heavy }); | |
| 251 | expect(impatient.some((f) => f.url.endsWith("/explore") && f.why.includes("took"))).toBe(true); | |
| 252 | const born2 = performance.now(); | |
| 253 | const stall2 = (p: string) => (p === "/explore" && performance.now() - born2 < 500 ? 300 : 0); | |
| 254 | const settled = await smoke(forge(0, [], () => undefined, stall2), "t", { heavy: { ...heavy, settleMs: 300 } }); | |
| 255 | expect(settled).toEqual([]); | |
| 256 | }); |
The stall window is wall-clock from forge creation. If earlier pages take more than about 200 ms on a busy runner, the impatient retry lands after 500 ms and the test fails. Count the slow answers instead: the first two /explore answers are slow.