Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 1090 minutes today: Release 0.7.354 is unproven: https://forge.open-platform.sh/boards as desktop /boards took 1031 ms, over 1000 ms (statu… #9033
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9022-sentinel-release-was-degraded-1090-minut → main 1 commit fork
2 files
changed +46
−2
src/release/smoke.ts +20 −1
| @@ -11,12 +11,17 @@ | ||
| 11 | 11 | * must answer 200 within `SLOW_MS`, then polls `/healthz` once a second for |
| 12 | 12 | * `WATCH_MS`, rotating through those pages, and fails naming the page that |
| 13 | 13 | * was slow or failed (#6898: `/readyz` never touches the database, 0.7.277 hung). |
| 14 | * An answer slow on the probe's clock whose `Server-Timing` says the forge | |
| 15 | * itself spent `SLOW_MS` or less is the road to the public edge, not the | |
| 16 | * release, and passes (#9022); a hang still fails. | |
| 14 | 17 | * |
| 15 | 18 | * The running forge proves each new version with it (`release/prove.ts`), so |
| 16 | 19 | * it lives here: the image ships src/ and nothing else (#2652). |
| 17 | 20 | * `bun scripts/smoke.ts <url>` is the same checks from a shell. |
| 18 | 21 | */ |
| 19 | 22 | |
| 23 | import { parseServerTiming } from "../perf/record.ts"; | |
| 24 | ||
| 20 | 25 | export const AGENTS: Record<string, string> = { |
| 21 | 26 | phone: |
| 22 | 27 | "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1", |
| @@ -33,6 +38,15 @@ export const REPO_HOME = "/trevato/forge"; | ||
| 33 | 38 | /** The heaviest pages; `/boards` is fetched with the token. */ |
| 34 | 39 | export const HEAVY_PAGES: readonly string[] = ["/boards", "/", "/explore", "/search?q=a", REPO_HOME]; |
| 35 | 40 | |
| 41 | /** | |
| 42 | * How long the forge says it spent on `res`: the `app` entry of its | |
| 43 | * `Server-Timing` (`http/server.ts`), or undefined when there is none. | |
| 44 | */ | |
| 45 | export function forgeMs(res: Response): number | undefined { | |
| 46 | const app = parseServerTiming(res.headers.get("server-timing") ?? "").find((p) => p.name === "app"); | |
| 47 | return app && Number.isFinite(app.ms) ? app.ms : undefined; | |
| 48 | } | |
| 49 | ||
| 36 | 50 | export interface Heavy { |
| 37 | 51 | watchMs?: number; |
| 38 | 52 | pollMs?: number; |
| @@ -151,7 +165,12 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa | ||
| 151 | 165 | const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(slowMs * 2), headers: { ...headers, ...(path === "/healthz" ? heavy.probe : {}), ...auth } }); |
| 152 | 166 | const body = await res.text(); |
| 153 | 167 | const took = Math.round(performance.now() - started); |
| 154 | if (took > slowMs) return { failure: failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true }; | |
| 168 | if (took > slowMs) { | |
| 169 | // The forge's own time says whether the release was slow or the way to it was (#9022). | |
| 170 | const own = forgeMs(res); | |
| 171 | if (own === undefined || own > slowMs) return { failure: failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true }; | |
| 172 | console.warn(`smoke: ${path} took ${took} ms, the forge ${Math.round(own)} ms of it; the rest was the network`); | |
| 173 | } | |
| 155 | 174 | if (path === REPO_HOME && !heavy.repo && res.status === 404) return { failure: null, slow: false }; |
| 156 | 175 | if (path === "/healthz" && heavy.version && res.headers.get("x-forge-version") !== heavy.version) |
| 157 | 176 | return gone(`/healthz reported X-Forge-Version ${res.headers.get("x-forge-version") ?? "none"}, wanted ${heavy.version}`, res.status); |
test/release-smoke.test.ts +26 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | import { afterEach, expect, test } from "bun:test"; |
| 2 | import { HEAVY_PAGES, smoke } from "../src/release/smoke.ts"; | |
| 2 | import { HEAVY_PAGES, forgeMs, smoke } from "../src/release/smoke.ts"; | |
| 3 | 3 | |
| 4 | 4 | let server: ReturnType<typeof Bun.serve> | undefined; |
| 5 | 5 | afterEach(() => server?.stop(true)); |
| @@ -107,6 +107,31 @@ test("a polled heavy page that is slow twice in a row in the watch fails", async | ||
| 107 | 107 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("took"))).toBe(true); |
| 108 | 108 | }); |
| 109 | 109 | |
| 110 | test("a page slow on the wall clock whose Server-Timing says the forge was quick passes (#9022)", async () => { | |
| 111 | // Every /explore answer arrives late, but the forge says it spent 5 ms: the delay is the road, not the release. | |
| 112 | const quick = (p: string) => (p === "/explore" ? new Response("page", { headers: { "server-timing": "db;dur=1.0, app;dur=5.0" } }) : undefined); | |
| 113 | const failures = await smoke(forge(0, [], quick, (p) => (p === "/explore" ? 300 : 0)), "t", { heavy }); | |
| 114 | expect(failures).toEqual([]); | |
| 115 | }); | |
| 116 | ||
| 117 | test("a page whose Server-Timing says the forge itself was slow still fails", async () => { | |
| 118 | const slowOwn = (p: string) => (p === "/explore" ? new Response("page", { headers: { "server-timing": "app;dur=290.0" } }) : undefined); | |
| 119 | const failures = await smoke(forge(0, [], slowOwn, (p) => (p === "/explore" ? 300 : 0)), "t", { heavy }); | |
| 120 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("took"))).toBe(true); | |
| 121 | }); | |
| 122 | ||
| 123 | test("a hang is a failure whatever Server-Timing would have said", async () => { | |
| 124 | const quick = (p: string) => (p === "/explore" ? new Response("page", { headers: { "server-timing": "app;dur=5.0" } }) : undefined); | |
| 125 | const failures = await smoke(forge(0, [], quick, (p, hit) => (p === "/explore" && hit === 1 ? 2_000 : 0)), "t", { heavy }); | |
| 126 | expect(failures.some((f) => f.url.endsWith("/explore") && f.why.includes("did not answer"))).toBe(true); | |
| 127 | }); | |
| 128 | ||
| 129 | test("forgeMs reads the app entry of Server-Timing, and nothing when there is none", () => { | |
| 130 | expect(forgeMs(new Response("", { headers: { "server-timing": "db;dur=2.5, app;dur=7.6, render;dur=2.4" } }))).toBe(7.6); | |
| 131 | expect(forgeMs(new Response("", { headers: { "server-timing": "render;dur=2.4" } }))).toBeUndefined(); | |
| 132 | expect(forgeMs(new Response(""))).toBeUndefined(); | |
| 133 | }); | |
| 134 | ||
| 110 | 135 | test("a hung first load is not asked again", async () => { |
| 111 | 136 | const seen: string[] = []; |
| 112 | 137 | const failures = await smoke(forge(0, seen, () => undefined, (p, hit) => (p === "/boards" && hit === 1 ? 2_000 : 0)), "t", { heavy }); |
new above
offline