trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns13g rEventsFlows

The release smoke loads /boards and the heaviest public pages, then watches /healthz for 60 s before it calls a release live #6901

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/6898-the-release-smoke-loads-boards-and-the-h → main 2 commits fork

  • Merged. scout/6898-the-release-smoke-loads-boards-and-the-h was merged into main as 8a9fb44.

Review

Sign in to review this change.
6 files changed +178 −6
scripts/smoke.ts +10 −2
@@ -16,6 +16,9 @@
1616 * line per finding with its section (`--error-path` names a path that must answer
1717 * 500), and exits 0: it reports, it does not gate.
1818 *
19 * `--version` also loads `/boards`, `/`, `/explore`, `/search?q=a` and a repository home (`--repo`, default
20 * /trevato/forge, skipped on 404) and watches `/healthz` for `--watch` seconds (default 60); see `release/smoke.ts`.
21 *
1922 * `--app-host <url>` also fetches `/` there and fails if it is the forge's page (`<title>forge</title>`).
2023 *
2124 * `--outsider` also signs up a fresh account and proves each thing a
@@ -47,10 +50,13 @@ if (import.meta.main) {
4750 const appHost = flag("--app-host");
4851 const version = flag("--version");
4952 const wait = flag("--wait");
53 const watchFor = flag("--watch");
54 const repo = flag("--repo");
5055 const base = args[0];
5156 const seconds = wait === undefined ? 120 : Number(wait);
52 if (!base || !Number.isFinite(seconds) || seconds < 0 || args.length > 1) {
53 console.error("usage: bun scripts/smoke.ts <url> [--version <v>] [--wait <seconds>] [--outsider] [--app-host <url of an app>] [--standards [--error-path <path that answers 500>]] (--version needs FORGE_PROBE, the forge's probe)");
57 const watchSeconds = watchFor === undefined ? undefined : Number(watchFor);
58 if (!base || (watchSeconds !== undefined && !(watchSeconds >= 0)) || !Number.isFinite(seconds) || seconds < 0 || args.length > 1) {
59 console.error("usage: bun scripts/smoke.ts <url> [--version <v>] [--wait <seconds>] [--watch <seconds of /healthz watching, default 60>] [--repo </owner/name>] [--outsider] [--app-host <url of an app>] [--standards [--error-path <path that answers 500>]] (--version needs FORGE_PROBE, the forge's probe)");
5460 process.exit(2);
5561 }
5662 if (withOutsider && !process.env["FORGE_TOKEN"]) {
@@ -71,6 +77,8 @@ if (import.meta.main) {
7177 }
7278 const failures = await smoke(base, process.env["FORGE_TOKEN"] ?? "", {
7379 version,
80 // A release (`--version`) loads the heavy pages and watches /healthz after go-live (#6898).
81 ...(version ? { heavy: { ...(watchSeconds === undefined ? {} : { watchMs: watchSeconds * 1000 }), ...(repo ? { repo } : {}) } } : {}),
7482 waitMs: seconds * 1000,
7583 ...(appHost ? { appHost } : {}),
7684 // The forge tells only its probe its version (`src/kernel/probe.ts`).
src/release/prove.ts +8 −2
@@ -10,7 +10,7 @@
1010
1111 import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
1212 import { join } from "node:path";
13 import { smoke, type Failure } from "./smoke.ts";
13 import { smoke, type Failure, type Heavy } from "./smoke.ts";
1414 import { UNSTAMPED } from "../kernel/version.ts";
1515 import { ownProbeHeaders } from "../kernel/probe.ts";
1616
@@ -72,6 +72,8 @@ export interface ProveOptions {
7272 readonly now?: () => number;
7373 readonly sleep?: (ms: number) => Promise<void>;
7474 readonly giveUpMs?: number;
75 /** The heavy-page load and `/healthz` watch after go-live; a shorter window for a test. */
76 readonly heavy?: Heavy;
7577 }
7678
7779 /**
@@ -104,7 +106,11 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
104106 if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
105107 await sleep(POLL_MS);
106108 }
107 const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true, ...(o.appHost ? { appHost: o.appHost } : {}) });
109 const failures = await smoke(o.url, o.token ?? "", {
110 fetch: doFetch,
111 anonymousBoard: true,
112 heavy: { version: o.version, probe: ownProbeHeaders(o.dataDir), ...o.heavy }, ...(o.appHost ? { appHost: o.appHost } : {}),
113 });
108114 return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
109115 } catch (error) {
110116 return finish(`the smoke checks could not run: ${String(error)}`);
src/release/smoke.ts +74
@@ -6,6 +6,12 @@
66 * branch never answers with `Response.error()`. With `version`, `/healthz` is
77 * first polled until `X-Forge-Version` is that version.
88 *
9 * With `heavy`, it also loads the heaviest pages (`HEAVY_PAGES`: `/boards`
10 * signed in, `/`, `/explore`, `/search?q=a`, one repository's home), each of which
11 * must answer 200 within `SLOW_MS`, then polls `/healthz` once a second for
12 * `WATCH_MS`, rotating through those pages, and fails naming the page that
13 * was slow or failed (#6898: `/readyz` never touches the database, 0.7.277 hung).
14 *
915 * The running forge proves each new version with it (`release/prove.ts`), so
1016 * it lives here: the image ships src/ and nothing else (#2652).
1117 * `bun scripts/smoke.ts <url>` is the same checks from a shell.
@@ -18,6 +24,27 @@ export const AGENTS: Record<string, string> = {
1824 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
1925 };
2026
27 /** How long after go-live `/healthz` is watched, how often, and the slowest answer allowed. */
28 export const WATCH_MS = 60_000;
29 export const WATCH_POLL_MS = 1_000;
30 export const SLOW_MS = 1_000;
31 /** One repository's home, for the heavy-page load. */
32 export const REPO_HOME = "/trevato/forge";
33 /** The heaviest pages; `/boards` is fetched with the token. */
34 export const HEAVY_PAGES: readonly string[] = ["/boards", "/", "/explore", "/search?q=a", REPO_HOME];
35
36 export interface Heavy {
37 watchMs?: number;
38 pollMs?: number;
39 slowMs?: number;
40 /** The repository whose home is loaded; unset, `REPO_HOME` is tried and a 404 there is skipped. */
41 repo?: string;
42 /** Every `/healthz` answer in the watch must carry this `X-Forge-Version`. */
43 version?: string | undefined;
44 /** Sent with the watch's `/healthz`: the forge tells only its probe its version. */
45 probe?: Readonly<Record<string, string>>;
46 }
47
2148 export interface Failure {
2249 url: string;
2350 agent: string;
@@ -61,6 +88,8 @@ export interface Options {
6188 anonymousBoard?: boolean;
6289 /** An app's URL on this forge: `GET /` there must never be a forge page (#6726). */
6390 appHost?: string;
91 /** Load the heaviest pages, then watch `/healthz` (see the top of this file). Off unless given. */
92 heavy?: Heavy;
6493 }
6594
6695 /** The redirect statuses a sign-in answers with. */
@@ -106,6 +135,46 @@ async function waitForVersion(root: URL, version: string, waitMs: number, pollMs
106135 return { url, agent: "-", status, contentType: "", why: `wanted X-Forge-Version ${version}, last saw ${seen}` };
107136 }
108137
138 /** Loads each of `HEAVY_PAGES`, then polls `/healthz` for the window; the first failure, or null. */
139 async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globalThis.fetch, anonymousBoard: boolean): Promise<Failure | null> {
140 const slowMs = heavy.slowMs ?? SLOW_MS;
141 const headers: Record<string, string> = { "user-agent": AGENTS["desktop"]! };
142 const pages = HEAVY_PAGES.map((p) => (p === REPO_HOME && heavy.repo ? heavy.repo : p));
143 const timed = async (path: string): Promise<Failure | null> => {
144 const url = new URL(path, root).href;
145 const started = performance.now();
146 const gone = (why: string, status: number | string = "error"): Failure => ({ url, agent: "desktop", status, contentType: "", why });
147 try {
148 const auth: Record<string, string> = path === "/boards" && token ? { authorization: `Bearer ${token}` } : {};
149 const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(slowMs), headers: { ...headers, ...(path === "/healthz" ? heavy.probe : {}), ...auth } });
150 const body = await res.text();
151 const took = Math.round(performance.now() - started);
152 if (took > slowMs) return gone(`${path} took ${took} ms, over ${slowMs} ms`, res.status);
153 if (path === REPO_HOME && !heavy.repo && res.status === 404) return null;
154 if (path === "/healthz" && heavy.version && res.headers.get("x-forge-version") !== heavy.version)
155 return gone(`/healthz reported X-Forge-Version ${res.headers.get("x-forge-version") ?? "none"}, wanted ${heavy.version}`, res.status);
156 if (path === "/boards" && anonymousBoard && !token && isSignInRedirect(url, res)) return null;
157 if (res.status !== 200) return gone(`${path} expected 200`, res.status);
158 if (!body.trim()) return gone(`${path} answered with an empty body`, res.status);
159 return null;
160 } catch (error) {
161 return gone(`${path} did not answer within ${slowMs} ms (${error})`);
162 }
163 };
164 for (const path of pages) {
165 const failed = await timed(path);
166 if (failed) return failed;
167 }
168 const deadline = Date.now() + (heavy.watchMs ?? WATCH_MS);
169 const pollMs = heavy.pollMs ?? WATCH_POLL_MS;
170 for (let tick = 0; Date.now() < deadline; tick++) {
171 const failed = (await timed("/healthz")) ?? (await timed(pages[tick % pages.length]!));
172 if (failed) return failed;
173 await Bun.sleep(pollMs);
174 }
175 return null;
176 }
177
109178 /** Runs every check against `base`; returns the failures (empty is green). */
110179 export async function smoke(base: string, token = "", options: Options = {}): Promise<Failure[]> {
111180 const root = new URL(base);
@@ -181,5 +250,10 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr
181250 failures.push({ url, agent: "desktop", status: "error", contentType: "", why: String(error) });
182251 }
183252 }
253 if (options.heavy && failures.length === 0) {
254 const heavy = { version: options.version, ...(options.probe ? { probe: options.probe } : {}), ...options.heavy };
255 const slow = await watch(root, token, heavy, fetch, !!options.anonymousBoard);
256 if (slow) failures.push(slow);
257 }
184258 return failures;
185259 }
test/release-prove.test.ts +11 −1
@@ -29,6 +29,7 @@ function site(version: string, over: Record<string, () => Response> = {}) {
2929 return new Response('if (r.mode === "navigate") { e.respondWith(fetch(r)); return; }', {
3030 headers: { "content-type": "text/javascript" },
3131 });
32 if (path === "/explore" || path === "/search" || path === "/boards") return new Response("page");
3233 return new Response("no", { status: 404 });
3334 }) as unknown as typeof fetch;
3435 return { fake, hits };
@@ -41,7 +42,7 @@ describe("proving a release", () => {
4142 });
4243 afterEach(() => rmSync(dir, { recursive: true, force: true }));
4344 const run = (version: string, fake: typeof fetch, extra = {}) =>
44 proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, ...extra });
45 proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, heavy: { watchMs: 30, pollMs: 5 }, ...extra });
4546
4647 test("a pass is recorded with the version and raises nothing", async () => {
4748 const { fake } = site("1.2.3");
@@ -51,6 +52,15 @@ describe("proving a release", () => {
5152 expect(proofNeed(dir)).toBeUndefined();
5253 });
5354
55 test("the go-live proof loads the heavy pages and watches /healthz without being asked", async () => {
56 const { fake: base, hits } = site("1.2.5");
57 const fake = (async (input: string | URL, init?: RequestInit) =>
58 new URL(String(input)).pathname === "/healthz" && hits.includes("/explore") ? new Response("down", { status: 503 }) : base(input, init)) as unknown as typeof fetch;
59 const p = await proveRelease({ dataDir: dir, version: "1.2.5", url: "https://forge.test", fetch: fake, sleep: async () => {} });
60 expect(p?.ok).toBe(false);
61 expect(p?.failure).toContain("/healthz");
62 });
63
5464 test("an app host that answers as the forge fails the proof (#6726)", async () => {
5565 const { fake: base } = site("1.2.6");
5666 const fake = (async (input: string | URL, init?: RequestInit) =>
test/release-smoke.test.ts added +73
@@ -0,0 +1,73 @@
1 import { afterEach, expect, test } from "bun:test";
2 import { HEAVY_PAGES, smoke } from "../src/release/smoke.ts";
3
4 let server: ReturnType<typeof Bun.serve> | undefined;
5 afterEach(() => server?.stop(true));
6
7 /** A healthy forge; `over` answers some paths its own way, `/boards` waits `boardsMs`. */
8 function forge(boardsMs: number, seen: string[] = [], over: (path: string) => Response | undefined = () => undefined) {
9 server = Bun.serve({
10 port: 0,
11 hostname: "127.0.0.1",
12 async fetch(req) {
13 const path = new URL(req.url).pathname;
14 seen.push(path);
15 const made = over(path);
16 if (made) return made;
17 if (path === "/") return new Response('<link rel="stylesheet" href="/a.css">', { headers: { "content-type": "text/html" } });
18 if (path === "/a.css") return new Response("body{margin:0}", { headers: { "content-type": "text/css" } });
19 if (path === "/sw.js") return new Response("if (r.mode === 'navigate') { e.respondWith(fetch(r)); return; }", { headers: { "content-type": "text/javascript" } });
20 if (path === "/boards") await Bun.sleep(boardsMs);
21 return new Response("page " + path, { headers: { "x-forge-version": "v1" } });
22 },
23 });
24 return `http://127.0.0.1:${server.port}`;
25 }
26
27 const heavy = { watchMs: 300, pollMs: 50, slowMs: 200 };
28
29 test("a healthy forge passes the heavy pages and the /healthz watch", async () => {
30 const seen: string[] = [];
31 const failures = await smoke(forge(0, seen), "t", { heavy });
32 expect(failures.filter((f) => HEAVY_PAGES.some((page) => f.url.endsWith(page)) || f.url.endsWith("/healthz"))).toEqual([]);
33 for (const page of HEAVY_PAGES) expect(seen).toContain(new URL(page, "http://x").pathname);
34 expect(seen.filter((p) => p === "/healthz").length).toBeGreaterThan(4);
35 });
36
37 test("a /boards that blocks fails the smoke and names /boards", async () => {
38 const failures = await smoke(forge(500), "t", { heavy });
39 const slow = failures.find((f) => f.url.endsWith("/boards") && f.why.includes("/boards"));
40 expect(slow).toBeDefined();
41 });
42
43 test("a page that answers 500 fails, named", async () => {
44 const failures = await smoke(forge(0, [], (p) => (p === "/explore" ? new Response("x", { status: 500 }) : undefined)), "t", { heavy });
45 expect(failures.some((f) => f.url.endsWith("/explore") && f.status === 500)).toBe(true);
46 });
47
48 test("without heavy, the pages are not loaded", async () => {
49 const seen: string[] = [];
50 expect(await smoke(forge(0, seen), "t")).toEqual([]);
51 expect(seen).not.toContain("/explore");
52 });
53
54 test("a /healthz that reports another version mid-watch fails", async () => {
55 let n = 0;
56 const base = forge(0, [], (p) => (p === "/healthz" && ++n > 3 ? new Response("x", { headers: { "x-forge-version": "old" } }) : undefined));
57 const failures = await smoke(base, "t", { version: "v1", heavy });
58 expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("old"))).toBe(true);
59 });
60
61 test("a forge without the default repository is not failed for it, but a named one is", async () => {
62 const gone = (p: string) => (p.startsWith("/trevato/") ? new Response("no", { status: 404 }) : undefined);
63 expect(await smoke(forge(0, [], gone), "t", { heavy })).toEqual([]);
64 const named = await smoke(forge(0, [], gone), "t", { heavy: { ...heavy, repo: "/trevato/other" } });
65 expect(named.some((f) => f.url.endsWith("/trevato/other"))).toBe(true);
66 });
67
68 test("the watch is skipped when the checks already failed", async () => {
69 const seen: string[] = [];
70 server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: (req) => (seen.push(new URL(req.url).pathname), new Response("x", { status: 500 })) });
71 await smoke(`http://127.0.0.1:${server.port}`, "t", { heavy: { ...heavy, watchMs: 5_000 } });
72 expect(seen).not.toContain("/explore");
73 });
test/smoke.test.ts +2 −1
@@ -34,6 +34,7 @@ function stub(cssType: string, seen: string[] = []) {
3434 ? new Response("board")
3535 : new Response(null, { status: anonymousBoard.status, headers: { location: anonymousBoard.location } });
3636 if (path === "/healthz") return new Response("ok", { headers: { "x-forge-version": version() } });
37 if (["/explore", "/search", "/boards"].includes(path) || path.startsWith("/trevato/")) return new Response("page");
3738 if (path === "/sw.js") return new Response(sw.body, { status: sw.status, headers: { "content-type": sw.type } });
3839 return new Response("no", { status: 404 });
3940 },
@@ -104,7 +105,7 @@ test("the CLI takes --version and --wait", async () => {
104105 const base = stub("text/css");
105106 const run = (...a: string[]) =>
106107 Bun.spawn(["bun", "scripts/smoke.ts", base, ...a], { env: { ...process.env, FORGE_TOKEN: "t" }, stdout: "pipe", stderr: "pipe" }).exited;
107 expect(await run("--version", "v1")).toBe(0);
108 expect(await run("--version", "v1", "--watch", "0.2")).toBe(0);
108109 expect(await run("--version", "zz", "--wait", "0")).toBe(1);
109110 });
110111

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs