Public: anyone can read this; the crew can change it.

Every GET page walked against a budget, naming its slowest phase #10559

Parked opened this
Closed without merging

scout/forge:scout/10535-every-get-page-walked-against-a-budget-n → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
5 files changed +245 −7
scripts/unreached-allow.json −1
@@ -15,7 +15,6 @@
1515 "src/flow/review_rounds.ts",
1616 "src/flow/shared_red.ts",
1717 "src/flow/stale_fold.ts",
18 "src/kernel/why_now.ts",
1918 "src/spawn/pod.ts",
2019 "src/tui/gallery.ts",
2120 "src/tui/ui/qr.ts",
src/perf/budgets.ts +8 −5
@@ -8,13 +8,16 @@ export interface Budget {
88 readonly p95Ms: number;
99 }
1010
11 /** Every page's server p95, warm: one number, the same for all (#10535). */
12 export const PAGE_P95_MS = 150;
13
1114 export const BUDGETS: readonly Budget[] = [
12 { pattern: "/", words: "home p95 under 800 ms warm", p95Ms: 800 },
13 { pattern: "/boards", words: "boards p95 under 800 ms warm", p95Ms: 800 },
14 { pattern: "/explore", words: "explore p95 under 800 ms warm", p95Ms: 800 },
15 { pattern: "/", words: `home p95 under ${PAGE_P95_MS} ms warm`, p95Ms: PAGE_P95_MS },
16 { pattern: "/boards", words: `boards p95 under ${PAGE_P95_MS} ms warm`, p95Ms: PAGE_P95_MS },
17 { pattern: "/explore", words: `explore p95 under ${PAGE_P95_MS} ms warm`, p95Ms: PAGE_P95_MS },
1518 { pattern: "/api/v1/health", words: "health p95 under 100 ms", p95Ms: 100 },
16 { pattern: "/:owner/:name", words: "repo page p95 under 800 ms warm", p95Ms: 800 },
17 { pattern: "/:owner/:name/*", words: "repo sub-page p95 under 1000 ms warm", p95Ms: 1000 },
19 { pattern: "/:owner/:name", words: `repo page p95 under ${PAGE_P95_MS} ms warm`, p95Ms: PAGE_P95_MS },
20 { pattern: "/:owner/:name/*", words: `repo sub-page p95 under ${PAGE_P95_MS} ms warm`, p95Ms: PAGE_P95_MS },
1821 ];
1922
2023 /** The budget a route pattern is held to, or undefined for a route with none. */
test/helpers/statements.ts +2 −1
@@ -76,9 +76,10 @@ export async function quiet(db: Database): Promise<void> {
7676 export async function statements(
7777 db: Database,
7878 work: () => Promise<void> | void,
79 runs: number = RUNS,
7980 ): Promise<number> {
8081 let fewest = Number.POSITIVE_INFINITY;
81 for (let run = 0; run < RUNS; run++) {
82 for (let run = 0; run < runs; run++) {
8283 await quiet(db);
8384 fewest = Math.min(fewest, await once(db, work));
8485 }
test/helpers/work.ts +49
@@ -9,6 +9,7 @@
99 */
1010 import type { Database } from "bun:sqlite";
1111 import { expect } from "bun:test";
12 import { gitRecord } from "../../src/perf/live.ts";
1213 import { statements } from "./statements.ts";
1314
1415 /** The fewest SQL statements one request for `url` costs (server and test share `db`). */
@@ -22,3 +23,51 @@ export async function requestStatements(db: Database, url: string, init?: Reques
2223 export function expectWork(measured: number, opts: { max: number }): void {
2324 expect(measured).toBeLessThanOrEqual(opts.max);
2425 }
26
27 /** The git processes `work` spawns, counted from the spawn record (every `runGit` notes one). */
28 export async function gitSpawns(work: () => Promise<void> | void): Promise<number> {
29 const total = () => gitRecord.patterns().reduce((n, k) => n + gitRecord.traffic(k, 0).length, 0);
30 const before = total();
31 await work();
32 return total() - before;
33 }
34
35 /** What one warm request cost: counted work for the pull, the clock only to print. */
36 export interface PageWork {
37 status: number;
38 statements: number;
39 git: number;
40 /** Server-Timing phases in ms, slowest first. */
41 phases: Array<{ name: string; ms: number }>;
42 ms: number;
43 }
44
45 /** Parses a Server-Timing header into phases, slowest first. */
46 export function parsePhases(header: string): Array<{ name: string; ms: number }> {
47 return header
48 .split(",")
49 .map((p) => p.trim().split(";"))
50 .map(([name, ...rest]) => ({ name: name!, ms: Number(rest.find((r) => r.startsWith("dur="))?.slice(4) ?? 0) }))
51 .filter((p) => p.name !== "" && p.name !== "lag" && Number.isFinite(p.ms))
52 .sort((a, b) => b.ms - a.ms);
53 }
54
55 /** Walks one URL warm: statements (fewest of several runs), git spawns, and the last run's Server-Timing. */
56 export async function pageWork(db: Database, url: string, init?: RequestInit): Promise<PageWork> {
57 let status = 0;
58 let header = "";
59 let ms = 0;
60 let git = Number.POSITIVE_INFINITY;
61 const count = await statements(db, async () => {
62 const g = await gitSpawns(async () => {
63 const t = performance.now();
64 const res = await fetch(url, init);
65 await res.arrayBuffer();
66 ms = performance.now() - t;
67 status = res.status;
68 header = res.headers.get("server-timing") ?? "";
69 });
70 git = Math.min(git, g);
71 }, 2);
72 return { status, statements: count, git, phases: parsePhases(header), ms };
73 }
test/pages-budget.test.ts added +186
@@ -0,0 +1,186 @@
1 /**
2 * Every GET page, walked warm against a budget (#10535). The routes come from
3 * `server.ts` and `settings_routes.ts` themselves, as in the Server-Timing
4 * sweep, so a page is walked the day it is added.
5 *
6 * What a pull judges is counted work (`helpers/work.ts`, #7824): SQL
7 * statements and git processes per page, which an N+1 raises in every run.
8 * The clock is printed per phase and never judged here; set FORGE_JUDGE_CLOCK=1
9 * on a quiet machine to fail any page over `PAGE_P95_MS` too. Either way a
10 * failure names the route and its slowest Server-Timing phase.
11 * A route that needs an id the fixture does not hold is listed by name.
12 */
13 import { afterAll, beforeAll, expect, test } from "bun:test";
14 import { mkdtempSync, readFileSync, rmSync } from "node:fs";
15 import { tmpdir } from "node:os";
16 import { join } from "node:path";
17 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
18 import { createApp, type App } from "../src/app.ts";
19 import { createIssue } from "../src/core/issues.ts";
20 import { pin } from "../src/core/pins.ts";
21 import { getRepo } from "../src/core/repos.ts";
22 import { createUser, type User } from "../src/core/users.ts";
23 import { buildApiRouter, buildRouter, serve, type Servers } from "../src/http/server.ts";
24 import { PAGE_P95_MS } from "../src/perf/budgets.ts";
25 import { quietLogs } from "./helpers/globals.ts";
26 import { promote } from "./helpers/members.ts";
27 import { proveAll } from "./helpers/verified.ts";
28 import { pageWork, type PageWork } from "./helpers/work.ts";
29
30 quietLogs();
31
32 /** Statements and git processes one warm page may cost unless listed below. */
33 const DEFAULT = { statements: 60, git: 8 };
34 /** Pages that cost more today, with the most they may: lower these, never raise them. */
35 const HEAVIER: Record<string, { statements?: number; git?: number }> = {
36 "/:owner/:name/issues/:number": { statements: 70 },
37 "/:owner/:name/next": { statements: 64 },
38 "/:owner/:name/pulls/:number": { statements: 106 },
39 "/admin/limits": { statements: 80 },
40 "/board": { statements: 64 },
41 "/settings/capacity": { statements: 65 },
42 };
43
44 /** Streams and live fragments: no page. */
45 const NOT_PAGES = new Set([
46 "/events", "/system/stream", "/settings/teams/stream", "/fleet/graph/live",
47 "/settings/traffic/scrapes/export", "/settings/backups/live", "/:owner/:name/stream",
48 ]);
49
50 /** Patterns with an id in them, and the path the fixture fills them with. */
51 const FIXTURE: Record<string, string> = {
52 "/:owner/:name/issues/:number": "/alice/forge/issues/1",
53 "/:owner/:name/pulls/:number": "/alice/forge/pulls/2",
54 "/:owner/:name/display": "/alice/forge/display",
55 };
56
57 function registeredGets(): Set<string> {
58 const source = ["server.ts", "settings_routes.ts"]
59 .map((f) => readFileSync(join(import.meta.dir, "../src/http", f), "utf8"))
60 .join("\n");
61 const found = new Set<string>();
62 for (const m of source.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g)) found.add(m[1]!);
63 return found;
64 }
65
66 /** Every GET pattern the routers hold, split into walked paths and patterns with no fixture. */
67 export function plan(): { walked: Array<[pattern: string, path: string]>; unfixtured: string[] } {
68 const held = new Set([...buildRouter().patterns(), ...buildApiRouter().patterns()]);
69 const walked: Array<[string, string]> = [];
70 const unfixtured: string[] = [];
71 for (const p of [...registeredGets()].filter((p) => held.has(p)).sort()) {
72 if (NOT_PAGES.has(p) || p.startsWith("/assets/") || p.endsWith("/live") || p.includes("/__forge/")) continue;
73 const path = FIXTURE[p] ?? p.replace(":owner", "alice").replace(":name", "forge");
74 if (path.includes(":") || path.includes("*")) unfixtured.push(p);
75 else walked.push([p, path]);
76 }
77 return { walked, unfixtured };
78 }
79
80 /** The findings for one page's measured work: each names the route and its slowest phase. */
81 export function judge(pattern: string, w: PageWork, judgeClock: boolean): string[] {
82 const max = { ...DEFAULT, ...HEAVIER[pattern] };
83 const slowest = w.phases[0] === undefined ? "no phases" : `slowest phase ${w.phases[0].name} ${w.phases[0].ms.toFixed(1)} ms`;
84 const out: string[] = [];
85 if (w.statements > max.statements) out.push(`${pattern}: ${w.statements} SQL statements, budget ${max.statements} (${slowest})`);
86 if (w.git > max.git) out.push(`${pattern}: ${w.git} git processes, budget ${max.git} (${slowest})`);
87 if (judgeClock && w.ms > PAGE_P95_MS) out.push(`${pattern}: ${w.ms.toFixed(0)} ms, budget ${PAGE_P95_MS} ms (${slowest})`);
88 return out;
89 }
90
91 let root = "";
92 let app: App;
93 let servers: Servers;
94 let cookie = "";
95
96 async function user(name: string, first: boolean): Promise<User> {
97 const u = await createUser(app.db, {
98 origin: "seed", trusted: true, username: name, email: `${name}@forge.local`,
99 password: "a-good-passphrase", adminIfFirst: first,
100 });
101 if (!u.ok) throw new Error(u.error);
102 promote(app.db, u.value);
103 return u.value;
104 }
105
106 beforeAll(async () => {
107 root = mkdtempSync(join(tmpdir(), "forge-pages-budget-"));
108 app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
109 const alice = await user("alice", true);
110 const bob = await user("bob", false);
111 await user("carol", false);
112 proveAll(app);
113 for (const [who, name, visibility] of [
114 [alice, "forge", "public"], [alice, "vault", "private"], [bob, "notes", "public"], [bob, "diary", "private"],
115 ] as const) {
116 const made = await createRepo(app, who, { name, visibility });
117 if (!made.ok) throw new Error(String(made.error));
118 }
119 const forge = getRepo(app.db, "alice", "forge")!;
120 const files = await commitToBranch(app, alice, forge, {
121 message: "display",
122 files: [
123 { path: "README.md", content: "# forge\n" },
124 { path: "display.json", content: JSON.stringify({ title: "Night", layout: "grid", sources: ["alice/forge"] }) },
125 ],
126 });
127 if (!files.ok) throw new Error(JSON.stringify(files.error));
128 pin(app.db, alice.id, forge.id);
129 pin(app.db, alice.id, getRepo(app.db, "bob", "notes")!.id);
130 const issue = createIssue(app.db, { repoId: forge.id, authorId: bob.id, title: "An issue", body: "body" });
131 if (!issue.ok) throw new Error(issue.error);
132 const pull = createIssue(app.db, { repoId: forge.id, authorId: bob.id, title: "A pull", body: "body", kind: "pull" });
133 if (!pull.ok) throw new Error(pull.error);
134 app.db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid) VALUES (?1, 'main', 'topic', '', 'h')", [pull.value.id]);
135 servers = await serve(app);
136 const signin = await fetch(`${servers.url}/signin`, {
137 method: "POST",
138 headers: { origin: servers.url },
139 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
140 redirect: "manual",
141 });
142 cookie = (signin.headers.get("set-cookie") ?? "").split(";")[0]!;
143 }, 60_000);
144 afterAll(() => {
145 servers.stop();
146 app.close();
147 rmSync(root, { recursive: true, force: true });
148 });
149
150 test("every GET page is within its counted work, warm, naming its slowest phase", async () => {
151 const { walked, unfixtured } = plan();
152 expect(walked.length).toBeGreaterThan(20);
153 const judgeClock = process.env.FORGE_JUDGE_CLOCK === "1";
154 const findings: string[] = [];
155 const table: string[] = [];
156 for (const [pattern, path] of walked) {
157 const init = { headers: { cookie }, redirect: "manual" as const };
158 await (await fetch(`${servers.url}${path}`, init)).arrayBuffer(); // cold: caches filled, not judged
159 const w = await pageWork(app.db, `${servers.url}${path}`, init);
160 if (w.status >= 500) findings.push(`${pattern}: answered ${w.status}`);
161 if (w.status >= 300) continue; // a redirect is not a page (see server-timing-sweep)
162 const top = w.phases[0];
163 table.push(`${pattern}\t${w.statements} sql\t${w.git} git\t${w.ms.toFixed(1)} ms\t${top ? `${top.name} ${top.ms.toFixed(1)}` : "-"}`);
164 findings.push(...judge(pattern, w, judgeClock));
165 }
166 console.log(`pages walked warm (route, statements, git, clock, slowest phase):\n${table.join("\n")}`);
167 console.log(`no fixture, not walked (${unfixtured.length}):\n${unfixtured.join("\n")}`);
168 expect(findings).toEqual([]);
169 }, 300_000);
170
171 test("a page over its counted work is named with its route and slowest phase", () => {
172 const w: PageWork = { status: 200, statements: 99, git: 20, ms: 400, phases: [{ name: "git", ms: 300 }, { name: "render", ms: 5 }] };
173 expect(judge("/x", w, false)).toEqual([
174 "/x: 99 SQL statements, budget 60 (slowest phase git 300.0 ms)",
175 "/x: 20 git processes, budget 8 (slowest phase git 300.0 ms)",
176 ]);
177 expect(judge("/x", w, true).at(-1)).toBe("/x: 400 ms, budget 150 ms (slowest phase git 300.0 ms)");
178 expect(judge("/x", { ...w, statements: 3, git: 0, ms: 5 }, true)).toEqual([]);
179 });
180
181 test("a route the fixture cannot fill is listed, not skipped silently", () => {
182 const { walked, unfixtured } = plan();
183 expect(unfixtured.length).toBeGreaterThan(0);
184 expect(unfixtured.every((p) => p.includes(":") || p.includes("*"))).toBe(true);
185 expect(walked.some(([p]) => p === "/:owner/:name/issues/:number")).toBe(true);
186 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs