trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

A page-load budget check: the forge's five commonest pages answer under 50 ms warm, and the check names the page that does not #11326

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/11320-a-page-load-budget-check-the-forge-s-fiv → main 5 commits fork 2 threads · 2 unresolved

  • Merged. scout/11320-a-page-load-budget-check-the-forge-s-fiv was merged into main as 7d15d3e.

Review

Sign in to review this change.
3 files changed +265 −0
test/helpers/page_budget.ts added +67
@@ -0,0 +1,67 @@
1 /**
2 * The page-load budget (#11320): the pages people open first, each with the
3 * median it answers in warm and the SQL statements one request of it may run.
4 * Plain data, one line per page; `test/page-budget.test.ts` walks the table
5 * in-process. It lives beside the test because only the test reads it.
6 *
7 * The statement count is the hard limit: it is the same on a loaded machine
8 * as on an idle one, so an N+1 fails it every run. The median is the promise
9 * people feel; a loaded check host can miss it, so the test measures it again
10 * before it judges.
11 */
12
13 export interface PageBudget {
14 /** What the page is, in words, for the failure line. */
15 readonly page: string;
16 /** The router pattern; `:owner` and `:name` are filled by the test. */
17 readonly pattern: string;
18 /** The median of 20 warm requests stays under this many ms. */
19 readonly medianMs: number;
20 /** One warm request runs at most this many SQL statements. */
21 readonly statements: number;
22 }
23
24 /** Statement limits are today's count, signed in, plus a little: lower them as pages get cheaper, raise one only on purpose. */
25 export const PAGE_BUDGETS: readonly PageBudget[] = [
26 { page: "home", pattern: "/", medianMs: 50, statements: 46 },
27 { page: "explore", pattern: "/explore", medianMs: 50, statements: 18 },
28 { page: "map", pattern: "/map", medianMs: 50, statements: 21 },
29 { page: "an app page", pattern: "/:owner/:name", medianMs: 50, statements: 49 },
30 { page: "changelog", pattern: "/changelog", medianMs: 50, statements: 20 },
31 ];
32
33 /** The median of `xs` (the mean of the middle two for an even count); NaN for none. */
34 export function median(xs: readonly number[]): number {
35 if (xs.length === 0) return Number.NaN;
36 const s = [...xs].sort((a, b) => a - b);
37 const mid = s.length >> 1;
38 return s.length % 2 === 1 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2;
39 }
40
41 /** What one page measured: the fewest statements of several warm requests, the ms of each timed one, its phases. */
42 export interface PageMeasure {
43 readonly status: number;
44 readonly statements: number;
45 readonly samples: readonly number[];
46 /** Server-Timing phases of the last timed request, slowest first. */
47 readonly phases: ReadonlyArray<{ name: string; ms: number }>;
48 }
49
50 function phaseWords(phases: PageMeasure["phases"]): string {
51 return phases.length === 0 ? "no Server-Timing phases" : phases.map((p) => `${p.name} ${p.ms.toFixed(1)} ms`).join(", ");
52 }
53
54 /** The statement-count finding for a page, or none: the hard, deterministic one. */
55 export function countFinding(b: PageBudget, m: PageMeasure): string | undefined {
56 if (m.status !== 200) return `${b.pattern} (${b.page}): answered ${m.status}, not 200`;
57 if (m.statements > b.statements)
58 return `${b.pattern} (${b.page}): ${m.statements} SQL statements, budget ${b.statements} (${phaseWords(m.phases)})`;
59 return undefined;
60 }
61
62 /** The clock finding for a page, or none: names its median and its Server-Timing phases. */
63 export function timeFinding(b: PageBudget, m: PageMeasure): string | undefined {
64 const mid = median(m.samples);
65 if (mid < b.medianMs) return undefined;
66 return `${b.pattern} (${b.page}): median ${mid.toFixed(1)} ms over ${m.samples.length} warm requests, budget ${b.medianMs} ms (${phaseWords(m.phases)})`;
67 }
test/job-slices.test.ts +1
@@ -270,6 +270,7 @@ const UNTIMED: Record<string, string> = {
270270 "snapshot-journal": "inside a running snapshot upload: writes its journal; nightly, I/O",
271271 "snapshot-report": "inside a running snapshot upload: logs its progress; nightly",
272272 "system-live": "reads /proc and the cgroup roots for the system page while someone watches it",
273 entitlements: "reads each paid subscription from Stripe: network, daily, and the fixture has no entitlements",
junipercrew

Unrelated to the page budget. Rebase onto main or land it as its own pull.

junipercrew

Non-blocking: unrelated to the page budget. Drop this line once #11354 fixes it on main.
273274 traffic: "a closure in createTrafficWriter over its in-memory minute ledger; follow-up",
274275 };
275276
test/page-budget.test.ts added +197
@@ -0,0 +1,197 @@
1 /**
2 * The page-load budget (#11320): the five pages people open first answer
3 * under their median warm, on a seeded forge booted in-process and walked
4 * signed in, and run no more SQL statements than `test/helpers/page_budget.ts`
5 * allows.
6 *
7 * The count is the hard assertion and is judged first: it does not move with
8 * the machine's load, and it is the fewest of five warm requests. The clock is 20 warm requests per page after one to
9 * warm; a page whose median misses is measured again (up to three rounds)
10 * before it is named, so a busy check host does not fail a fast page.
11 */
12 import { afterAll, beforeAll, expect, test } from "bun:test";
13 import { mkdtempSync, rmSync } from "node:fs";
14 import { tmpdir } from "node:os";
15 import { join } from "node:path";
16 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
17 import { createApp, type App } from "../src/app.ts";
18 import { createIssue } from "../src/core/issues.ts";
19 import { getRepo } from "../src/core/repos.ts";
20 import { createUser, type User } from "../src/core/users.ts";
21 import { serve, type Servers } from "../src/http/server.ts";
22 import {
23 PAGE_BUDGETS, countFinding, median, timeFinding, type PageBudget, type PageMeasure,
24 } from "./helpers/page_budget.ts";
25 import { quietLogs } from "./helpers/globals.ts";
26 import { promote } from "./helpers/members.ts";
27 import { proveAll } from "./helpers/verified.ts";
28 import { statements } from "./helpers/statements.ts";
29 import { parsePhases } from "./helpers/work.ts";
30
31 quietLogs();
32
33 const TIMED = 20;
34 const COUNTED = 5;
35 const ROUNDS = 3;
36
37 let root = "";
38 let app: App;
39 let servers: Servers;
40 let cookie = "";
41
42 async function user(name: string, first: boolean): Promise<User> {
43 const u = await createUser(app.db, {
44 origin: "seed", trusted: true, username: name, email: `${name}@forge.local`,
45 password: "a-good-passphrase", adminIfFirst: first,
46 });
47 if (!u.ok) throw new Error(u.error);
48 promote(app.db, u.value);
49 return u.value;
50 }
51
52 beforeAll(async () => {
53 root = mkdtempSync(join(tmpdir(), "forge-page-budget-"));
54 app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
55 const alice = await user("alice", true);
56 const bob = await user("bob", false);
57 proveAll(app);
58 for (const name of ["forge", "notes"]) {
59 const made = await createRepo(app, alice, { name, visibility: "public" });
60 if (!made.ok) throw new Error(String(made.error));
61 }
62 const notes = getRepo(app.db, "alice", "notes")!;
63 const files = await commitToBranch(app, alice, notes, {
64 message: "an app",
65 files: [
66 { path: "README.md", content: "# notes\n\nA small app.\n" },
67 { path: "forge.json", content: JSON.stringify({ run: "bun run main.ts" }) },
68 { path: "main.ts", content: "Bun.serve({ fetch: () => new Response('ok') });\n" },
69 ],
70 });
71 if (!files.ok) throw new Error(JSON.stringify(files.error));
72 for (let i = 1; i <= 12; i++) {
73 const made = createIssue(app.db, { repoId: notes.id, authorId: bob.id, title: `Issue ${i}`, body: "body" });
74 if (!made.ok) throw new Error(made.error);
75 }
76 servers = await serve(app);
77 const signin = await fetch(`${servers.url}/signin`, {
78 method: "POST",
79 headers: { origin: servers.url },
80 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
81 redirect: "manual",
82 });
83 cookie = (signin.headers.get("set-cookie") ?? "").split(";")[0]!;
84 if (!cookie) throw new Error("alice could not sign in");
85 }, 60_000);
86
87 afterAll(() => {
88 servers?.stop();
89 app?.close();
90 rmSync(root, { recursive: true, force: true });
91 });
92
93 /** Every page is walked as alice, signed in: the map asks for a viewer. */
94 const INIT = (): RequestInit => ({ headers: { cookie }, redirect: "manual" });
95
96 function url(b: PageBudget): string {
97 return `${servers.url}${b.pattern.replace(":owner", "alice").replace(":name", "notes")}`;
98 }
99
100 /** One request warms; then the fewest statements of `COUNTED` warm requests is the count, so a background tick's queries do not land on a page. */
101 async function count(b: PageBudget): Promise<{ status: number; statements: number; phases: PageMeasure["phases"] }> {
102 const first = await fetch(url(b), INIT());
103 await first.arrayBuffer();
104 let status = first.status;
105 let header = "";
106 let fewest = Number.POSITIVE_INFINITY;
107 for (let i = 0; i < COUNTED; i++) {
108 const n = await statements(app.db, async () => {
109 const res = await fetch(url(b), INIT());
110 await res.arrayBuffer();
111 status = res.status;
112 header = res.headers.get("server-timing") ?? "";
113 });
114 fewest = Math.min(fewest, n);
115 if (status !== 200) break;
116 }
117 return { status, statements: fewest, phases: parsePhases(header) };
118 }
119
120 /** One warm-up, then `TIMED` timed requests; the phases are the last request's. */
121 async function time(b: PageBudget): Promise<{ samples: number[]; phases: PageMeasure["phases"] }> {
122 await (await fetch(url(b), INIT())).arrayBuffer();
123 const samples: number[] = [];
124 let header = "";
125 for (let i = 0; i < TIMED; i++) {
126 const t = performance.now();
127 const res = await fetch(url(b), INIT());
128 await res.arrayBuffer();
129 samples.push(performance.now() - t);
130 header = res.headers.get("server-timing") ?? "";
131 }
132 return { samples, phases: parsePhases(header) };
133 }
134
135 test("the budget table names five pages, each once, with a median and a statement limit", () => {
136 expect(PAGE_BUDGETS.map((b) => b.pattern)).toEqual(["/", "/explore", "/map", "/:owner/:name", "/changelog"]);
137 for (const b of PAGE_BUDGETS) {
138 expect(b.medianMs).toBeLessThanOrEqual(50);
139 expect(Number.isInteger(b.statements) && b.statements > 0).toBe(true);
140 }
141 });
142
143 test("each first page runs no more SQL statements warm than its budget", async () => {
144 const findings: string[] = [];
145 const table: string[] = [];
146 for (const b of PAGE_BUDGETS) {
147 const c = await count(b);
148 table.push(`${b.pattern}\t${c.status}\t${c.statements} sql`);
149 const f = countFinding(b, { ...c, samples: [] });
150 if (f) findings.push(f);
151 }
152 console.log(`first pages, counted warm:\n${table.join("\n")}`);
153 expect(findings).toEqual([]);
154 }, 60_000);
155
156 test("each first page answers under its median warm, retried before it is named", async () => {
157 const findings: string[] = [];
158 const table: string[] = [];
159 for (const b of PAGE_BUDGETS) {
160 let found: string | undefined;
161 let mid = Number.NaN;
162 for (let round = 0; round < ROUNDS; round++) {
163 const t = await time(b);
164 mid = median(t.samples);
165 found = timeFinding(b, { status: 200, statements: 0, ...t });
166 if (!found) break;
167 }
168 table.push(`${b.pattern}\tmedian ${mid.toFixed(1)} ms`);
169 if (found) findings.push(found);
170 }
171 console.log(`first pages, timed warm:\n${table.join("\n")}`);
172 expect(findings).toEqual([]);
173 }, 120_000);
174
175 test("a miss names the page, its median and its Server-Timing phases", () => {
176 const b = PAGE_BUDGETS[1]!;
177 const m: PageMeasure = {
178 status: 200, statements: 99, samples: [70, 60, 80],
179 phases: [{ name: "read", ms: 41.25 }, { name: "render", ms: 3 }],
180 };
181 expect(timeFinding(b, m)).toBe(
182 "/explore (explore): median 70.0 ms over 3 warm requests, budget 50 ms (read 41.3 ms, render 3.0 ms)",
183 );
184 expect(countFinding(b, m)).toBe(
185 `/explore (explore): 99 SQL statements, budget ${b.statements} (read 41.3 ms, render 3.0 ms)`,
186 );
187 expect(countFinding(b, { ...m, status: 302 })).toBe("/explore (explore): answered 302, not 200");
188 expect(timeFinding(b, { ...m, samples: [1, 2, 90] })).toBeUndefined();
189 expect(countFinding(b, { ...m, statements: 1 })).toBeUndefined();
190 });
191
192 test("the median is the middle sample, or the mean of the middle two", () => {
193 expect(median([5, 1, 3])).toBe(3);
194 expect(median([4, 1, 3, 2])).toBe(2.5);
195 expect(Number.isNaN(median([]))).toBe(true);
196 });
197

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs