trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

Profile tree, blob, raw and media pages under a production-shaped copy (#9405 box 2) #9516

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/9503-profile-tree-blob-raw-and-media-pages-un → main 2 commits fork

  • Merged. torque/9503-profile-tree-blob-raw-and-media-pages-un was merged into main as 411dac7.

Review

Sign in to review this change.
5 files changed +285 −0
bench/code-phases.ts added +88
@@ -0,0 +1,88 @@
1 /**
2 * The per-phase profile of tree, blob, raw and media pages (#9503), measured
3 * under load: it holds `--rate` requests a second against a copy of the forge
4 * (as bench/load.ts does, open loop) and tallies each answer's Server-Timing
5 * by page kind.
6 *
7 * bun bench/code-phases.ts --url https://copy.example --repo owner/name --rate 500 --seconds 20
8 *
9 * Run it for each repository of the copy and attach the tables to the issue.
10 * A private copy needs the forge token: `--token` or FORGE_TOKEN, sent on every request.
11 * Redirects and refusals are not samples; they are counted and printed.
12 * Point it only at a copy you run. A kind with no link on the repository's
13 * pages (a repo without an image or PDF has no media link) shows n = 0.
14 */
15 import { KINDS, dropped, gatherLinks, kindOf, parseTiming, profile, table, type Observation } from "../src/perf/phase_profile.ts";
16
17 const arg = (name: string, fallback?: string): string => {
18 const at = process.argv.indexOf(`--${name}`);
19 const v = at >= 0 ? process.argv[at + 1] : fallback;
20 if (v === undefined) throw new Error(`--${name} is required`);
21 return v;
22 };
23 const base = arg("url").replace(/\/$/, "");
24 const repo = arg("repo");
25 const rate = Number(arg("rate", "100"));
26 const seconds = Number(arg("seconds", "10"));
27 const token = process.argv.includes("--token") ? arg("token") : process.env.FORGE_TOKEN;
28 const headers: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
29 const MAX_INFLIGHT = 2000;
30 if (!/^[\w.-]+\/[\w.-]+$/.test(repo) || !(rate > 0) || !(seconds > 0)) throw new Error("bad --repo, --rate or --seconds");
31
32 const page = async (path: string): Promise<string> => {
33 const r = await fetch(`${base}${path}`, { headers, redirect: "manual" });
34 if (!r.ok) throw new Error(`${path} answered ${r.status}${r.status === 401 || r.status === 403 ? " (a private copy needs --token or FORGE_TOKEN)" : ""}`);
35 return r.text();
36 };
37 const pages = new Set<string>();
38 for (const path of [`/${repo}`, `/${repo}/tree/main`]) for (const l of gatherLinks(await page(path), repo)) pages.add(l);
39 // Blob pages name their media (an image or PDF is drawn from /media/…): follow the first files.
40 for (const p of [...pages].filter((p) => p.includes("/blob/")).slice(0, 40)) {
41 try {
42 for (const l of gatherLinks(await page(p), repo)) if (l.includes("/media/")) pages.add(l);
43 } catch (e) {
44 console.error(`skipped ${p}: ${(e as Error).message}`);
45 }
46 }
47 const byKind = Object.fromEntries(KINDS.map((k) => [k, [...pages].filter((p) => kindOf(p) === k)]));
48 console.error(Object.entries(byKind).map(([k, v]) => `${k}: ${v.length} links`).join(", "));
49 const all = [...pages];
50 if (all.length === 0) throw new Error("no commit-pinned links found on the repository's pages");
51
52 const seen: Observation[] = [];
53 let failed = 0;
54 let skipped = 0;
55 const inflight = new Set<Promise<void>>();
56 const ask = async (path: string) => {
57 const began = performance.now();
58 try {
59 const r = await fetch(`${base}${path}`, { headers, redirect: "manual" });
60 await r.arrayBuffer();
61 seen.push({ kind: kindOf(path)!, wall: performance.now() - began, phases: parseTiming(r.headers.get("server-timing")), status: r.status });
62 } catch {
63 failed++;
64 }
65 };
66 const started = performance.now();
67 let sent = 0;
68 while (performance.now() - started < seconds * 1000) {
69 const due = Math.floor(((performance.now() - started) / 1000) * rate);
70 while (sent < due) {
71 // Capped, so a slow copy is not buried and this client is not the bottleneck: what it could not send is counted.
72 if (inflight.size >= MAX_INFLIGHT) {
73 skipped += due - sent;
74 sent = due;
75 break;
76 }
77 const p = ask(all[sent % all.length]!);
78 inflight.add(p);
79 void p.finally(() => inflight.delete(p));
80 sent++;
81 }
82 await Bun.sleep(2);
83 }
84 await Promise.all(inflight);
85 const drop = Object.entries(dropped(seen)).map(([k, n]) => `${k}: ${n}`).join(", ");
86 console.log(`asked ${sent} at ${rate}/s for ${seconds}s, answered ${seen.length}, connection failures ${failed}, not sent (client cap ${MAX_INFLIGHT}) ${skipped}`);
87 console.log(`not counted as samples (by status): ${drop === "" ? "none" : drop}\n`);
88 console.log(table(profile(seen)));
docs/features/load-test.md +12
@@ -20,3 +20,15 @@ No baseline is kept in `bench/results/` yet: the pull's preview holds no reposit
2020 Follow-ups: profile #9503, baseline and release wiring #9504, media #9505.
2121
2222 Not done here: the profile of tree/blob/raw/media under production's shape, the p95 targets at 500/s, and wiring the comparison into the release check. Those need a machine and data this run does not have.
23
24 ## The per-phase profile (#9503)
25
26 `bench/code-phases.ts` holds the same open-loop rate and tallies each answer's `Server-Timing` by page kind (tree, blob, raw, media), printing p50/p95 of wall time and of every named phase as a table. `git`, `render`, `highlight` and `session` are always listed; n = 0 means that page never names the phase (today `/raw` and `/media` name none, so only their wall time is profiled).
27
28 ```sh
29 bun bench/code-phases.ts --url https://copy.example --repo owner/name --rate 500 --seconds 20
30 ```
31
32 The numbers themselves are **not attached**: this run's sandbox reaches no copy holding production's repositories, and the load must not be aimed at production. Run the command per repository on such a copy and post the tables on #9503.
33
34 Only 200 (and 206) answers are samples. A redirect, 401 or 404 is counted by status in the line printed under `answered N`, so a run that was all refusals says so. A private copy needs `--token` (or `FORGE_TOKEN`); the first pages answering non-200 stops the run with the status. Connection failures and requests the client itself could not send (2000 in flight) are counted, not hidden.
src/perf/phase_profile.ts added +98
@@ -0,0 +1,98 @@
1 /**
2 * Per-phase profile of the code pages (#9503): which of tree, blob, raw and
3 * media a request is, what its Server-Timing header named, and the p50/p95 of
4 * each phase. Plain data in, plain data out; `bench/code-phases.ts` asks the
5 * pages, this reads the answers.
6 */
7
8 export type CodeKind = "tree" | "blob" | "raw" | "media";
9 export const KINDS: readonly CodeKind[] = ["tree", "blob", "raw", "media"];
10
11 /** The kind of a repository path (`/o/r/blob/<ref>/f`), undefined for any other page. */
12 export function kindOf(path: string): CodeKind | undefined {
13 const m = /^\/[^/]+\/[^/]+\/(tree|blob|raw|media)\//.exec(path);
14 return m === null ? undefined : (m[1] as CodeKind);
15 }
16
17 /** `git;dur=1.5, render;dur=3` as phase → ms; malformed entries are skipped. */
18 export function parseTiming(header: string | null): Map<string, number> {
19 const out = new Map<string, number>();
20 for (const m of (header ?? "").matchAll(/([A-Za-z][\w-]*);dur=(\d+(?:\.\d+)?)/g)) out.set(m[1]!, Number(m[2]));
21 return out;
22 }
23
24 /** Nearest-rank percentile of an ascending list; NaN when empty. */
25 export const percentile = (sorted: readonly number[], p: number): number =>
26 sorted.length === 0 ? NaN : sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))]!;
27
28 /** Only a page that was served counts: 200, or 206 for a ranged media read. A redirect or a refusal is not a sample. */
29 export const counted = (status: number): boolean => status === 200 || status === 206;
30
31 /** How many answers were not counted, by status (0 for a failed connection), e.g. `{ "404": 3 }`. */
32 export function dropped(observed: readonly { readonly status: number }[]): Record<string, number> {
33 const out: Record<string, number> = {};
34 for (const o of observed) if (!counted(o.status)) out[String(o.status)] = (out[String(o.status)] ?? 0) + 1;
35 return out;
36 }
37
38 /**
39 * The commit-pinned tree, blob, raw and media paths of `repo` named in `html`;
40 * each blob also yields its raw sibling. Query strings and fragments are cut.
41 */
42 export function gatherLinks(html: string, repo: string): string[] {
43 const esc = repo.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
44 const found = new Set<string>();
45 for (const m of html.matchAll(new RegExp(`/${esc}/(?:tree|blob|raw|media)/[0-9a-f]{40}[^"'<> ?#]*`, "g"))) {
46 found.add(m[0]);
47 if (m[0].includes("/blob/")) found.add(m[0].replace("/blob/", "/raw/"));
48 }
49 return [...found];
50 }
51
52 export interface Observation {
53 readonly kind: CodeKind;
54 /** Wall time as the client saw it, ms. */
55 readonly wall: number;
56 readonly phases: ReadonlyMap<string, number>;
57 readonly status: number;
58 }
59
60 export interface Row {
61 readonly kind: CodeKind;
62 readonly phase: string;
63 readonly n: number;
64 readonly p50: number;
65 readonly p95: number;
66 }
67
68 /**
69 * One row per kind and phase, `wall` (the client's clock) first, then the
70 * named phases. The four phases asked for — git, render, highlight, session —
71 * are always listed, with n = 0 where the page never named them, so a missing
72 * phase reads as "not measured" and not as "free".
73 */
74 export function profile(observed: readonly Observation[]): Row[] {
75 const rows: Row[] = [];
76 for (const kind of KINDS) {
77 const mine = observed.filter((o) => o.kind === kind && counted(o.status));
78 const names = new Set<string>(["git", "render", "highlight", "session"]);
79 for (const o of mine) for (const k of o.phases.keys()) names.add(k);
80 const row = (phase: string, xs: number[]): Row => {
81 xs.sort((a, b) => a - b);
82 return { kind, phase, n: xs.length, p50: percentile(xs, 50), p95: percentile(xs, 95) };
83 };
84 rows.push(row("wall", mine.map((o) => o.wall)));
85 for (const name of names) rows.push(row(name, mine.flatMap((o) => (o.phases.has(name) ? [o.phases.get(name)!] : []))));
86 }
87 return rows;
88 }
89
90 /** A Markdown table of `profile`, ms to one decimal; "–" where nothing was observed. */
91 export function table(rows: readonly Row[]): string {
92 const f = (x: number) => (Number.isNaN(x) ? "–" : x.toFixed(1));
93 return [
94 "| page | phase | n | p50 ms | p95 ms |",
95 "| ---- | ----- | -: | -----: | -----: |",
96 ...rows.map((r) => `| ${r.kind} | ${r.phase} | ${r.n} | ${f(r.p50)} | ${f(r.p95)} |`),
97 ].join("\n");
98 }
test/code-phases-bench.test.ts added +29
@@ -0,0 +1,29 @@
1 import { expect, test } from "bun:test";
2
3 const oid = "a".repeat(40);
4 const run = async (status: number) => {
5 const server = Bun.serve({
6 port: 0,
7 fetch: (req) => {
8 const u = new URL(req.url).pathname;
9 if (!u.includes(oid)) return new Response(`<a href="/o/r/blob/${oid}/f.txt">f</a>`, { headers: { "content-type": "text/html" } });
10 return new Response("x", { status, headers: { "server-timing": "git;dur=2.5, session;dur=0.5" } });
11 },
12 });
13 try {
14 const p = Bun.spawn(["bun", "bench/code-phases.ts", "--url", `http://127.0.0.1:${server.port}`, "--repo", "o/r", "--rate", "50", "--seconds", "1"], { stdout: "pipe", stderr: "pipe" });
15 return { code: await p.exited, out: await new Response(p.stdout).text() };
16 } finally {
17 server.stop(true);
18 }
19 };
20
21 test("served pages are profiled; refusals are counted by status, not tallied", async () => {
22 const ok = await run(200);
23 expect(ok.code).toBe(0);
24 expect(ok.out).toContain("not counted as samples (by status): none");
25 expect(ok.out).toMatch(/\| blob \| git \| \d+ \| 2\.5 \| 2\.5 \|/);
26 const no = await run(404);
27 expect(no.out).toMatch(/not counted as samples \(by status\): 404: \d+/);
28 expect(no.out).toMatch(/\| blob \| wall \| 0 \| – \| – \|/);
29 });
test/phase-profile.test.ts added +58
@@ -0,0 +1,58 @@
1 import { expect, test } from "bun:test";
2 import { kindOf, parseTiming, percentile, profile, table } from "../src/perf/phase_profile.ts";
3
4 test("kindOf names the four code pages and nothing else", () => {
5 expect(kindOf("/o/r/tree/abc/")).toBe("tree");
6 expect(kindOf("/o/r/blob/abc/a.ts")).toBe("blob");
7 expect(kindOf("/o/r/raw/abc/a.ts")).toBe("raw");
8 expect(kindOf("/o/r/media/abc/a.png")).toBe("media");
9 expect(kindOf("/o/r/issues")).toBeUndefined();
10 });
11
12 test("parseTiming reads phases and skips junk", () => {
13 const m = parseTiming("session;dur=0.5, git;dur=12.25, junk, render;dur=3");
14 expect(m.get("session")).toBe(0.5);
15 expect(m.get("git")).toBe(12.25);
16 expect(m.size).toBe(3);
17 expect(parseTiming(null).size).toBe(0);
18 });
19
20 test("percentile is nearest-rank: p50 of 1..10 is 5, p95 is 10", () => {
21 const xs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
22 expect(percentile(xs, 50)).toBe(5);
23 expect(percentile(xs, 95)).toBe(10);
24 expect(percentile([], 50)).toBeNaN();
25 });
26
27 test("profile lists git, render, highlight and session even when unnamed, and drops errors", () => {
28 const o = (wall: number, status = 200, phases: [string, number][] = []) => ({ kind: "blob" as const, wall, status, phases: new Map(phases) });
29 const rows = profile([o(10, 200, [["git", 4]]), o(20, 200, [["git", 8]]), o(999, 500)]);
30 const blob = rows.filter((r) => r.kind === "blob");
31 expect(blob.find((r) => r.phase === "wall")).toMatchObject({ n: 2, p50: 10, p95: 20 });
32 expect(blob.find((r) => r.phase === "git")).toMatchObject({ n: 2, p50: 4, p95: 8 });
33 expect(blob.find((r) => r.phase === "highlight")?.n).toBe(0);
34 expect(table(rows)).toContain("| blob | highlight | 0 | – | – |");
35 expect(rows.filter((r) => r.kind === "media").find((r) => r.phase === "wall")?.n).toBe(0);
36 });
37
38 import { counted, dropped, gatherLinks } from "../src/perf/phase_profile.ts";
39
40 test("only 200 and 206 are samples; redirects and refusals are counted apart", () => {
41 const rows = profile([
42 { kind: "tree", wall: 1, status: 302, phases: new Map() },
43 { kind: "tree", wall: 30, status: 200, phases: new Map() },
44 ]);
45 expect(rows.find((r) => r.kind === "tree" && r.phase === "wall")).toMatchObject({ n: 1, p50: 30 });
46 expect(counted(206)).toBe(true);
47 expect(counted(301)).toBe(false);
48 expect(dropped([{ status: 401 }, { status: 401 }, { status: 0 }, { status: 200 }])).toEqual({ "401": 2, "0": 1 });
49 });
50
51 test("gatherLinks takes only this repository's pinned links, adds raw for a blob, cuts the query", () => {
52 const oid = "a".repeat(40);
53 const html = `<a href="/o/r.x/blob/${oid}/a.ts?x=1">a</a> <a href="/o/r.x/tree/${oid}/src">t</a>
54 <a href="/o/r.x/media/${oid}/i.png">i</a> <a href="/o/rZx/blob/${oid}/no">n</a> <a href="/o/r.x/blob/main/f">m</a>`;
55 expect(gatherLinks(html, "o/r.x").sort()).toEqual(
56 [`/o/r.x/blob/${oid}/a.ts`, `/o/r.x/raw/${oid}/a.ts`, `/o/r.x/tree/${oid}/src`, `/o/r.x/media/${oid}/i.png`].sort(),
57 );
58 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs