Public: anyone can read this; the crew can change it.
The forge holds 500 anonymous code pages a second, with people still fast #9461
torque/forge:torque/9405-the-forge-holds-500-anonymous-code-pages → main 3 commits fork
7 files
changed +289
−13
bench/load.ts added +82
| @@ -0,0 +1,82 @@ | ||
| 1 | /** | |
| 2 | * The load test (#9405): an open-loop reader of code pages against a copy of | |
| 3 | * the forge, run from any machine that can reach it. | |
| 4 | * | |
| 5 | * bun bench/load.ts --url https://copy.example --repo owner/name \ | |
| 6 | * --rate 500 --seconds 20 --version 0.7.386 --out bench/results/0.7.386.json | |
| 7 | * bun bench/load.ts ... --compare bench/results/0.7.385.json # exits 1 on a >20% fall | |
| 8 | * | |
| 9 | * It reads the repository's front page, collects its commit-pinned tree and | |
| 10 | * blob links, then asks for them at `--rate` a second regardless of how fast | |
| 11 | * the forge answers (open loop: a slow forge queues, as strangers would). | |
| 12 | * Each request is a tree, blob or raw page of a pinned commit, anonymous. | |
| 13 | * Point it only at a copy you run; never at a forge you do not own. | |
| 14 | */ | |
| 15 | ||
| 16 | import { readFileSync, writeFileSync } from "node:fs"; | |
| 17 | import { fell, summarize, type LoadResult } from "../src/perf/load_verdict.ts"; | |
| 18 | ||
| 19 | const arg = (name: string, fallback?: string): string => { | |
| 20 | const at = process.argv.indexOf(`--${name}`); | |
| 21 | const v = at >= 0 ? process.argv[at + 1] : fallback; | |
| 22 | if (v === undefined) throw new Error(`--${name} is required`); | |
| 23 | return v; | |
| 24 | }; | |
| 25 | ||
| 26 | const base = arg("url").replace(/\/$/, ""); | |
| 27 | const repo = arg("repo"); | |
| 28 | const rate = Number(arg("rate", "100")); | |
| 29 | const seconds = Number(arg("seconds", "10")); | |
| 30 | const version = arg("version", "unversioned"); | |
| 31 | if (!/^[\w.-]+\/[\w.-]+$/.test(repo) || !(rate > 0) || !(seconds > 0)) throw new Error("bad --repo, --rate or --seconds"); | |
| 32 | ||
| 33 | const front = await (await fetch(`${base}/${repo}`)).text(); | |
| 34 | const home = await (await fetch(`${base}/${repo}/tree/main`)).text(); | |
| 35 | const pinned = new Set<string>(); | |
| 36 | for (const m of (front + home).matchAll(new RegExp(`/${repo}/(?:tree|blob)/[0-9a-f]{40}[^"'<> ]*`, "g"))) { | |
| 37 | pinned.add(m[0]); | |
| 38 | pinned.add(m[0].replace("/tree/", "/raw/").replace("/blob/", "/raw/")); | |
| 39 | } | |
| 40 | const paths = [...pinned].filter((p) => !p.includes("/raw/") || p.split("/").length > 5); | |
| 41 | if (paths.length === 0) throw new Error("no commit-pinned links found on the repository's pages"); | |
| 42 | ||
| 43 | const latencies: number[] = []; | |
| 44 | let refused = 0; | |
| 45 | const inflight = new Set<Promise<void>>(); | |
| 46 | const ask = (path: string): Promise<void> => { | |
| 47 | const began = performance.now(); | |
| 48 | return fetch(`${base}${path}`, { redirect: "manual" }) | |
| 49 | .then(async (r) => { | |
| 50 | await r.arrayBuffer(); | |
| 51 | if (r.status < 400) latencies.push(performance.now() - began); | |
| 52 | else refused++; | |
| 53 | }) | |
| 54 | .catch(() => { | |
| 55 | refused++; | |
| 56 | }); | |
| 57 | }; | |
| 58 | ||
| 59 | const started = performance.now(); | |
| 60 | let sent = 0; | |
| 61 | while (performance.now() - started < seconds * 1000) { | |
| 62 | const due = Math.floor(((performance.now() - started) / 1000) * rate); | |
| 63 | while (sent < due) { | |
| 64 | const p = ask(paths[Math.floor(Math.random() * paths.length)]!); | |
| 65 | inflight.add(p); | |
| 66 | void p.finally(() => inflight.delete(p)); | |
| 67 | sent++; | |
| 68 | } | |
| 69 | await Bun.sleep(2); | |
| 70 | } | |
| 71 | await Promise.all(inflight); | |
| 72 | ||
| 73 | const result: LoadResult = summarize(version, rate, seconds, latencies, refused); | |
| 74 | console.log(JSON.stringify(result, null, 2)); | |
| 75 | const out = process.argv.includes("--out") ? arg("out") : undefined; | |
| 76 | if (out !== undefined) writeFileSync(out, JSON.stringify(result, null, 2) + "\n"); | |
| 77 | const last = process.argv.includes("--compare") ? (JSON.parse(readFileSync(arg("compare"), "utf8")) as LoadResult) : undefined; | |
| 78 | const why = last === undefined ? undefined : fell(last, result); | |
| 79 | if (why !== undefined) { | |
| 80 | console.error(`LOAD REGRESSION: ${why}`); | |
| 81 | process.exit(1); | |
| 82 | } |
docs/features/load-test.md added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | # Load test (#9405) | |
| 2 | ||
| 3 | `bench/load.ts` reads a copy of the forge the way strangers do: anonymous tree, blob and raw pages of commit-pinned links, asked at a fixed rate whether or not the forge keeps up. It prints rate served, refused, p50 and p95 as JSON. | |
| 4 | ||
| 5 | sh | |
| 6 | bun bench/load.ts --url https://copy.example --repo owner/name --rate 500 --seconds 20 \ | |
| 7 | --version 0.7.386 --out bench/results/0.7.386.json --compare bench/results/0.7.385.json | |
| 8 | ||
| 9 | ||
| 10 | Run it from a machine that is not the one serving. `--compare` exits 1 when the served rate is more than 20% under the last kept result (`src/perf/load_verdict.ts`), so run it in the release step and keep each release's file in `bench/results/`. | |
| 11 | ||
| 12 | What is kept after the first read (`web/pages/tree_cache.ts`): a stranger's tree and blob page of a public repository, and the bytes of a small `/raw` file, at a full commit id. A second read of any of them spawns no git (`test/anonymous-deep.test.ts`). | |
| 13 | ||
| 14 | Not done here: the profile of tree/blob/raw/media under production's shape, the p95 targets at 500/s, and wiring the comparison into the release check. Those need a machine and data this run does not have. |
src/perf/load_verdict.ts added +60
| @@ -0,0 +1,60 @@ | ||
| 1 | /** | |
| 2 | * What a load run reached, and whether a release fell from the last one (#9405). | |
| 3 | * Plain numbers in, plain numbers out: `bench/load.ts` measures, this judges. | |
| 4 | * | |
| 5 | * Served rate = answered 2xx/3xx responses ÷ run seconds (requests/s). | |
| 6 | * Percentile = nearest-rank: the value at index ceil(p/100·n) − 1 of the sorted | |
| 7 | * latencies (ms), the usual definition for a latency report. | |
| 8 | */ | |
| 9 | ||
| 10 | export interface LoadResult { | |
| 11 | readonly version: string; | |
| 12 | /** The rate asked for, requests per second. */ | |
| 13 | readonly asked: number; | |
| 14 | readonly seconds: number; | |
| 15 | /** Responses that were 2xx or 3xx. */ | |
| 16 | readonly served: number; | |
| 17 | /** Responses that were 429, 5xx or never came back. */ | |
| 18 | readonly refused: number; | |
| 19 | /** Served per second over the run. */ | |
| 20 | readonly servedPerSecond: number; | |
| 21 | readonly p50: number; | |
| 22 | readonly p95: number; | |
| 23 | } | |
| 24 | ||
| 25 | /** The share of the last release's served rate a release may lose (20%). */ | |
| 26 | export const ALLOWED_FALL = 0.2; | |
| 27 | ||
| 28 | export function percentile(sortedMs: readonly number[], p: number): number { | |
| 29 | if (sortedMs.length === 0) return 0; | |
| 30 | const rank = Math.ceil((p / 100) * sortedMs.length); | |
| 31 | return sortedMs[Math.min(sortedMs.length, Math.max(1, rank)) - 1]!; | |
| 32 | } | |
| 33 | ||
| 34 | export function summarize( | |
| 35 | version: string, | |
| 36 | asked: number, | |
| 37 | seconds: number, | |
| 38 | latenciesMs: readonly number[], | |
| 39 | refused: number, | |
| 40 | ): LoadResult { | |
| 41 | const sorted = [...latenciesMs].sort((a, b) => a - b); | |
| 42 | const round = (n: number) => Math.round(n * 10) / 10; | |
| 43 | return { | |
| 44 | version, | |
| 45 | asked, | |
| 46 | seconds, | |
| 47 | served: sorted.length, | |
| 48 | refused, | |
| 49 | servedPerSecond: round(sorted.length / seconds), | |
| 50 | p50: round(percentile(sorted, 50)), | |
| 51 | p95: round(percentile(sorted, 95)), | |
| 52 | }; | |
| 53 | } | |
| 54 | ||
| 55 | /** Words for a failed comparison, or undefined when `now` holds within `ALLOWED_FALL` of `last`. */ | |
| 56 | export function fell(last: LoadResult, now: LoadResult): string | undefined { | |
| 57 | const floor = last.servedPerSecond * (1 - ALLOWED_FALL); | |
| 58 | if (now.servedPerSecond >= floor) return undefined; | |
| 59 | return `served ${now.servedPerSecond}/s at ${now.version}, under ${Math.round(floor * 10) / 10}/s (80% of ${last.servedPerSecond}/s at ${last.version})`; | |
| 60 | } |
src/web/pages/repo.ts +52 −12
| @@ -11,7 +11,7 @@ | ||
| 11 | 11 | import { recordLook } from "../../flow/valve_store.ts"; |
| 12 | 12 | import { PINNED_PUBLIC, immutableWhenAddressed, mayShare } from "../../http/addressed.ts"; |
| 13 | 13 | import { EDGE_VARY } from "../../http/edge_cache.ts"; |
| 14 | import { cachedTree, keepTree, resolveCommit } from "./tree_cache.ts"; | |
| 14 | import { cachedBlobPage, cachedRaw, cachedTree, keepBlobPage, keepRaw, keepTree, resolveCommit } from "./tree_cache.ts"; | |
| 15 | 15 | import { behindLine, followingLine, originAheadLine } from "./upstream-follow.ts"; |
| 16 | 16 | import { CONFLICT_WORDS, UNREAD_WORDS } from "../../origins/ahead.ts"; |
| 17 | 17 | import { upstreamOffer } from "../../actions/offer_upstream.ts"; |
| @@ -51,7 +51,7 @@ import { | ||
| 51 | 51 | visibleForks, |
| 52 | 52 | } from "../../core/repos.ts"; |
| 53 | 53 | import { isSafeRef, isSafeTreePath } from "../../git/guard.ts"; |
| 54 | import { blobForRender, type TreeEntry } from "../../git/read.ts"; | |
| 54 | import { blobForRender, type BlobForRender, type TreeEntry } from "../../git/read.ts"; | |
| 55 | 55 | import { repoDir } from "../../git/repos.ts"; |
| 56 | 56 | import { estimateTokens } from "../../kernel/tokens-format.ts"; |
| 57 | 57 | import { safeBack, type Ctx } from "../../http/context.ts"; |
| @@ -1593,6 +1593,32 @@ export async function blobPage(ctx: Ctx): Promise<Response> { | ||
| 1593 | 1593 | const path = ctx.params.path ?? ""; |
| 1594 | 1594 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 1595 | 1595 | const git = gitClock(ctx.request); |
| 1596 | // A stranger's page at a full commit id is kept under that id (#9405): the | |
| 1597 | // second read of it walks no git at all. | |
| 1598 | const kept = | |
| 1599 | ctx.auth.user === undefined && repo.visibility === "public" && FULL_OID.test(ref) && ctx.url.searchParams.get("fragment") !== "1"; | |
| 1600 | const keptView = blobView(ctx.url.searchParams.get("view")); | |
| 1601 | const keptAll = ctx.url.searchParams.get("all") === "1"; | |
| 1602 | if (kept) { | |
| 1603 | const hit = cachedBlobPage(repo.id, ref, path, String(keptView), keptAll); | |
| 1604 | if (hit !== undefined) { | |
| 1605 | git.done(); | |
| 1606 | return immutableWhenAddressed( | |
| 1607 | ctx.request, | |
| 1608 | true, | |
| 1609 | repo, | |
| 1610 | ref, | |
| 1611 | codePage(ctx, scope, { | |
| 1612 | title: hit.title, | |
| 1613 | ref, | |
| 1614 | explorer: hit.explorer, | |
| 1615 | content: hit.content, | |
| 1616 | objectSrc: hit.objectSrc, | |
| 1617 | ...(hit.csv ? { extraJs: csvJs } : {}), | |
| 1618 | }), | |
| 1619 | ); | |
| 1620 | } | |
| 1621 | } | |
| 1596 | 1622 | const blob = await git.read(() => scope.source.blob(ref, path)); |
| 1597 | 1623 | if (!blob.ok) return notFoundPage(ctx); |
| 1598 | 1624 | const refs = await git.read(() => refSetFrom(scope.source)); |
| @@ -1626,19 +1652,29 @@ export async function blobPage(ctx: Ctx): Promise<Response> { | ||
| 1626 | 1652 | git.done(); |
| 1627 | 1653 | const filename = path.split("/").pop() ?? path; |
| 1628 | 1654 | const pageStart = performance.now(); |
| 1655 | const explorerHtml = explorerColumn(scope, explorer, { | |
| 1656 | finder: true, | |
| 1657 | actions: newFileAction( | |
| 1658 | scope, | |
| 1659 | ref, | |
| 1660 | path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "", | |
| 1661 | ), | |
| 1662 | }); | |
| 1663 | const objectSrc = drawsPdf(scope, path, shown, view); | |
| 1664 | if (kept) | |
| 1665 | keepBlobPage(repo.id, ref, path, String(keptView), keptAll, { | |
| 1666 | title: `${filename} · ${repo.name}`, | |
| 1667 | explorer: explorerHtml, | |
| 1668 | content: pane, | |
| 1669 | objectSrc, | |
| 1670 | csv: isCsvPath(path), | |
| 1671 | }); | |
| 1629 | 1672 | const response = await codePage(ctx, scope, { |
| 1630 | 1673 | title: `${filename} · ${repo.name}`, |
| 1631 | 1674 | ref, |
| 1632 | explorer: explorerColumn(scope, explorer, { | |
| 1633 | finder: true, | |
| 1634 | actions: newFileAction( | |
| 1635 | scope, | |
| 1636 | ref, | |
| 1637 | path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "", | |
| 1638 | ), | |
| 1639 | }), | |
| 1675 | explorer: explorerHtml, | |
| 1640 | 1676 | content: pane, |
| 1641 | objectSrc: drawsPdf(scope, path, shown, view), | |
| 1677 | objectSrc, | |
| 1642 | 1678 | ...(isCsvPath(path) ? { extraJs: csvJs } : {}), |
| 1643 | 1679 | }); |
| 1644 | 1680 | notePhase(ctx.request, "render", paneMs + performance.now() - pageStart); |
| @@ -1691,9 +1727,13 @@ export async function rawBlob(ctx: Ctx): Promise<Response> { | ||
| 1691 | 1727 | if (!isSafeRef(ref) || !isSafeTreePath(path)) return notFoundPage(ctx); |
| 1692 | 1728 | |
| 1693 | 1729 | const dir = repoDir(ctx.app.reposRoot, repo.ownerName, repo.name); |
| 1694 | const blob = await blobForRender(dir, ref, path, 25_000_000); | |
| 1730 | // At a full commit id the bytes never change: a small file is kept (#9405). | |
| 1731 | const pinned = FULL_OID.test(ref); | |
| 1732 | const held = pinned ? cachedRaw<BlobForRender>(repo.id, ref, path) : undefined; | |
| 1733 | const blob = held ?? (await blobForRender(dir, ref, path, 25_000_000)); | |
| 1695 | 1734 | if (blob.kind === "missing" || blob.kind === "too_large") |
| 1696 | 1735 | return notFoundPage(ctx); |
| 1736 | if (pinned && held === undefined) keepRaw(repo.id, ref, path, blob); | |
| 1697 | 1737 | return rawResponse(blob, { |
| 1698 | 1738 | oid: blob.oid, |
| 1699 | 1739 | immutable: FULL_OID.test(ref), |
src/web/pages/tree_cache.ts +37 −1
| @@ -45,4 +45,40 @@ export function keepTree(repoId: string, commit: string, ref: string, path: stri | ||
| 45 | 45 | export const treeCacheStats = () => cache.stats(); |
| 46 | 46 | |
| 47 | 47 | /** Test seam: start from nothing. */ |
| 48 | export const clearTreeCache = (): void => cache.clear(); | |
| 48 | export const clearTreeCache = (): void => { | |
| 49 | cache.clear(); | |
| 50 | rawCache.clear(); | |
| 51 | }; | |
| 52 | ||
| 53 | /** | |
| 54 | * A file's page at one full commit id (#9405). Same reasoning as the listing: | |
| 55 | * a commit names its bytes forever, so an anonymous reader's markup depends on | |
| 56 | * nothing but the commit, the path and the view; after the first read the page | |
| 57 | * costs no git read at all. | |
| 58 | */ | |
| 59 | export interface RenderedBlob extends RenderedTree { | |
| 60 | readonly title: string; | |
| 61 | readonly objectSrc: boolean; | |
| 62 | readonly csv: boolean; | |
| 63 | } | |
| 64 | ||
| 65 | const blobKey = (repoId: string, commit: string, path: string, view: string, all: boolean): string => | |
| 66 | ["blob", repoId, commit, path, view, all ? "all" : "window"].join("\0"); | |
| 67 | ||
| 68 | export const cachedBlobPage = (repoId: string, commit: string, path: string, view: string, all: boolean) => | |
| 69 | cache.get<RenderedBlob>(blobKey(repoId, commit, path, view, all)); | |
| 70 | ||
| 71 | export function keepBlobPage(repoId: string, commit: string, path: string, view: string, all: boolean, page: RenderedBlob): void { | |
| 72 | cache.set(blobKey(repoId, commit, path, view, all), page, TTL_MS, page.explorer.text.length + page.content.text.length); | |
| 73 | } | |
| 74 | ||
| 75 | /** A file's bytes at a full commit id, for `/raw`: small bodies only, so a few big files cannot fill the cache. */ | |
| 76 | const RAW_MAX = 1024 * 1024; | |
| 77 | const rawCache = new ViewCache({ maxEntries: 512, maxBytes: 32 * 1024 * 1024 }); | |
| 78 | const rawKey = (repoId: string, commit: string, path: string): string => ["raw", repoId, commit, path].join("\0"); | |
| 79 | ||
| 80 | export const cachedRaw = <T>(repoId: string, commit: string, path: string) => rawCache.get<T>(rawKey(repoId, commit, path)); | |
| 81 | ||
| 82 | export function keepRaw(repoId: string, commit: string, path: string, blob: { readonly size: number }): void { | |
| 83 | if (blob.size <= RAW_MAX) rawCache.set(rawKey(repoId, commit, path), blob, TTL_MS, blob.size); | |
| 84 | } |
test/anonymous-deep.test.ts +28
| @@ -221,3 +221,31 @@ test("a made-up Authorization header buys nothing: it's counted like anyone with | ||
| 221 | 221 | for (let i = 0; i <= STRANGER_BUDGET.address; i++) statuses.push((await get(deep, from("10.8.0.1", { authorization: "Bearer made-up" }))).status); |
| 222 | 222 | expect(statuses.at(-1)).toBe(429); |
| 223 | 223 | }); |
| 224 | ||
| 225 | test("a stranger's file page and raw bytes at a commit id are served after the first read with git gone (#9405)", async () => { | |
| 226 | setCrawlClock(app, () => 15_000_000); | |
| 227 | clearTreeCache(); | |
| 228 | const gitDir = join(dataDir, "repos", "alice", "open.git"); | |
| 229 | const anon = from("10.7.0.1"); | |
| 230 | const page = await get(`/alice/open/blob/${sha}/a.txt`, anon); | |
| 231 | const raw = await get(`/alice/open/raw/${sha}/a.txt`, anon); | |
| 232 | expect(page.status).toBe(200); | |
| 233 | const first = await page.text(); | |
| 234 | expect(await raw.text()).toBe("hi\n"); | |
| 235 | ||
| 236 | await rename(join(gitDir, "objects"), join(gitDir, "objects.away")); | |
| 237 | try { | |
| 238 | const again = await get(`/alice/open/blob/${sha}/a.txt`, anon); | |
| 239 | expect(again.status).toBe(200); | |
| 240 | expect(again.headers.get("cache-control")).toContain("immutable"); | |
| 241 | expect(await again.text()).toContain("hi"); | |
| 242 | expect(first).toContain("hi"); | |
| 243 | const bytes = await get(`/alice/open/raw/${sha}/a.txt`, anon); | |
| 244 | expect(bytes.status).toBe(200); | |
| 245 | expect(await bytes.text()).toBe("hi\n"); | |
| 246 | // a path never read is not in the cache, so it is a miss, not a page | |
| 247 | expect((await get(`/alice/open/blob/${sha}/never.txt`, anon)).status).toBe(404); | |
| 248 | } finally { | |
| 249 | await rename(join(gitDir, "objects.away"), join(gitDir, "objects")); | |
| 250 | } | |
| 251 | }); |
test/load-verdict.test.ts added +16
| @@ -0,0 +1,16 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { fell, percentile, summarize } from "../src/perf/load_verdict.ts"; | |
| 3 | ||
| 4 | // Hand calculation: ten latencies 1..10 ms, nearest-rank p50 = ceil(5) = 5th = 5 ms, p95 = ceil(9.5) = 10th = 10 ms. | |
| 5 | test("percentiles are nearest-rank and the served rate is served ÷ seconds", () => { | |
| 6 | const ms = [10, 1, 2, 3, 4, 5, 6, 7, 8, 9]; | |
| 7 | expect(percentile([...ms].sort((a, b) => a - b), 50)).toBe(5); | |
| 8 | expect(percentile([...ms].sort((a, b) => a - b), 95)).toBe(10); | |
| 9 | expect(summarize("1", 5, 2, ms, 3)).toEqual({ version: "1", asked: 5, seconds: 2, served: 10, refused: 3, servedPerSecond: 5, p50: 5, p95: 10 }); | |
| 10 | }); | |
| 11 | ||
| 12 | test("a release fails when its served rate falls more than 20% from the last", () => { | |
| 13 | const last = summarize("1", 500, 1, Array(500).fill(5), 0); | |
| 14 | expect(fell(last, summarize("2", 500, 1, Array(400).fill(5), 0))).toBeUndefined(); | |
| 15 | expect(fell(last, summarize("2", 500, 1, Array(399).fill(5), 0))).toContain("80% of 500/s"); | |
| 16 | }); |
new above
offline