trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

A cold code page renders off the main thread, so a cold forge serves more than ~220 a second #9633

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/9630-a-cold-code-page-renders-off-the-main-th → main 3 commits fork

  • Merged. torque/9630-a-cold-code-page-renders-off-the-main-th was merged into main as 1b1f8f3.

Review

Sign in to review this change.
7 files changed +248 −17
bench/results/9630-worker.md added +16
@@ -0,0 +1,16 @@
1 # Cold code pages rendered off the main thread (#9630)
2
3 Command: `bun bench/code-pages-at-rate.ts --rate 500 --seconds 20 --warm 0 --crawl 3000`
4 (in-process copy, edge bypassed, 3,000 distinct pinned code pages).
5
6 | | served | refused (429) | served/s | p95 |
7 |---|---|---|---|---|
8 | Baseline (9566-cold-forge.md) | 4,424 | 5,574 | 221 | 827 ms |
9 | Rows in a worker only | 3,812 | 6,187 | 191 | 943 ms (within noise of baseline) |
10 | Rows + page encode in workers, run 1 | 6,436 | 3,563 | 322 | 547 ms |
11 | Rows + page encode in workers, run 2 | 7,015 | 2,984 | 351 | 473 ms |
12
13 Highlight + row layout alone moved nothing: on the main thread the brotli/gzip of the ~150 KB page
14 was the larger share (`codePage+keep` ~4 ms vs `pane` ~1 ms once rows were out). Both now run on two
15 workers (`src/web/code_rows_pool.ts`); past 64 jobs in flight, or a 5 s timeout, the page does the work inline.
16 Runs differ ~15% on a shared machine. Refused strangers fall from 5,574 to ~3,000-3,560; still 429.
src/web/code_rows.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * The table rows of a highlighted file: plain data in, one string out. The
3 * page builds them inline for a warm read and in a worker (`code_rows_pool.ts`)
4 * for a cold one, from this one function, so the markup is the same either way.
5 */
6 import { countLines, highlightPrefix } from "./code.ts";
7 import { html, type Html } from "./html.ts";
8
9 /** What a file page draws of a text file. */
10 export interface CodeRows {
11 /** Lines in the whole file. */
12 readonly total: number;
13 /** Lines drawn (the first `limit`). */
14 readonly shown: number;
15 /** The `<tr>` rows, already escaped. */
16 readonly rows: string;
17 }
18
19 /** One `<tr>` per highlighted line, numbered from 1. */
20 export function rowsMarkup(lines: readonly Html[]): string {
21 return lines
22 .map((line, i) => html`<tr id="L${i + 1}"><td class="ln"><a href="#L${i + 1}">${i + 1}</a></td><td><pre>${line}</pre></td></tr>`.text)
23 .join("");
24 }
25
26 /** The first `limit` lines of `text` highlighted for `path`, as rows. */
27 export function codeRows(text: string, path: string, limit: number): CodeRows {
28 const lines = highlightPrefix(text, path, limit);
29 return { total: countLines(text), shown: lines.length, rows: rowsMarkup(lines) };
30 }
src/web/code_rows_pool.ts added +84
@@ -0,0 +1,84 @@
1 /**
2 * Cold code pages highlight and lay out their rows on worker threads, so the
3 * main loop keeps serving while a crawl asks for pages nobody has kept (#9630).
4 * Two workers, started on first use; a job that fails or runs past
5 * `JOB_MS` answers undefined and the caller does the work inline.
6 */
7 import type { Encoding } from "../http/compress.ts";
8 import type { CodeRows } from "./code_rows.ts";
9 import type { Job } from "./code_rows_worker.ts";
10
11 const WORKERS = 2;
12 const JOB_MS = 5_000;
13 /** Jobs in flight past which the caller renders inline instead of queueing. */
14 const MAX_PENDING = 64;
15
16 type Reply = { id: number; result?: unknown; error?: string };
17
18 const pool: Worker[] = [];
19 const waiting = new Map<number, { worker: Worker; done: (result: unknown) => void }>();
20 let next = 0;
21
22 function start(): Worker | undefined {
23 try {
24 const w = new Worker(new URL("./code_rows_worker.ts", import.meta.url).href);
25 w.onmessage = (e: MessageEvent<Reply>) => {
26 const job = waiting.get(e.data.id);
27 waiting.delete(e.data.id);
28 job?.done(e.data.result);
29 };
30 w.onerror = () => {
31 const at = pool.indexOf(w);
32 if (at >= 0) pool.splice(at, 1);
33 w.terminate();
34 // Its in-flight jobs will never answer: let their callers fall back now, not after JOB_MS.
35 for (const [id, job] of waiting) {
36 if (job.worker !== w) continue;
37 waiting.delete(id);
38 job.done(undefined);
39 }
40 };
41 w.unref();
42 return w;
43 } catch {
44 return undefined;
45 }
46 }
47
48 type Spec = Job extends infer J ? (J extends Job ? Omit<J, "id"> : never) : never;
49
50 function run<T>(job: Spec): Promise<T | undefined> {
51 if (waiting.size >= MAX_PENDING) return Promise.resolve(undefined);
52 if (pool.length < WORKERS) {
53 const w = start();
54 if (w) pool.push(w);
55 }
56 const worker = pool[next++ % Math.max(pool.length, 1)];
57 if (!worker) return Promise.resolve(undefined);
58 const id = next;
59 return new Promise((resolve) => {
60 const timer = setTimeout(() => {
61 waiting.delete(id);
62 resolve(undefined);
63 }, JOB_MS);
64 waiting.set(id, {
65 worker,
66 done: (result) => {
67 clearTimeout(timer);
68 resolve(result as T | undefined);
69 },
70 });
71 worker.postMessage({ ...job, id });
72 });
73 }
74
75 /** Rows for a file's bytes, or undefined when the caller should build them itself. */
76 export const rowsOffThread = (body: Uint8Array, path: string, limit: number): Promise<CodeRows | undefined> =>
77 run<CodeRows>({ kind: "rows", body, path, limit });
78
79 /** A finished page's bytes encoded, or undefined when the caller should encode them itself. */
80 export const encodeOffThread = (body: Uint8Array<ArrayBuffer>, encoding: Encoding): Promise<Uint8Array | undefined> =>
81 run<Uint8Array>({ kind: "encode", body, encoding });
82
83 /** Test seam: the live workers. */
84 export const workersForTest = (): readonly Worker[] => pool;
src/web/code_rows_worker.ts added +23
@@ -0,0 +1,23 @@
1 /** The worker side of `code_rows_pool.ts`: lay out rows, or encode a finished page. */
2 import { compressBytes, type Encoding } from "../http/compress.ts";
3 import { codeRows } from "./code_rows.ts";
4
5 declare const self: Worker;
6
7 export type Job =
8 | { id: number; kind: "rows"; body: Uint8Array; path: string; limit: number }
9 | { id: number; kind: "encode"; body: Uint8Array<ArrayBuffer>; encoding: Encoding };
10
11 self.onmessage = (e: MessageEvent<Job>) => {
12 const job = e.data;
13 try {
14 if (job.kind === "rows") {
15 const text = new TextDecoder().decode(job.body).replace(/\n$/, "");
16 postMessage({ id: job.id, result: codeRows(text, job.path, job.limit) });
17 } else {
18 postMessage({ id: job.id, result: compressBytes(job.body, job.encoding) });
19 }
20 } catch (error) {
21 postMessage({ id: job.id, error: String(error) });
22 }
23 };
src/web/pages/kept_page.ts +7 −3
@@ -25,6 +25,7 @@
2525 import { encodeRecent, negotiate, type Encoding } from "../../http/compress.ts";
2626 import type { Ctx } from "../../http/context.ts";
2727 import { ViewCache } from "../../view/cache.ts";
28 import { encodeOffThread } from "../code_rows_pool.ts";
2829 import { readFlash } from "../flash.ts";
2930 import { signInHref } from "../layout.ts";
3031
@@ -85,11 +86,14 @@ export function keptPage(key: string): Response | undefined {
8586 return new Response(hit.body, { status: 200, headers: new Headers([...hit.headers]) });
8687 }
8788
88 /** Keep `response`'s bytes, encoded, under `key`, and answer with them. Anything but a plain 200 goes out as it came. */
89 export async function keepPage(key: string, response: Response): Promise<Response> {
89 /** Keep `response`'s bytes, encoded, under `key`, and answer with them. `offThread` encodes on a worker. Anything but a plain 200 goes out as it came. */
90 export async function keepPage(key: string, response: Response, offThread = false): Promise<Response> {
9091 if (response.status !== 200 || response.headers.getSetCookie().length > 0 || response.headers.has("content-encoding")) return response;
9192 const encoding = encodingOf(key);
92 const body = encodeRecent(new Uint8Array(await response.arrayBuffer()), encoding);
93 const bytes = new Uint8Array(await response.arrayBuffer());
94 // A code page's encode is its biggest main-thread cost; a worker does it when one is free (#9630).
95 // Only the code page asks: other pages are small and keep the inline encode.
96 const body = (offThread ? await encodeOffThread(bytes, encoding) : undefined) ?? encodeRecent(bytes, encoding);
9397 const headers: [string, string][] = [...response.headers].filter(([name]) => KEPT_HEADERS.has(name));
9498 headers.push(["content-encoding", encoding]);
9599 store.set(key, { body, headers }, TTL_MS, body.length + 1024);
src/web/pages/repo.ts +32 −14
@@ -78,7 +78,9 @@ import { appStrip } from "./apps.ts";
7878 import { studioApp } from "../studio/app.ts";
7979 import { getAppByRepo } from "../../core/apps.ts";
8080 import { field, readForm } from "../form.ts";
81 import { html, when, type Html } from "../html.ts";
81 import { html, raw, when, type Html } from "../html.ts";
82 import { type CodeRows, rowsMarkup } from "../code_rows.ts";
83 import { rowsOffThread } from "../code_rows_pool.ts";
8284 import { flashTo, fragment, page, redirect } from "../layout.ts";
8385 import { renderMarkdown } from "../markdown.ts";
8486 import { csvJs, isCsvPath } from "../render/csv.ts";
@@ -693,6 +695,7 @@ export function blobBody(
693695 blob: Blob,
694696 highlighted?: (ms: number) => void,
695697 all = false,
698 ready?: CodeRows,
696699 ): { content: Html; lines: number } {
697700 const encoded = encodeURIComponent(ref);
698701 if (blob.kind === "binary") {
@@ -725,30 +728,35 @@ export function blobBody(
725728 }),
726729 };
727730 }
728 const text = new TextDecoder().decode(blob.body).replace(/\n$/, "");
729731 const limit = all ? MAX_RENDER_LINES : FIRST_LINES;
730732 // Highlighting runs left to right, so the first N lines highlight the same
731733 // alone as inside the whole file; only that prefix is read and painted.
732 const began = performance.now();
733 const total = countLines(text);
734 const lines = highlightPrefix(text, path, limit);
735 highlighted?.(performance.now() - began);
736 const truncated = total > lines.length;
734 let total: number;
735 let shown: number;
736 let rows: string;
737 if (ready !== undefined) ({ total, shown, rows } = ready);
738 else {
739 const text = new TextDecoder().decode(blob.body).replace(/\n$/, "");
740 const began = performance.now();
741 total = countLines(text);
742 const lines = highlightPrefix(text, path, limit);
743 highlighted?.(performance.now() - began);
744 shown = lines.length;
745 rows = rowsMarkup(lines);
746 }
747 const truncated = total > shown;
737748 const fileHref = `${base}/blob/${encoded}/${path}`;
738749 return {
739750 lines: total,
740751 content: html`<div class="card code">
741752 <table>
742 ${lines.map(
743 (line, i) =>
744 html`<tr id="L${i + 1}"><td class="ln"><a href="#L${i + 1}">${i + 1}</a></td><td><pre>${line}</pre></td></tr>`,
745 )}
753 ${raw(rows)}
746754 </table>
747755 </div>
748756 ${when(truncated, () =>
749757 empty({
750758 icon: "file",
751 title: `${(total - lines.length).toLocaleString("en-US")} more lines`,
759 title: `${(total - shown).toLocaleString("en-US")} more lines`,
752760 note: all
753761 ? "The rest is in the raw file."
754762 : "The page draws the first lines so it opens at once. A link to a later line (#L…) lands once you show all lines.",
@@ -1113,6 +1121,8 @@ export function renderBlob(
11131121 highlighted?: (ms: number) => void;
11141122 /** Draw every line instead of the first 1,500. */
11151123 all?: boolean;
1124 /** Rows a worker already built for this file. */
1125 rows?: CodeRows;
11161126 },
11171127 ): Html {
11181128 const base = scope.base;
@@ -1135,7 +1145,7 @@ export function renderBlob(
11351145 const { content, lines, summary } =
11361146 drawn !== undefined && fellBack === undefined
11371147 ? drawn
1138 : { ...blobBody(base, o.ref, o.path, blob, o.highlighted, o.all), summary: undefined };
1148 : { ...blobBody(base, o.ref, o.path, blob, o.highlighted, o.all, o.rows), summary: undefined };
11391149 const sizeText =
11401150 blob.kind === "too_large" ? "" : sizeWithTokens(blob.size, o.path);
11411151 const fileHref = `${base}/blob/${encoded}/${o.path}`;
@@ -1620,7 +1630,7 @@ export async function blobPage(ctx: Ctx): Promise<Response> {
16201630 git.done();
16211631 return immutableWhenAddressed(ctx.request, true, repo, ref, finished);
16221632 }
1623 const keepFinished = (response: Response) => (keptKey === undefined ? response : keepPage(keptKey, response));
1633 const keepFinished = (response: Response) => (keptKey === undefined ? response : keepPage(keptKey, response, true));
16241634 if (kept) {
16251635 const hit = cachedBlobPage(repo.id, ref, path, String(view), all);
16261636 if (hit !== undefined) {
@@ -1649,7 +1659,15 @@ export async function blobPage(ctx: Ctx): Promise<Response> {
16491659 let highlightMs: number | undefined;
16501660 const renderStart = performance.now();
16511661 const shown = capped(blob.value);
1662 // A cold page for a stranger highlights and lays out its rows on a worker (#9630).
1663 let rows: CodeRows | undefined;
1664 if (kept && shown.kind === "text" && (view === "source" || !blobRenderer(scope, path, shown))) {
1665 const began = performance.now();
1666 rows = await rowsOffThread(shown.body, path, all ? MAX_RENDER_LINES : FIRST_LINES);
1667 if (rows !== undefined) notePhase(ctx.request, "highlight", performance.now() - began);
1668 }
16521669 const pane = renderBlob(scope, {
1670 ...(rows !== undefined ? { rows } : {}),
16531671 refs,
16541672 ref,
16551673 path,
test/code-rows-worker.test.ts added +56
@@ -0,0 +1,56 @@
1 import { expect, test } from "bun:test";
2 import { codeRows } from "../src/web/code_rows.ts";
3 import { rowsOffThread } from "../src/web/code_rows_pool.ts";
4
5 const src = new TextEncoder().encode("const a = 1;\nif (a < 2) { return \"<b>\"; }\nlast\n");
6
7 test("a worker builds the same rows the main thread would", async () => {
8 const made = await rowsOffThread(src, "x.ts", 1_500);
9 const text = new TextDecoder().decode(src).replace(/\n$/, "");
10 expect(made).toEqual(codeRows(text, "x.ts", 1_500));
11 expect(made!.shown).toBe(3);
12 expect(made!.rows).toContain('id="L3"');
13 expect(made!.rows).not.toContain("<b>");
14 });
15
16 test("only the first lines are drawn but the whole file is counted", async () => {
17 const body = new TextEncoder().encode(Array.from({ length: 50 }, (_, i) => `line ${i}`).join("\n"));
18 const made = await rowsOffThread(body, "a.txt", 10);
19 expect(made).toMatchObject({ total: 50, shown: 10 });
20 });
21
22 test("a worker encodes a page to the bytes the main thread would", async () => {
23 const { compressBytes } = await import("../src/http/compress.ts");
24 const { encodeOffThread } = await import("../src/web/code_rows_pool.ts");
25 const page = new TextEncoder().encode("<p>hello</p>".repeat(500));
26 const made = await encodeOffThread(page, "gzip");
27 expect(Buffer.from(made!).equals(Buffer.from(compressBytes(page, "gzip")))).toBe(true);
28 });
29
30 test("a saturated pool answers undefined at once, and keepPage still stores a correct page", async () => {
31 const { keepPage, keptPage } = await import("../src/web/pages/kept_page.ts");
32 const { encodeRecent } = await import("../src/http/compress.ts");
33 const big = new TextEncoder().encode("x\n".repeat(20_000));
34 const jobs = Array.from({ length: 64 }, () => rowsOffThread(big, "a.txt", 1_500));
35 expect(await rowsOffThread(src, "x.ts", 10)).toBeUndefined(); // the 65th: MAX_PENDING
36 const html = new TextEncoder().encode("<p>kept</p>".repeat(300));
37 const key = "gzip\0saturated-test";
38 const served = await keepPage(key, new Response(html, { headers: { "content-type": "text/html" } }), true);
39 const expected = encodeRecent(html, "gzip");
40 expect(Buffer.from(await served.arrayBuffer()).equals(Buffer.from(expected))).toBe(true);
41 const again = keptPage(key);
42 expect(Buffer.from(await again!.arrayBuffer()).equals(Buffer.from(expected))).toBe(true);
43 await Promise.all(jobs);
44 });
45
46 test("a worker that errors answers its in-flight jobs undefined at once, not after the timeout", async () => {
47 const { workersForTest } = await import("../src/web/code_rows_pool.ts");
48 const big = new TextEncoder().encode("x\n".repeat(200_000));
49 const pending = Array.from({ length: 4 }, () => rowsOffThread(big, "a.txt", 1_500));
50 const began = performance.now();
51 for (const w of [...workersForTest()]) w.onerror?.(new ErrorEvent("error"));
52 expect(await Promise.all(pending)).toEqual([undefined, undefined, undefined, undefined]);
53 expect(performance.now() - began).toBeLessThan(1_000);
54 // The pool starts fresh workers for the next job.
55 expect(await rowsOffThread(src, "x.ts", 1_500)).toBeDefined();
56 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs