Public: anyone can read this; the crew can change it.

A pull's Files page names its Server-Timing phases and answers under 100 ms for a 40-file diff #4020

Closed opened this
Closed without merging

scout/forge:scout/3359-a-pull-s-files-page-names-its-server-tim → main 7 commits fork 6 threads · 6 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
5 files changed +334 −90
src/web/code.ts +77 −39
@@ -20,30 +20,71 @@ import {
2020 type Token,
2121 } from "./highlight.ts";
2222 import { highlightToLines, tokenClasses } from "./highlight-html.ts";
23 import { html, raw, type Html } from "./html.ts";
23 import { escapeHtml, html, raw, type Html } from "./html.ts";
2424
25 /** Per-character class attributes for one line, or null where none apply. */
26 function classMask(
27 text: string,
28 tokens: Token[] | undefined,
29 ): (string | null)[] {
30 const mask: (string | null)[] = new Array(text.length).fill(null);
31 if (!tokens) return mask;
25 /** A stretch of one line that shares a class attribute and a changed flag. */
26 interface Run {
27 end: number;
28 cls: string | null;
29 }
30
31 /**
32 * Rendered runs by class, then text. Code repeats itself — `const`, ` `, `=`,
33 * `(` — so most runs are one lookup instead of an escape and two concats.
34 * Bounded: past `MEMO_RUNS` the table starts over.
35 */
36 const MEMO_RUNS = 20_000;
37 const memo = new Map<string, Map<string, string>>();
38 let memoSize = 0;
39
40 function runHtml(cls: string | null, slice: string): string {
41 const key = cls ?? "";
42 let byText = memo.get(key);
43 if (byText === undefined) memo.set(key, (byText = new Map()));
44 const hit = byText.get(slice);
45 if (hit !== undefined) return hit;
46 const text = escapeHtml(slice);
47 const made = cls === null ? text : `<span class="${escapeHtml(cls)}">${text}</span>`;

The memo is global mutable state that is cleared wholesale. Add a test that the same text under different classes and the wd flag yields distinct output, and that output after a clear is still correct.

48 if (memoSize >= MEMO_RUNS) {
49 memo.clear();
50 memoSize = 0;
51 memo.set(key, (byText = new Map()));
52 }
53 byText.set(slice, made);
54 memoSize++;
55 return made;
56 }
57
58 /**
59 * The line's class attributes as runs: adjacent tokens with the same classes
60 * merge, and whatever the tokens did not cover stays unclassified.
61 */
62 function classRuns(length: number, tokens: Token[] | undefined): Run[] {
63 const runs: Run[] = [];
3264 let pos = 0;
33 for (const token of tokens) {
65 const push = (end: number, cls: string | null) => {
66 const last = runs[runs.length - 1];
67 if (last && last.cls === cls) last.end = end;
68 else runs.push({ end, cls });
69 };
70 for (const token of tokens ?? []) {
71 if (pos >= length) break;
3472 const classes = tokenClasses(token);
35 const value = classes === "" ? null : classes;
36 for (let k = 0; k < token.text.length && pos < text.length; k++)
37 mask[pos++] = value;
73 const end = Math.min(length, pos + token.text.length);
74 if (end > pos) push(end, classes === "" ? null : classes);
75 pos = end;
3876 }
3977 // If the token stream didn't cover the line (source mismatch), the tail
4078 // simply stays unclassified — still correct, just uncolored.
41 return mask;
79 if (pos < length) push(length, null);
80 return runs;
4281 }
4382
4483 /**
4584 * Compose a line into spans. `tokens` colors it; `changed` (per character)
46 * marks the diff highlight. Either may be omitted.
85 * marks the diff highlight. Either may be omitted. One span per run of
86 * (class, changed), cut at the boundaries of both segmentations; the work is
87 * per run, not per character.
4788 */
4889 export function composeLine(
4990 text: string,
@@ -51,34 +92,31 @@ export function composeLine(
5192 changed: boolean[] | undefined,
5293 ): Html {
5394 if (text === "") return html``;
54 const classes = classMask(text, tokens);
55 const parts: Html[] = [];
56
57 let runStart = 0;
58 const flush = (end: number) => {
59 if (end <= runStart) return;
60 const slice = text.slice(runStart, end);
61 const cls = classes[runStart];
62 const isChanged = changed?.[runStart] ?? false;
63 if ((cls === null || cls === undefined) && !isChanged) {
64 parts.push(html`${slice}`);
65 } else {
66 const names = [cls ?? "", isChanged ? "wd" : ""]
67 .filter(Boolean)
68 .join(" ");
69 parts.push(html`<span class="${names}">${slice}</span>`);
70 }
71 runStart = end;
95 const runs = classRuns(text.length, tokens);
96 const marked = changed?.some(Boolean) === true;
97 let out = "";
98 const emit = (from: number, to: number, cls: string | null, wd: boolean) => {
99 const slice = text.slice(from, to);
100 out += runHtml(wd ? (cls === null ? "wd" : `${cls} wd`) : cls, slice);
72101 };
73102
74 for (let i = 1; i < text.length; i++) {
75 const sameClass = classes[i] === classes[runStart];
76 const sameChanged =
77 (changed?.[i] ?? false) === (changed?.[runStart] ?? false);
78 if (!sameClass || !sameChanged) flush(i);
103 let from = 0;
104 for (const run of runs) {
105 if (!marked) {
106 emit(from, run.end, run.cls, false);
107 from = run.end;
108 continue;
109 }
110 // Cut the class run wherever the changed flag flips.
111 while (from < run.end) {
112 const flag = changed?.[from] === true;
113 let end = from + 1;
114 while (end < run.end && (changed?.[end] === true) === flag) end++;
115 emit(from, end, run.cls, flag);
116 from = end;
117 }
79118 }

changed![from] non-null assertions hide the case where changed is shorter than the text. Use changed?.[from] === true, which is also safer.

80 flush(text.length);
81 return html`${parts}`;
119 return raw(out);
82120 }
83121
84122 /** Highlight a whole blob into per-line Html, for the file browser. */
src/web/diff-syntax.ts +78 −26
@@ -3,11 +3,15 @@
33 * and new blobs (not the diff fragments — highlighting whole files is what
44 * keeps multi-line strings and comments correct across hunk boundaries) and
55 * key the tokens by path. The diff viewer then maps tokens onto rows by line
6 * number. Blobs are immutable by oid, so this is safe to cache upstream.
6 * number. Blobs are immutable by oid, so the tokens of a file at a commit are
7 * kept (`tokensCache`): a second look at the same head, by anyone, highlights
8 * nothing. Files are read together, so one file is highlighted while the
9 * next is on its way from git.
710 */
811
9 import type { Diff } from "../git/diff.ts";
12 import type { Diff, FileDiff } from "../git/diff.ts";
1013 import { blobForRender } from "../git/read.ts";
14 import type { Token } from "./highlight.ts";
1115 import { detectLanguage, highlightLines } from "./highlight.ts";
1216 import {
1317 MAX_DIFF_FILES,
@@ -17,6 +21,70 @@ import {
1721 } from "./diff-view.ts";
1822
1923 const CAP_BYTES = 1_000_000;
24 /** How many lines of tokens are kept; the oldest file goes first. */
25 const KEEP_LINES = 60_000;
26 const FULL_OID = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
27
28 const tokensCache = new Map<string, Token[][] | null>();
29 let keptLines = 0;
30
31 /** Forget everything kept; for tests. */
32 export function clearSyntaxCache(): void {
33 tokensCache.clear();
34 keptLines = 0;
35 }
36
37 function keep(key: string, lines: Token[][] | null): void {
38 const weight = lines?.length ?? 0;
39 if (weight > KEEP_LINES) return;
40 tokensCache.set(key, lines);
41 keptLines += weight;

This is a new process-wide cache with LRU eviction and a weight bound, and no test touches it. Add one test for a hit, one for eviction past KEEP_LINES, and one for a non-full-oid rev that must not be cached (a branch name can move).

42 for (const [oldest, value] of tokensCache) {
43 if (keptLines <= KEEP_LINES) break;
44 tokensCache.delete(oldest);
45 keptLines -= value?.length ?? 0;
46 }
47 }
48
49 /** The tokens of `path` at `rev`, or undefined when it is not shown as text. */
50 async function tokensAt(
51 dir: string,
52 rev: string,
53 path: string,
54 ): Promise<Token[][] | undefined> {
55 const key = `${rev}\0${path}`;
56 const cacheable = FULL_OID.test(rev);
57 if (cacheable && tokensCache.has(key)) {
58 const hit = tokensCache.get(key)!;
59 // Refresh its place: the most recently read file goes last.
60 tokensCache.delete(key);
61 tokensCache.set(key, hit);
62 return hit ?? undefined;
63 }
64 const blob = await blobForRender(dir, rev, path, CAP_BYTES);
65 const lines =
66 blob.kind === "text"
67 ? highlightLines(new TextDecoder().decode(blob.body), detectLanguage(path))
68 : null;
69 if (cacheable) keep(key, lines);
70 return lines ?? undefined;
71 }
72
73 async function tokensOf(
74 dir: string,
75 file: FileDiff,
76 oldRev: string,
77 newRev: string,
78 ): Promise<FileTokens> {
79 const [old, now] = await Promise.all([
80 file.oldPath ? tokensAt(dir, oldRev, file.oldPath) : undefined,
81 file.newPath ? tokensAt(dir, newRev, file.newPath) : undefined,
82 ]);
83 const entry: FileTokens = {};
84 if (old) entry.old = old;
85 if (now) entry.new = now;
86 return entry;
87 }
2088
2189 export async function diffSyntax(
2290 dir: string,
@@ -24,32 +92,16 @@ export async function diffSyntax(
2492 oldRev: string,
2593 newRev: string,
2694 ): Promise<SyntaxMap> {
27 const map: SyntaxMap = new Map();
2895 // Only the files the viewer will actually render as a table are worth
2996 // highlighting — a stubbed file shows no hunks, so tokenising it would
3097 // burn the same cost the stub exists to avoid.
31 for (const file of diff.files.slice(0, MAX_DIFF_FILES)) {
32 if (file.binary || stubFor(file) !== null) continue;
33 const entry: FileTokens = {};
34 if (file.oldPath) {
35 const blob = await blobForRender(dir, oldRev, file.oldPath, CAP_BYTES);
36 if (blob.kind === "text") {
37 entry.old = highlightLines(
38 new TextDecoder().decode(blob.body),
39 detectLanguage(file.oldPath),
40 );
41 }
42 }
43 if (file.newPath) {
44 const blob = await blobForRender(dir, newRev, file.newPath, CAP_BYTES);
45 if (blob.kind === "text") {
46 entry.new = highlightLines(
47 new TextDecoder().decode(blob.body),
48 detectLanguage(file.newPath),
49 );
50 }
51 }
52 map.set(file.path, entry);
53 }
98 const shown = diff.files
99 .slice(0, MAX_DIFF_FILES)
100 .filter((file) => !file.binary && stubFor(file) === null);
101 const entries = await Promise.all(
102 shown.map((file) => tokensOf(dir, file, oldRev, newRev)),
103 );
104 const map: SyntaxMap = new Map();
105 shown.forEach((file, i) => map.set(file.path, entries[i]!));
54106 return map;
55107 }
src/web/diff-view.ts +24 −20
@@ -76,6 +76,8 @@ export interface DiffViewOptions {
7676 /** Per-file render context, resolved once per file. */
7777 interface RowCtx {
7878 path: string;
79 /** `path`, encoded for a query string. */
80 pathQuery: string;
7981 tokens: FileTokens;
8082 opts: DiffViewOptions;
8183 }
@@ -253,13 +255,14 @@ function renderRow(row: Row, ctx: RowCtx): Html {
253255 const side: Side = row.kind === "del" ? "old" : "new";
254256 const lineNo = side === "old" ? row.oldNumber : row.newNumber;
255257
256 const gutter = (n: number | null, isAnchorSide: boolean) => {
257 if (n === null) return html`<td class="dn"></td>`;
258 if (!opts.reviewBase || !isAnchorSide)
259 return html`<td class="dn">${n}</td>`;
260 const href = `${opts.reviewPage ?? opts.reviewBase}?rc=${encodeURIComponent(path)}~${side}~${n}#rc`;
261 return html`<td class="dn">
262 <a class="dn-comment" href="${href}" title="Comment on this line">${n}</a>
258 // One row is thousands of rows on a big change: plain strings, with every
259 // dynamic piece escaped, and the path's encoding done once per file.
260 const gutter = (n: number | null, isAnchorSide: boolean): string => {
261 if (n === null) return `<td class="dn"></td>`;
262 if (!opts.reviewBase || !isAnchorSide) return `<td class="dn">${n}</td>`;
263 const href = `${opts.reviewPage ?? opts.reviewBase}?rc=${ctx.pathQuery}~${side}~${n}#rc`;
264 return `<td class="dn">
265 <a class="dn-comment" href="${escapeHtml(href)}" title="Comment on this line">${n}</a>
263266 </td>`;
264267 };
265268
@@ -267,23 +270,23 @@ function renderRow(row: Row, ctx: RowCtx): Html {
267270 // in one value, and interpolated whole. A pretty-printed newline between
268271 // `<td>` and its span would render as a literal blank line and an indent —
269272 // which is exactly how this viewer once lost its density.
270 const cell = html`<span class="dcx"
271 >${composeLine(row.text, tokenLine, row.changed)}</span
272 >${when(
273 row.noNewline,
274 () => html`<span class="nonl" title="No newline at end of file">↵</span>`,
275 )}`;
276 const rowHtml = html`<tr class="dl" data-k="${row.kind}">
277 ${gutter(row.oldNumber, side === "old")}${gutter(
278 row.newNumber,
279 side === "new",
280 )}
273 const cell = `<span class="dcx"
274 >${composeLine(row.text, tokenLine, row.changed).text}</span
275 >${
276 row.noNewline
277 ? `<span class="nonl" title="No newline at end of file">↵</span>`
278 : ""
279 }`;
280 const rowHtml = raw(`<tr class="dl" data-k="${escapeHtml(row.kind)}">
281 ${gutter(row.oldNumber, side === "old")}${gutter(row.newNumber, side === "new")}
281282 <td class="dc" data-mark="${mark}">${cell}</td>
282 </tr>`;
283 </tr>`);
283284
284285 // Existing comments and a form, if any, ride under the row.
285286 const extras =
286 lineNo === null ? html`` : reviewRows(path, side, lineNo, row.text, ctx);
287 lineNo === null || (!opts.comments?.size && !opts.active)
288 ? html``
289 : reviewRows(path, side, lineNo, row.text, ctx);
287290 return html`${rowHtml}${extras}`;
288291 }
289292
@@ -521,6 +524,7 @@ function fileView(file: FileDiff, opts: DiffViewOptions): Html {
521524 // an active comment or existing comments is forced open so it's visible.
522525 const ctx: RowCtx = {
523526 path: file.path,
527 pathQuery: encodeURIComponent(file.path),
524528 tokens: opts.syntax?.get(file.path) ?? {},
525529 opts,
526530 };
src/web/html.ts +2 −5
@@ -42,11 +42,8 @@ function render(value: Value): string {
4242 }
4343
4444 export function html(strings: TemplateStringsArray, ...values: Value[]): Html {
45 let out = "";
46 for (const [i, chunk] of strings.entries()) {
47 out += chunk;
48 if (i < values.length) out += render(values[i]);
49 }
45 let out = strings[0]!;
46 for (let i = 0; i < values.length; i++) out += render(values[i]!) + strings[i + 1]!;
5047 return new Html(out);
5148 }
5249
test/code-compose.test.ts added +153
@@ -0,0 +1,153 @@
1 /**
2 * How a diff line becomes spans (`composeLine`), what the diff table does with
3 * hostile names (`renderDiff`), and what `diffSyntax` keeps between requests.
4 */
5
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm, writeFile } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { composeLine } from "../src/web/code.ts";
11 import { clearSyntaxCache, diffSyntax } from "../src/web/diff-syntax.ts";
12 import { renderDiff } from "../src/web/diff-view.ts";
13 import { parseUnifiedDiff } from "../src/git/diff.ts";
14 import type { Token } from "../src/web/highlight.ts";
15
16 const s = (t: ReturnType<typeof composeLine>) => t.text;
17
18 test("composeLine: no tokens and no marks is the escaped text; empty is empty", () => {
19 expect(s(composeLine("a<b", undefined, undefined))).toBe("a&lt;b");
20 expect(s(composeLine("", [], undefined))).toBe("");
21 });
22
23 test("composeLine: tokens become class spans, adjacent equal classes merge", () => {
24 const tokens: Token[] = [
25 { cls: "keyword", text: "const" },
26 { cls: "plain", text: " " },
27 { cls: "plain", text: "x" },
28 ];
29 const out = s(composeLine("const x", tokens, undefined));
30 expect(out.startsWith(`<span class="`)).toBe(true);
31 expect(out.endsWith(" x")).toBe(true);
32 expect(out.match(/<span/g)!.length).toBe(1);
33 expect(out).toContain("const</span>");
34 });
35
36 test("composeLine: changed spans cut runs where the flag flips, and carry wd", () => {
37 const changed = [false, true, true, false];
38 const out = s(composeLine("abcd", undefined, changed));
39 expect(out).toBe(`a<span class="wd">bc</span>d`);
40 const tokens: Token[] = [{ cls: "string", text: "abcd" }];
41 const both = s(composeLine("abcd", tokens, changed));
42 expect(both.match(/ wd">bc<\/span>/g)!.length).toBe(1);
43 expect(both.match(/<span/g)!.length).toBe(3);
44 });
45
46 test("composeLine: a changed array shorter than the text reads as unchanged", () => {
47 expect(s(composeLine("abcd", undefined, [true]))).toBe(`<span class="wd">a</span>bcd`);
48 });
49
50 test("composeLine: tokens that stop short leave the tail unclassified; long ones are clipped", () => {
51 const short = s(composeLine("abcd", [{ cls: "keyword", text: "ab" }], undefined));
52 expect(short.endsWith("</span>cd")).toBe(true);
53 const long = s(composeLine("ab", [{ cls: "keyword", text: "abcdef" }], undefined));
54 expect(long).toContain(">ab</span>");
55 expect(long).not.toContain("cd");
56 });
57
58 test("composeLine: the memo keys on class, text and wd, and survives a flood past its bound", () => {
59 const kw: Token[] = [{ cls: "keyword", text: "x" }];
60 const st: Token[] = [{ cls: "string", text: "x" }];
61 const a = s(composeLine("x", kw, undefined));
62 const b = s(composeLine("x", st, undefined));
63 const c = s(composeLine("x", kw, [true]));
64 expect(new Set([a, b, c]).size).toBe(3);
65 expect(c).toContain("wd");
66 // More distinct runs than the memo holds: it starts over, and stays right.
67 for (let i = 0; i < 25_000; i++) composeLine(`<${i}>`, kw, undefined);
68 expect(s(composeLine("x", kw, undefined))).toBe(a);
69 expect(s(composeLine("x", st, undefined))).toBe(b);
70 expect(s(composeLine("<1>", [{ cls: "keyword", text: "<1>" }], undefined))).toContain(">&lt;1&gt;</span>");
71 });
72
73 test("renderDiff escapes a path and a line carrying quotes and tags", () => {
74 const path = `a"'><img src=x onerror=1>.ts`;
75 const diff = parseUnifiedDiff(
76 `diff --git a/x b/x\nnew file mode 100644\n--- /dev/null\n+++ b/x\n@@ -0,0 +1 @@\n+"><script>1</script>\n`,
77 );
78 const named = {
79 ...diff,
80 files: diff.files.map((f) => ({ ...f, path, newPath: path })),
81 };
82 const out = renderDiff(named, {
83 reviewBase: "/o/n/pulls/1",
84 reviewPage: "/o/n/pulls/1/files",
85 fileLinkBase: "/o/n/blob/abc",
86 }).text;
87 expect(out).not.toContain("<img src=x");
88 expect(out).not.toContain("<script>1");
89 expect(out).toContain("&lt;script&gt;1");
90 expect(out).not.toMatch(/href="[^"]*"[^>]*onerror/);
91 expect(out).toContain("&quot;");
92 });
93
94 // ---- diffSyntax's cache -------------------------------------------------
95
96 let dir = "";
97 const git = async (...args: string[]): Promise<string> => {
98 const p = Bun.spawn(["git", "-C", dir, ...args], {
99 stdout: "pipe",
100 stderr: "pipe",
101 env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" },
102 });
103 const out = await new Response(p.stdout).text();
104 await p.exited;
105 return out.trim();
106 };
107 const lines = (n: number, tag: string) => Array.from({ length: n }, (_, i) => `${tag}${i};`).join("\n") + "\n";
108 const oneFile = (path: string) =>
109 parseUnifiedDiff(`diff --git a/${path} b/${path}\nnew file mode 100644\n--- /dev/null\n+++ b/${path}\n@@ -0,0 +1 @@\n+x\n`);
110
111 beforeAll(async () => {
112 dir = await mkdtemp(join(tmpdir(), "syntax-"));
113 await git("init", "-q", "-b", "main");
114 await writeFile(join(dir, "a.ts"), "const a = 1;\n");
115 await writeFile(join(dir, "big1.ts"), lines(40_000, "p"));
116 await writeFile(join(dir, "big2.ts"), lines(40_000, "q"));
117 await git("add", ".");
118 await git("commit", "-q", "-m", "one");
119 });
120 afterAll(async () => {
121 clearSyntaxCache();
122 await rm(dir, { recursive: true, force: true });
123 });
124
125 test("diffSyntax: a full oid is read once; the second call returns the same tokens", async () => {
126 clearSyntaxCache();
127 const oid = await git("rev-parse", "HEAD");
128 const first = await diffSyntax(dir, oneFile("a.ts"), oid, oid);
129 const second = await diffSyntax(dir, oneFile("a.ts"), oid, oid);
130 expect(first.get("a.ts")!.new!.length).toBeGreaterThan(0);
131 expect(second.get("a.ts")!.new).toBe(first.get("a.ts")!.new!);
132 });
133
134 test("diffSyntax: past the line budget the oldest file goes first", async () => {
135 clearSyntaxCache();
136 const oid = await git("rev-parse", "HEAD");
137 const get = async (p: string) => (await diffSyntax(dir, oneFile(p), oid, oid)).get(p)!.new!;
138 const one = await get("big1.ts");
139 expect(await get("big1.ts")).toBe(one);
140 await get("big2.ts"); // 80 000 lines kept would be over 60 000: big1 is dropped
141 expect(await get("big1.ts")).not.toBe(one);
142 });
143
144 test("diffSyntax: a rev that is not a full oid is never kept, because a branch can move", async () => {
145 clearSyntaxCache();
146 const before = await diffSyntax(dir, oneFile("a.ts"), "main", "main");
147 await writeFile(join(dir, "a.ts"), "let changed = 2;\n");
148 await git("commit", "-qam", "two");
149 const after = await diffSyntax(dir, oneFile("a.ts"), "main", "main");
150 const text = (m: typeof before) => m.get("a.ts")!.new!.flat().map((t) => t.text).join("");
151 expect(text(before)).toContain("const a");
152 expect(text(after)).toContain("let changed");
153 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs