Public: anyone can read this; the crew can change it.

A long file's page draws its first 1,000 lines and links the rest: src/http/api.ts's page under 400 KB and 60 ms #6610

Parked opened this
Closed without merging

scout/forge:scout/6222-a-long-file-s-page-draws-its-first-1-000 → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
4 files changed +124 −16
src/web/client.ts +3
@@ -1387,6 +1387,9 @@ export const shellJs = /* js */ `
13871387 var lo = Number(m[1]); var hi = Number(m[2] || m[1]);
13881388 if (hi < lo) { var t = lo; lo = hi; hi = t; }
13891389 anchor = lo;
1390 // An anchor past the cut: the page that draws every line.
1391 var cut = doc.getElementById('code-cut');
1392 if (cut && !doc.getElementById('L' + lo)) { location.replace(cut.getAttribute('href') + location.hash); return; }
13901393 for (var n = lo; n <= hi; n++) {
13911394 var row = doc.getElementById('L' + n);
13921395 if (row) row.setAttribute('data-hl', '');
src/web/code.ts +38
@@ -97,3 +97,41 @@ const firstLineOf = (content: string): string => {
9797 };
9898
9999 export { detectLanguage, highlightLines };
100
101 /** Highlighted lines kept by content hash, so a repeat view highlights nothing. */
102 const CACHE_MAX = 32;
103 const cache = new Map<string, Html[]>();
104
105 export const countLines = (text: string): number => {
106 let n = 1;
107 for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) n++;
108 return n;
109 };
110
111 /**
112 * The first `limit` lines of a blob, highlighted. Keyed by what the file
113 * says (a blob's oid is a hash of the same bytes), its name and the limit;
114 * bounded to the last CACHE_MAX files, least recently used out.
115 */
116 export function highlightPrefix(
117 text: string,
118 path: string,
119 limit: number,
120 ): Html[] {
121 const key = `${Bun.hash(text).toString(36)}:${text.length}:${limit}:${path.slice(path.lastIndexOf("/") + 1)}`;
122 const hit = cache.get(key);
123 if (hit) {
124 cache.delete(key);
125 cache.set(key, hit);
126 return hit;
127 }
128 let end = -1;
129 for (let n = 0; n < limit; n++) {
130 end = text.indexOf("\n", end + 1);
131 if (end === -1) break;
132 }
133 const lines = highlightBlob(end === -1 ? text : text.slice(0, end), path);
134 cache.set(key, lines);
135 if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value!);
136 return lines;
137 }
src/web/pages/repo.ts +26 −16
@@ -60,7 +60,7 @@ import {
6060 signInFirst,
6161 } from "../chrome.ts";
6262 import { codeJs } from "../client.ts";
63 import { highlightBlob } from "../code.ts";
63 import { countLines, highlightPrefix } from "../code.ts";
6464 import { latestWave } from "../../story/waves.ts";
6565 import { notePhase, takeSessionTime, timePhase, timePhaseAsync } from "../../core/route_timing.ts";
6666 import { liveClientJs, liveMount } from "../live.ts";
@@ -92,7 +92,9 @@ import {
9292 const MAX_RENDER_BYTES = 1_000_000;
9393 // A blob under the byte cap can still be pathological (a megabyte of newlines
9494 // is ~490k lines). Bounding rendered lines bounds the response independently.
95 const MAX_RENDER_LINES = 5_000;
95 /** Lines a file page draws before it links the rest (`?all=1` draws every one). */
96 const FIRST_LINES = 1_000;
97 const MAX_RENDER_LINES = 20_000;
9698
9799 import { pathCrumbs, refMenu, refSetFrom, type RefSet } from "./refs.ts";
98100 import {
@@ -613,6 +615,7 @@ export function blobBody(
613615 path: string,
614616 blob: Blob,
615617 highlighted?: (ms: number) => void,
618 all = false,
616619 ): { content: Html; lines: number } {
617620 const encoded = encodeURIComponent(ref);
618621 if (blob.kind === "binary") {
@@ -645,32 +648,36 @@ export function blobBody(
645648 };
646649 }
647650 const text = new TextDecoder().decode(blob.body).replace(/\n$/, "");
648 // Highlight the whole file (multi-line constructs need full context),
649 // then take the first N lines for rendering.
651 const limit = all ? MAX_RENDER_LINES : FIRST_LINES;
652 // Highlighting runs left to right, so the first N lines highlight the same
653 // alone as inside the whole file; only that prefix is read and painted.
650654 const began = performance.now();
651 const highlight = highlightBlob(text, path);
655 const total = countLines(text);
656 const lines = highlightPrefix(text, path, limit);
652657 highlighted?.(performance.now() - began);
653 const lines = highlight.slice(0, MAX_RENDER_LINES);
654 const truncated = highlight.length > MAX_RENDER_LINES;
658 const truncated = total > lines.length;
659 const fileHref = `${base}/blob/${encoded}/${path}`;
655660 return {
656 lines: highlight.length,
661 lines: total,
657662 content: html`<div class="card code">
658663 <table>
659664 ${lines.map(
660665 (line, i) =>
661 html`<tr id="L${i + 1}">
662 <td class="ln"><a href="#L${i + 1}">${i + 1}</a></td>
663 <td><pre>${line}</pre></td>
664 </tr>`,
666 html`<tr id="L${i + 1}"><td class="ln"><a href="#L${i + 1}">${i + 1}</a></td><td><pre>${line}</pre></td></tr>`,
665667 )}
666668 </table>
667669 </div>
668670 ${when(truncated, () =>
669671 empty({
670672 icon: "alert",
671 title: "Showing the first 5,000 lines",
672 note: `This file has ${highlight.length.toLocaleString()} lines. Fetch the raw bytes for all of them.`,
673 action: btn("View raw", { href: `${base}/raw/${encoded}/${path}` }),
673 title: `Showing ${lines.length.toLocaleString()} of ${total.toLocaleString()} lines`,
674 note: all
675 ? "The rest is in the raw file."
676 : "The page draws the first lines so it opens at once.",
677 action: html`${all
678 ? html``
679 : html`<a id="code-cut" data-variant="primary" class="btn" href="${fileHref}?all=1">Show all lines</a>`}
680 ${btn("View raw", { href: `${base}/raw/${encoded}/${path}` })}`,
674681 }),
675682 )}`,
676683 };
@@ -998,6 +1005,8 @@ export function renderBlob(
9981005 view?: BlobView;
9991006 /** Told how long highlighting took, for Server-Timing. */
10001007 highlighted?: (ms: number) => void;
1008 /** Draw every line instead of the first thousand. */
1009 all?: boolean;
10011010 },
10021011 ): Html {
10031012 const base = scope.base;
@@ -1007,7 +1016,7 @@ export function renderBlob(
10071016 const { content, lines } =
10081017 view === "rendered" && o.blob.kind === "text"
10091018 ? renderedMarkdown(o.blob)
1010 : blobBody(base, o.ref, o.path, o.blob, o.highlighted);
1019 : blobBody(base, o.ref, o.path, o.blob, o.highlighted, o.all);
10111020 const sizeText =
10121021 o.blob.kind === "too_large" ? "" : sizeWithTokens(o.blob.size, o.path);
10131022 const fileHref = `${base}/blob/${encoded}/${o.path}`;
@@ -1432,6 +1441,7 @@ export async function blobPage(ctx: Ctx): Promise<Response> {
14321441 blob: capped(blob.value),
14331442 view: blobView(ctx.url.searchParams.get("view")),
14341443 highlighted: (ms) => (highlightMs = (highlightMs ?? 0) + ms),
1444 all: ctx.url.searchParams.get("all") === "1",
14351445 });
14361446 if (highlightMs !== undefined) notePhase(ctx.request, "highlight", highlightMs);
14371447 const paneMs = performance.now() - renderStart - (highlightMs ?? 0);
test/blob-long-file.test.ts added +57
@@ -0,0 +1,57 @@
1 /** A long file's page draws its first 1,000 lines and links the rest. */
2 import { describe, expect, test } from "bun:test";
3 import { blobBody } from "../src/web/pages/repo.ts";
4 import { highlightPrefix } from "../src/web/code.ts";
5 import type { Blob } from "../src/view/source.ts";
6
7 const source = Array.from(
8 { length: 7_000 },
9 (_, i) => ` const v${i} = compute(${i}); // line ${i}`,
10 ).join("\n");
11 const body = new TextEncoder().encode(source);
12 const blob: Blob = { kind: "text", body, size: body.length };
13 const draw = (all = false) =>
14 blobBody("/alice/app", "main", "src/big.ts", blob, undefined, all);
15 const rows = (h: string) => h.split('<tr id="L').length - 1;
16
17 describe("long file page", () => {
18 test("default draws 1,000 of 7,000 lines, small and fast", () => {
19 draw(); // first view pays the highlight
20 const began = performance.now();
21 const page = String(draw().content);
22 const ms = performance.now() - began;
23 expect(rows(page)).toBe(1_000);
24 expect(page).toContain("Showing 1,000 of 7,000 lines");
25 expect(page).toContain("/alice/app/blob/main/src/big.ts?all=1");
26 expect(page).toContain("/alice/app/raw/main/src/big.ts");
27 expect(new TextEncoder().encode(page).length).toBeLessThan(400_000);
28 expect(ms).toBeLessThan(60);
29 });
30
31 test("the first view alone stays under 60 ms", () => {
32 const fresh = source.replace("line 0", "line zero");
33 const b = new TextEncoder().encode(fresh);
34 const began = performance.now();
35 blobBody("/a/b", "main", "x.ts", { kind: "text", body: b, size: b.length });
36 expect(performance.now() - began).toBeLessThan(60);
37 });
38
39 test("?all=1 draws every line", () => {
40 const page = String(draw(true).content);
41 expect(rows(page)).toBe(7_000);
42 expect(page).not.toContain("Show all lines");
43 });
44
45 test("a repeat view returns the cached highlight", () => {
46 expect(highlightPrefix(source, "src/big.ts", 1_000)).toBe(
47 highlightPrefix(source, "src/big.ts", 1_000),
48 );
49 });
50
51 test("the prefix highlights like the whole file's first lines", () => {
52 const whole = highlightPrefix(source, "src/big.ts", 7_000);
53 const part = highlightPrefix(source, "src/big.ts", 1_000);
54 expect(part.length).toBe(1_000);
55 expect(String(part[999])).toBe(String(whole[999]));
56 });
57 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs