Public: anyone can read this; the crew can change it.
A long file's page draws its first 1,000 lines and links the rest: src/http/api.ts's page under 400 KB and 60 ms #6617
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6222-a-long-file-s-page-draws-its-first-1-000 → main 6 commits fork 2 threads · 2 unresolved
7 files
changed +153
−23
docs/STATUS.md +6
| @@ -1,5 +1,11 @@ | ||
| 1 | 1 | ## What works today |
| 2 | 2 | |
| 3 | - **A long file's page** draws its first 1,000 lines and says "Showing 1,000 of | |
| 4 | N lines", with *Show all lines* (`?all=1`) and the raw file. `?all=1` stops at | |
| 5 | 10,000 lines (it was 5,000 for every view before); past that, the raw file is | |
| 6 | the way. Highlighted lines are cached by content hash (32 files, least | |
| 7 | recently used out). A `#L…` anchor past the cut is redirected to `?all=1` by | |
| 8 | the page script; with scripts off the notice says to show all lines. | |
| 3 | 9 | - **A repository's runs page filters by state**: a row of links (All, then each |
| 4 | 10 | state) above the list, or `/:owner/:name/runs?state=failed` by hand. An unknown |
| 5 | 11 | state is ignored, and the word filter and the pager keep the chosen state. The |
src/web/client.ts +3
| @@ -1387,6 +1387,9 @@ export const shellJs = /* js */ ` | ||
| 1387 | 1387 | var lo = Number(m[1]); var hi = Number(m[2] || m[1]); |
| 1388 | 1388 | if (hi < lo) { var t = lo; lo = hi; hi = t; } |
| 1389 | 1389 | anchor = lo; |
| 1390 | // An anchor past the cut: the page that draws every line. | |
| 1391 | var cut = doc.getElementById('code-cut'); | |
| 1392 | if (cut && !doc.getElementById('L' + lo)) { location.replace(cut.getAttribute('href') + location.hash); return; } | |
| 1390 | 1393 | for (var n = lo; n <= hi; n++) { |
| 1391 | 1394 | var row = doc.getElementById('L' + n); |
| 1392 | 1395 | if (row) row.setAttribute('data-hl', ''); |
src/web/code.ts +38
| @@ -97,3 +97,41 @@ const firstLineOf = (content: string): string => { | ||
| 97 | 97 | }; |
| 98 | 98 | |
| 99 | 99 | export { detectLanguage, highlightLines }; |
| 100 | ||
| 101 | /** Highlighted lines kept by content hash, so a repeat view highlights nothing. */ | |
| 102 | const CACHE_MAX = 32; | |
| 103 | const cache = new Map<string, Html[]>(); | |
| 104 | ||
| 105 | export const countLines = (text: string): number => { | |
| 106 | let n = 1; | |
| 107 | for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) n++; | |
| 108 | return n; | |
| 109 | }; | |
| 110 | ||
| 111 | /** | |
| 112 | * The first `limit` lines of a blob, highlighted. Keyed by what the file | |
| 113 | * says (a blob's oid is a hash of the same bytes), its name and the limit; | |
| 114 | * bounded to the last CACHE_MAX files, least recently used out. | |
| 115 | */ | |
| 116 | export function highlightPrefix( | |
| 117 | text: string, | |
| 118 | path: string, | |
| 119 | limit: number, | |
| 120 | ): Html[] { | |
| 121 | const key = `${Bun.hash(text).toString(36)}:${text.length}:${limit}:${path.slice(path.lastIndexOf("/") + 1)}`; | |
| 122 | const hit = cache.get(key); | |
| 123 | if (hit) { | |
| 124 | cache.delete(key); | |
| 125 | cache.set(key, hit); | |
| 126 | return hit; | |
| 127 | } | |
| 128 | let end = -1; | |
| 129 | for (let n = 0; n < limit; n++) { | |
| 130 | end = text.indexOf("\n", end + 1); | |
| 131 | if (end === -1) break; | |
| 132 | } | |
| 133 | const lines = highlightBlob(end === -1 ? text : text.slice(0, end), path); | |
| 134 | cache.set(key, lines); | |
| 135 | if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value!); | |
| 136 | return lines; | |
| 137 | } |
src/web/pages/repo.ts +34 −17
| @@ -60,7 +60,7 @@ import { | ||
| 60 | 60 | signInFirst, |
| 61 | 61 | } from "../chrome.ts"; |
| 62 | 62 | import { codeJs } from "../client.ts"; |
| 63 | import { highlightBlob } from "../code.ts"; | |
| 63 | import { countLines, highlightPrefix } from "../code.ts"; | |
| 64 | 64 | import { latestWave } from "../../story/waves.ts"; |
| 65 | 65 | import { notePhase, takeSessionTime, timePhase, timePhaseAsync } from "../../core/route_timing.ts"; |
| 66 | 66 | import { liveClientJs, liveMount } from "../live.ts"; |
| @@ -92,7 +92,9 @@ import { | ||
| 92 | 92 | const MAX_RENDER_BYTES = 1_000_000; |
| 93 | 93 | // A blob under the byte cap can still be pathological (a megabyte of newlines |
| 94 | 94 | // is ~490k lines). Bounding rendered lines bounds the response independently. |
| 95 | const MAX_RENDER_LINES = 5_000; | |
| 95 | /** Lines a file page draws before it links the rest (`?all=1` draws every one). */ | |
| 96 | const FIRST_LINES = 1_000; | |
| 97 | const MAX_RENDER_LINES = 10_000; | |
| 96 | 98 | |
| 97 | 99 | import { pathCrumbs, refMenu, refSetFrom, type RefSet } from "./refs.ts"; |
| 98 | 100 | import { |
| @@ -613,6 +615,7 @@ export function blobBody( | ||
| 613 | 615 | path: string, |
| 614 | 616 | blob: Blob, |
| 615 | 617 | highlighted?: (ms: number) => void, |
| 618 | all = false, | |
| 616 | 619 | ): { content: Html; lines: number } { |
| 617 | 620 | const encoded = encodeURIComponent(ref); |
| 618 | 621 | if (blob.kind === "binary") { |
| @@ -645,32 +648,43 @@ export function blobBody( | ||
| 645 | 648 | }; |
| 646 | 649 | } |
| 647 | 650 | const text = new TextDecoder().decode(blob.body).replace(/\n$/, ""); |
| 648 | // Highlight the whole file (multi-line constructs need full context), | |
| 649 | // then take the first N lines for rendering. | |
| 651 | const limit = all ? MAX_RENDER_LINES : FIRST_LINES; | |
| 652 | // Highlighting runs left to right, so the first N lines highlight the same | |
| 653 | // alone as inside the whole file; only that prefix is read and painted. | |
| 650 | 654 | const began = performance.now(); |
| 651 | const highlight = highlightBlob(text, path); | |
| 655 | const total = countLines(text); | |
| 656 | const lines = highlightPrefix(text, path, limit); | |
| 652 | 657 | highlighted?.(performance.now() - began); |
| 653 | const lines = highlight.slice(0, MAX_RENDER_LINES); | |
| 654 | const truncated = highlight.length > MAX_RENDER_LINES; | |
| 658 | const truncated = total > lines.length; | |
| 659 | const fileHref = `${base}/blob/${encoded}/${path}`; | |
| 655 | 660 | return { |
| 656 | lines: highlight.length, | |
| 661 | lines: total, | |
| 657 | 662 | content: html`<div class="card code"> |
| 658 | 663 | <table> |
| 659 | 664 | ${lines.map( |
| 660 | 665 | (line, i) => |
| 661 | html`<tr id="L${i + 1}"> | |
| 662 | <td class="ln"><a href="#L${i + 1}">${i + 1}</a></td> | |
| 663 | <td><pre>${line}</pre></td> | |
| 664 | </tr>`, | |
| 666 | html`<tr id="L${i + 1}"><td class="ln"><a href="#L${i + 1}">${i + 1}</a></td><td><pre>${line}</pre></td></tr>`, | |
| 665 | 667 | )} |
| 666 | 668 | </table> |
| 667 | 669 | </div> |
| 668 | 670 | ${when(truncated, () => |
| 669 | 671 | empty({ |
| 670 | icon: "alert", | |
| 671 | title: "Showing the first 5,000 lines", | |
| 672 | note: `This file has ${highlight.length.toLocaleString()} lines. Fetch the raw bytes for all of them.`, | |
| 673 | action: btn("View raw", { href: `${base}/raw/${encoded}/${path}` }), | |
| 672 | icon: "file", | |
| 673 | title: `Showing ${lines.length.toLocaleString()} of ${total.toLocaleString()} lines`, | |
| 674 | note: all | |
| 675 | ? "The rest is in the raw file." | |
| 676 | : "The page draws the first lines so it opens at once. A link to a later line (#L…) lands once you show all lines.", | |
| 677 | action: html`${all | |
| 678 | ? html`` | |
| 679 | : btn("Show all lines", { | |
| 680 | href: `${fileHref}?all=1`, | |
| 681 | id: "code-cut", | |
| 682 | icon: "file", | |
| 683 | })} | |
| 684 | ${btn("View raw", { | |
| 685 | href: `${base}/raw/${encoded}/${path}`, | |
| 686 | variant: "outline", | |
| 687 | })}`, | |
| 674 | 688 | }), |
| 675 | 689 | )}`, |
| 676 | 690 | }; |
| @@ -998,6 +1012,8 @@ export function renderBlob( | ||
| 998 | 1012 | view?: BlobView; |
| 999 | 1013 | /** Told how long highlighting took, for Server-Timing. */ |
| 1000 | 1014 | highlighted?: (ms: number) => void; |
| 1015 | /** Draw every line instead of the first thousand. */ | |
| 1016 | all?: boolean; | |
| 1001 | 1017 | }, |
| 1002 | 1018 | ): Html { |
| 1003 | 1019 | const base = scope.base; |
| @@ -1007,7 +1023,7 @@ export function renderBlob( | ||
| 1007 | 1023 | const { content, lines } = |
| 1008 | 1024 | view === "rendered" && o.blob.kind === "text" |
| 1009 | 1025 | ? renderedMarkdown(o.blob) |
| 1010 | : blobBody(base, o.ref, o.path, o.blob, o.highlighted); | |
| 1026 | : blobBody(base, o.ref, o.path, o.blob, o.highlighted, o.all); | |
| 1011 | 1027 | const sizeText = |
| 1012 | 1028 | o.blob.kind === "too_large" ? "" : sizeWithTokens(o.blob.size, o.path); |
| 1013 | 1029 | const fileHref = `${base}/blob/${encoded}/${o.path}`; |
| @@ -1432,6 +1448,7 @@ export async function blobPage(ctx: Ctx): Promise<Response> { | ||
| 1432 | 1448 | blob: capped(blob.value), |
| 1433 | 1449 | view: blobView(ctx.url.searchParams.get("view")), |
| 1434 | 1450 | highlighted: (ms) => (highlightMs = (highlightMs ?? 0) + ms), |
| 1451 | all: ctx.url.searchParams.get("all") === "1", | |
| 1435 | 1452 | }); |
| 1436 | 1453 | if (highlightMs !== undefined) notePhase(ctx.request, "highlight", highlightMs); |
| 1437 | 1454 | const paneMs = performance.now() - renderStart - (highlightMs ?? 0); |
src/web/ui.ts +3 −1
| @@ -172,10 +172,12 @@ export interface BtnOptions { | ||
| 172 | 172 | disabled?: boolean; |
| 173 | 173 | current?: boolean; |
| 174 | 174 | extra?: string; |
| 175 | /** The element's id, for a script or a fragment link that finds it. */ | |
| 176 | id?: string; | |
| 175 | 177 | } |
| 176 | 178 | |
| 177 | 179 | function btnAttrs(o: BtnOptions): Html { |
| 178 | return html`class="btn${o.extra ? ` ${o.extra}` : ""}" | |
| 180 | return html`${o.id ? html`id="${o.id}" ` : null}class="btn${o.extra ? ` ${o.extra}` : ""}" | |
| 179 | 181 | ${o.variant ? html`data-variant="${o.variant}"` : null} |
| 180 | 182 | ${o.size ? html`data-size="${o.size}"` : null} |
| 181 | 183 | ${o.title ? html`title="${o.title}"` : null} |
test/blob-long-file.test.ts added +57
| @@ -0,0 +1,57 @@ | ||
| 1 | /** A long file's page draws its first 1,000 lines and links the rest. */ | |
| 2 | import { describe, expect, test } from "bun:test"; | |
| 3 | import { blobBody } from "../src/web/pages/repo.ts"; | |
| 4 | import { highlightPrefix } from "../src/web/code.ts"; | |
| 5 | import { shellJs, codeJs } from "../src/web/client.ts"; | |
| 6 | import type { Blob } from "../src/view/source.ts"; | |
| 7 | ||
| 8 | const source = Array.from( | |
| 9 | { length: 7_000 }, | |
| 10 | (_, i) => ` const v${i} = compute(${i}); // line ${i}`, | |
| 11 | ).join("\n"); | |
| 12 | const body = new TextEncoder().encode(source); | |
| 13 | const blob: Blob = { kind: "text", body, size: body.length }; | |
| 14 | const draw = (all = false) => | |
| 15 | blobBody("/alice/app", "main", "src/big.ts", blob, undefined, all); | |
| 16 | const rows = (h: string) => h.split('<tr id="L').length - 1; | |
| 17 | ||
| 18 | describe("long file page", () => { | |
| 19 | test("default draws 1,000 of 7,000 lines, small and fast", () => { | |
| 20 | const page = String(draw().content); | |
| 21 | expect(rows(page)).toBe(1_000); | |
| 22 | expect(page).toContain("Showing 1,000 of 7,000 lines"); | |
| 23 | expect(page).toContain("/alice/app/blob/main/src/big.ts?all=1"); | |
| 24 | expect(page).toContain("/alice/app/raw/main/src/big.ts"); | |
| 25 | expect(new TextEncoder().encode(page).length).toBeLessThan(400_000); | |
| 26 | expect(page).toContain('id="code-cut"'); | |
| 27 | }); | |
| 28 | ||
| 29 | test("?all=1 draws every line", () => { | |
| 30 | const page = String(draw(true).content); | |
| 31 | expect(rows(page)).toBe(7_000); | |
| 32 | expect(page).not.toContain("Show all lines"); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a repeat view returns the cached highlight", () => { | |
| 36 | expect(highlightPrefix(source, "src/big.ts", 1_000)).toBe( | |
| 37 | highlightPrefix(source, "src/big.ts", 1_000), | |
| 38 | ); | |
| 39 | }); | |
| 40 | ||
| 41 | test("the prefix highlights like the whole file's first lines", () => { | |
| 42 | const whole = highlightPrefix(source, "src/big.ts", 7_000); | |
| 43 | const part = highlightPrefix(source, "src/big.ts", 1_000); | |
| 44 | expect(part.length).toBe(1_000); | |
| 45 | expect(String(part[999])).toBe(String(whole[999])); | |
| 46 | }); | |
| 47 | ||
| 48 | test("an anchor past the cut is sent to the page that has the line", () => { | |
| 49 | const js = shellJs + codeJs; | |
| 50 | expect(js).toContain("getElementById('code-cut')"); | |
| 51 | expect(js).toContain("location.replace(cut.getAttribute('href') + location.hash)"); | |
| 52 | }); | |
| 53 | ||
| 54 | test("without script the notice says where a later line is", () => { | |
| 55 | expect(String(draw().content)).toContain("lands once you show all lines"); | |
| 56 | }); | |
| 57 | }); |
test/hardening.test.ts +12 −5
| @@ -249,11 +249,18 @@ describe("running-forge attacks", () => { | ||
| 249 | 249 | |
| 250 | 250 | test("blob rendering caps line count (C-1)", async () => { |
| 251 | 251 | const html = await (await fetch(`${base}/t/r/blob/main/many.txt`)).text(); |
| 252 | expect(html).toContain('id="L5000"'); | |
| 253 | expect(html).not.toContain('id="L5001"'); // capped, not 20k rows | |
| 254 | expect(html).toContain("Showing the first"); | |
| 255 | // The 20k-line file must not produce a multi-megabyte page. | |
| 256 | expect(html.length).toBeLessThan(2_000_000); | |
| 252 | expect(html).toContain('id="L1000"'); | |
| 253 | expect(html).not.toContain('id="L1001"'); // the first thousand only | |
| 254 | expect(html).toContain("Showing 1,000 of"); | |
| 255 | expect(html).toContain("?all=1"); | |
| 256 | expect(html.length).toBeLessThan(400_000); | |
| 257 | // Even ?all=1 stops at a ceiling: the 20k-line file is not drawn whole. | |
| 258 | const all = await ( | |
| 259 | await fetch(`${base}/t/r/blob/main/many.txt?all=1`) | |
| 260 | ).text(); | |
| 261 | expect(all).toContain('id="L10000"'); | |
| 262 | expect(all).not.toContain('id="L10001"'); | |
| 263 | expect(all).toContain("Showing 10,000 of"); | |
| 257 | 264 | }); |
| 258 | 265 | |
| 259 | 266 | test("oversized blob is refused by stat, not read into memory (C-2)", async () => { |
new above
offline
Only the script text is tested here. Run this with a stub document that has #code-cut and no #L4200, and check that location.replace gets the ?all=1 URL plus the hash.