Public: anyone can read this; the crew can change it.
A JSON file over 5 MB opens one first-level key on its own, read by byte range #14177
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14031-a-json-file-over-5-mb-opens-one-first-le → main 7 commits fork
12 files
changed +481
−33
docs/features/json-big.md +11 −2
| @@ -20,10 +20,19 @@ children it has. | ||
| 20 | 20 | - **The terminal.** The workspace screen draws the same rows as aligned text, |
| 21 | 21 | read-only, with where the whole file is (`jsonOutlineText`). |
| 22 | 22 | |
| 23 | What it does not do: opening a key on its own is not built. A person who needs a | |
| 24 | key's value downloads the file. | |
| 23 | A key opens on its own too: see "One key on its own" below. | |
| 25 | 24 | |
| 26 | 25 | Code: `src/web/render/json-big.ts` (facts, table, text), `json-doc.ts` |
| 27 | 26 | (`outlineJson`), `kinds.ts` (`JSON_PREFIX_BYTES`), `src/view/local.ts` (the |
| 28 | 27 | prefix read). Tests: `test/render-json-big.test.ts`, `test/json-big-page.test.ts`, |
| 29 | 28 | `test/tui-workspace-formats.test.ts`. |
| 29 | ||
| 30 | ## One key on its own | |
| 31 | ||
| 32 | Each key (an object) or item (an array) in the outline links to `?key=<name>` or `?index=<n>`. The outline scan keeps every member's character span; the member's byte offset and length come from it, and only that range is read (`RepoSource.range`, `blobRange` in `src/git/read.ts`). Git cannot seek inside a blob, so bytes before the member are streamed past and dropped, never held; the read stops at the member's end. A member up to 1 MB (`JSON_VALUE_CAP`) is shown pretty-printed; a bigger one, or one the 4 MB prefix ended inside, shows its own first level with the note on what was not shown. A key or index the outline does not list (including members past the prefix) is a 404 with a plain message. | |
| 33 | ||
| 34 | **What "not read" means.** The member is never held whole in memory or parsed beyond its own bytes, and only the 4 MB prefix is scanned. The I/O is another matter: `blobRange` streams every byte before the member through the `git cat-file` pipe and drops it, because git cannot seek inside a blob. So "the whole file is never read" holds for memory and parsing, not for I/O; a member late in a 20 MB file costs a pass over the bytes before it (`test/json-big-page.test.ts` commits a ~20 MB blob and reads a late member through `localSource.range` in under 500 ms). | |
| 35 | ||
| 36 | **The terminal.** On a big JSON file the workspace screen binds `m` (`openKey` in `src/tui/screens/workspace.ts`): it asks for a key (an item's number for an array) and opens that member through `openMember`, the page's own code, so the view and the words are the same. A key the outline does not list keeps the prompt open with the page's message ("This file has no such key or item in its first level."). `r` returns to the outline. Test: `test/tui-workspace-formats.test.ts`. | |
| 37 | ||
| 38 | **The page.** The member sits in a focusable, horizontally scrolling region (`table-scroll`, `tabindex=0`, labelled by its key), so a long line never widens a phone's page. The path is the design system's `.crumbs` (file / key), the key is the page's heading. A member cut at `JSON_VALUE_CAP` is trimmed to its last whole UTF-8 sequence (`wholeUtf8`). A member's page at a commit id carries the same immutable cache headers as the file's own page (`immutableWhenAddressed`); at a branch name it does not. |
src/git/read.ts +45
| @@ -435,6 +435,51 @@ export async function blobHead( | ||
| 435 | 435 | return head.subarray(0, filled); |
| 436 | 436 | } |
| 437 | 437 | |
| 438 | /** | |
| 439 | * `length` bytes of a blob from byte `start`, or undefined when it cannot be read. Git cannot seek | |
| 440 | * inside a blob, so the bytes before `start` are streamed past and dropped, never held; the read | |
| 441 | * stops (the process is killed) at the end of the range. | |
| 442 | */ | |
| 443 | export async function blobRange( | |
| 444 | dir: string, | |
| 445 | rev: string, | |
| 446 | path: string, | |
| 447 | start: number, | |
| 448 | length: number, | |
| 449 | ): Promise<Uint8Array | undefined> { | |
| 450 | const spec = `${rev}:${path}`; | |
| 451 | assertBatchSafe(spec); | |
| 452 | if (!Number.isInteger(start) || !Number.isInteger(length) || start < 0 || length < 0) return undefined; | |
| 453 | const hold = readerHold(dir); | |
| 454 | if (hold !== undefined && !(await namesOnlyOwn(hold, [rev]))) return undefined; | |
| 455 | let proc: Bun.Subprocess<"ignore", "pipe", "ignore">; | |
| 456 | try { | |
| 457 | proc = Bun.spawn(["git", "cat-file", "blob", spec], { cwd: dir, env: gitEnv(), stdin: "ignore", stdout: "pipe", stderr: "ignore" }); | |
| 458 | } catch { | |
| 459 | return undefined; | |
| 460 | } | |
| 461 | const reader = proc.stdout.getReader(); | |
| 462 | const out = new Uint8Array(length); | |
| 463 | let seen = 0; | |
| 464 | let filled = 0; | |
| 465 | try { | |
| 466 | while (filled < length) { | |
| 467 | const next = await reader.read(); | |
| 468 | if (next.done || next.value === undefined) break; | |
| 469 | const chunk = next.value; | |
| 470 | const skip = Math.max(0, Math.min(chunk.length, start - seen)); | |
| 471 | seen += chunk.length; | |
| 472 | const take = Math.min(chunk.length - skip, length - filled); | |
| 473 | if (take > 0) out.set(chunk.subarray(skip, skip + take), filled); | |
| 474 | filled += Math.max(take, 0); | |
| 475 | } | |
| 476 | } finally { | |
| 477 | proc.kill("SIGKILL"); | |
| 478 | } | |
| 479 | if (filled === 0 && length > 0) return undefined; | |
| 480 | return out.subarray(0, filled); | |
| 481 | } | |
| 482 | ||
| 438 | 483 | function classify(body: Uint8Array, size: number, oid: string): BlobForRender { |
| 439 | 484 | // NUL in the first 8KB is git's own binary heuristic. |
| 440 | 485 | if (body.subarray(0, 8000).includes(0)) |
src/tui/screens/workspace.ts +50 −3
| @@ -50,7 +50,7 @@ import { canAdmin, canWrite, DECISIONS_BY, getRepoById, type Repo } from "../../ | ||
| 50 | 50 | import type { TreeEntry } from "../../git/read.ts"; |
| 51 | 51 | import { parseUnifiedDiff, type Diff } from "../../git/diff.ts"; |
| 52 | 52 | import { renderDiff as renderPatch } from "../ui/diff.ts"; |
| 53 | import { jsonOutlineText } from "../../web/render/json-big.ts"; | |
| 53 | import { hasMember, jsonOutlineText, NO_MEMBER, openMember } from "../../web/render/json-big.ts"; | |
| 54 | 54 | import { fileKind, isEnvExample, isJsonPath, kindByName, NO_BYTES, RENDER_CAP, sniffType } from "../../web/render/kinds.ts"; |
| 55 | 55 | import { tailOf, tailSentence } from "../../web/log_tail.ts"; |
| 56 | 56 | import { downloadHref } from "../../web/pages/media.ts"; |
| @@ -919,6 +919,11 @@ class Tab { | ||
| 919 | 919 | /** A SQLite file's bytes, kept so `>`/`<` and `}`/`{` can page it; null for any other file. */ |
| 920 | 920 | db: Uint8Array | null = null; |
| 921 | 921 | dbAt: SqliteAt = { table: 0, page: 1 }; |
| 922 | /** | |
| 923 | * A big JSON file's prefix and size, kept so `m` can open one of its keys by byte range; null for | |
| 924 | * any other file. | |
| 925 | */ | |
| 926 | big: { readonly prefix: Uint8Array; readonly size: number } | null = null; | |
| 922 | 927 | /** Cues of a subtitle file that could not be read. */ |
| 923 | 928 | unread = 0; |
| 924 | 929 | /** Why a file whose name promises a view is shown as source instead; "" when nothing failed. */ |
| @@ -1220,6 +1225,7 @@ interface CacheEntry { | ||
| 1220 | 1225 | readonly picture: Uint8Array | null; |
| 1221 | 1226 | readonly sheet: Table | null; |
| 1222 | 1227 | readonly db: Uint8Array | null; |
| 1228 | readonly big?: { readonly prefix: Uint8Array; readonly size: number }; | |
| 1223 | 1229 | } |
| 1224 | 1230 | |
| 1225 | 1231 | // ---- what a workspace remembers between visits ---- |
| @@ -1509,6 +1515,7 @@ export function workspaceScreen( | ||
| 1509 | 1515 | let picture: Uint8Array | null = null; |
| 1510 | 1516 | let sheet: Table | null = null; |
| 1511 | 1517 | let db: Uint8Array | null = null; |
| 1518 | let big: { prefix: Uint8Array; size: number } | undefined; | |
| 1512 | 1519 | if (!read.ok) |
| 1513 | 1520 | note = |
| 1514 | 1521 | read.error === "not_found" |
| @@ -1561,12 +1568,13 @@ export function workspaceScreen( | ||
| 1561 | 1568 | // A big JSON file arrives cut: its first level is drawn, as the page draws it. |
| 1562 | 1569 | if (read.value.partial === true && isJsonPath(t.path)) { |
| 1563 | 1570 | raw = new TextEncoder().encode(jsonOutlineText(read.value.body, read.value.size, downloadAt(t.path))); |
| 1564 | cut = "First level of a file too big to open, read-only."; | |
| 1571 | cut = "First level of a file too big to open, read-only; m opens one key."; | |
| 1572 | if (src.range !== undefined) big = { prefix: read.value.body, size: read.value.size }; | |
| 1565 | 1573 | } else if (read.value.partial === true) |
| 1566 | 1574 | cut = `Showing the first ${formatSize(read.value.body.length)} of ${formatSize(read.value.size)}, read-only. The rest is in the raw file.`; |
| 1567 | 1575 | } |
| 1568 | 1576 | } |
| 1569 | entry = { note, noteTitle, noteDetail, cut, raw, picture, sheet, db }; | |
| 1577 | entry = { note, noteTitle, noteDetail, cut, raw, picture, sheet, db, ...(big === undefined ? {} : { big }) }; | |
| 1570 | 1578 | remember(key, entry); |
| 1571 | 1579 | } |
| 1572 | 1580 | if (stale(mine)) return; |
| @@ -1579,6 +1587,7 @@ export function workspaceScreen( | ||
| 1579 | 1587 | if (t.cut !== "") ctx.toast(t.cut, "attn"); |
| 1580 | 1588 | t.sheet = entry.sheet; |
| 1581 | 1589 | t.db = entry.db; |
| 1590 | t.big = entry.big ?? null; | |
| 1582 | 1591 | t.dbAt = { table: 0, page: 1 }; |
| 1583 | 1592 | t.image = |
| 1584 | 1593 | entry.picture === null |
| @@ -2274,6 +2283,7 @@ export function workspaceScreen( | ||
| 2274 | 2283 | }), |
| 2275 | 2284 | ]), |
| 2276 | 2285 | ...(t !== undefined && t.tabular && isTablePath(t.path) ? [binding(["o"], "order rows by a column")] : []), |
| 2286 | ...(t?.big == null ? [] : [binding(["m"], "open one key of the first level")]), | |
| 2277 | 2287 | ...(t?.db == null ? [] : [binding([">", "<", "}", "{"], "next / previous 100 rows, next / previous table")]), |
| 2278 | 2288 | binding(["tab"], "focus explorer"), |
| 2279 | 2289 | binding(["e"], "edit", { disabled: !editable() }), |
| @@ -3084,6 +3094,42 @@ export function workspaceScreen( | ||
| 3084 | 3094 | return true; |
| 3085 | 3095 | } |
| 3086 | 3096 | |
| 3097 | /** | |
| 3098 | * `m` on a big JSON file: name a key (or an item's number) of its first level and it opens on its | |
| 3099 | * own, read by byte range, as the page's `?key=` view does. A name the outline does not list is | |
| 3100 | * told in the page's words and the prompt stays open. | |
| 3101 | */ | |
| 3102 | function openKey(t: Tab): boolean { | |
| 3103 | const big = t.big; | |
| 3104 | const range = src.range; | |
| 3105 | if (big === null || range === undefined) return false; | |
| 3106 | const root = hasMember(big.prefix, { index: 0 }) ? "array" : "object"; | |
| 3107 | const want = (name: string): { key: string } | { index: number } => (root === "array" ? { index: /^\d{1,9}$/.test(name) ? Number(name) : -1 } : { key: name }); | |
| 3108 | ctx.push( | |
| 3109 | promptScreen(ctx, screen, { | |
| 3110 | title: "Open one key", | |
| 3111 | label: root === "array" ? "item number, from 0" : "key", | |
| 3112 | hint: "read by byte range; the rest of the file is not read", | |
| 3113 | onSubmit: (name) => { | |
| 3114 | if (name === "" || !hasMember(big.prefix, want(name))) return NO_MEMBER; | |
| 3115 | return (async () => { | |
| 3116 | const got = await openMember(big.prefix, big.size, want(name), (start, length) => range.call(src, ref, t.path, start, length), downloadAt(t.path)); | |
| 3117 | if (!got.ok) { | |
| 3118 | ctx.toast(got.message, "attn"); | |
| 3119 | return; | |
| 3120 | } | |
| 3121 | t.raw = new TextEncoder().encode(`${got.label} · ${got.summary}\n\n${got.text}`); | |
| 3122 | t.scroll.top = 0; | |
| 3123 | t.left = 0; | |
| 3124 | t.rebuild(ctx.theme.c); | |
| 3125 | ctx.invalidate(); | |
| 3126 | })(); | |
| 3127 | }, | |
| 3128 | }), | |
| 3129 | ); | |
| 3130 | return true; | |
| 3131 | } | |
| 3132 | ||
| 3087 | 3133 | /** Read mode: scroll, search, and the two keys that change what is open. */ |
| 3088 | 3134 | function viewKey(t: Tab, e: KeyEvent): boolean { |
| 3089 | 3135 | if (e.key === "/") { |
| @@ -3092,6 +3138,7 @@ export function workspaceScreen( | ||
| 3092 | 3138 | } |
| 3093 | 3139 | if (e.key === "n" && !e.ctrl) return t.step(1); |
| 3094 | 3140 | if (e.key === "N") return t.step(-1); |
| 3141 | if (e.key === "m" && !e.ctrl && t.big !== null) return openKey(t); | |
| 3095 | 3142 | if (t.db !== null && t.asTable && (e.key === ">" || e.key === "<" || e.key === "}" || e.key === "{")) return pageDb(t, e.key); |
| 3096 | 3143 | // Horizontal scrolling belongs to code, which has long lines and a |
| 3097 | 3144 | // gutter; prose and pictures are already fitted to the pane. |
src/view/local.ts +6
| @@ -35,6 +35,7 @@ import { pathsAtTree } from "../git/path_listing.ts"; | ||
| 35 | 35 | import { |
| 36 | 36 | blobForRender, |
| 37 | 37 | blobHead, |
| 38 | blobRange, | |
| 38 | 39 | blobTail, |
| 39 | 40 | commitLog, |
| 40 | 41 | listRefs, |
| @@ -235,6 +236,11 @@ export function localSource( | ||
| 235 | 236 | return ok({ ...blob, ...(head === undefined ? {} : { head }), ...(tail === undefined ? {} : { tail }) }); |
| 236 | 237 | }, |
| 237 | 238 | |
| 239 | async range(ref: string, path: string, start: number, length: number): Promise<Uint8Array | undefined> { | |
| 240 | if (!isSafeRef(ref) || !isSafeTreePath(path)) return undefined; | |
| 241 | return blobRange(dir, ref, path, start, length); | |
| 242 | }, | |
| 243 | ||
| 238 | 244 | async readme(ref) { |
| 239 | 245 | if (!isSafeRef(ref)) return err("not_found"); |
| 240 | 246 | const entries = await listTree(dir, ref, ""); |
src/view/source.ts +2
| @@ -266,6 +266,8 @@ export interface RepoSource { | ||
| 266 | 266 | /** Every file path under `ref`, flat, for a fuzzy finder. Capped and cached. */ |
| 267 | 267 | paths(ref: string): Promise<Result<PathListing, SourceError>>; |
| 268 | 268 | blob(ref: string, path: string, opts?: { readonly prefix?: boolean }): Promise<Result<Blob, SourceError>>; |
| 269 | /** `length` bytes of a file from byte `start`, read by range; absent where a source cannot. */ | |
| 270 | range?(ref: string, path: string, start: number, length: number): Promise<Uint8Array | undefined>; | |
| 269 | 271 | readme( |
| 270 | 272 | ref: string, |
| 271 | 273 | ): Promise<Result<{ path: string; markdown: string } | null, SourceError>>; |
src/web/pages/repo.ts +51
| @@ -78,6 +78,7 @@ import { | ||
| 78 | 78 | repoHeader, |
| 79 | 79 | scopeHeader, |
| 80 | 80 | signInFirst, |
| 81 | pageHead, | |
| 81 | 82 | } from "../chrome.ts"; |
| 82 | 83 | import { codeJs, liveRegionsJs } from "../client.ts"; |
| 83 | 84 | import { countLines, highlightPrefix } from "../code.ts"; |
| @@ -104,6 +105,7 @@ import { fontJs } from "../render/font.ts"; | ||
| 104 | 105 | import { INLINE_MAX, keepGcode, keptGcode, layerCount } from "../render/gcode.ts"; |
| 105 | 106 | import { hasTable } from "../render/sqlite.ts"; |
| 106 | 107 | import { READ_CEILING } from "../render/psd-read.ts"; |
| 108 | import { NO_MEMBER, openMember } from "../render/json-big.ts"; | |
| 107 | 109 | import { CSV_PREFIX_BYTES, fileKind, HEAD_BYTES, isJsonPath, isTablePath, JSON_PREFIX_BYTES, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts"; |
| 108 | 110 | import { RENDERERS, rendererFor } from "../render/renderers.ts"; |
| 109 | 111 | import { downloadHref, mediaHref } from "./media.ts"; |
| @@ -1809,6 +1811,48 @@ export async function treePage(ctx: Ctx): Promise<Response> { | ||
| 1809 | 1811 | return immutableWhenAddressed(ctx.request, anonymous, repo, ref, response); |
| 1810 | 1812 | } |
| 1811 | 1813 | |
| 1814 | /** | |
| 1815 | * A big JSON file's one member on its own page, read by byte range. Undefined when the file is not | |
| 1816 | * a big one (it opens as usual). A key or index the outline does not hold is a 404 that says so. | |
| 1817 | */ | |
| 1818 | async function jsonMemberPage( | |
| 1819 | ctx: Ctx, | |
| 1820 | scope: ReturnType<typeof localScope>, | |
| 1821 | repo: Repo, | |
| 1822 | ref: string, | |
| 1823 | path: string, | |
| 1824 | keyText: string | null, | |
| 1825 | indexText: string | null, | |
| 1826 | ): Promise<Response | undefined> { | |
| 1827 | const range = scope.source.range; | |
| 1828 | if (range === undefined || !isSafeRef(ref) || !isSafeTreePath(path)) return undefined; | |
| 1829 | const blob = await scope.source.blob(ref, path, { prefix: true }); | |
| 1830 | if (!blob.ok || blob.value.kind !== "text" || blob.value.partial !== true || blob.value.body.length > JSON_PREFIX_BYTES) return undefined; | |
| 1831 | const key = keyText !== null && keyText.length <= 300 ? keyText : undefined; | |
| 1832 | const index = indexText !== null && /^\d{1,9}$/.test(indexText) ? Number(indexText) : undefined; | |
| 1833 | const filename = path.split("/").pop() ?? path; | |
| 1834 | const download = `${repoBase(repo)}/raw/${ref}/${path.split("/").map(encodeURIComponent).join("/")}`; | |
| 1835 | const member = | |
| 1836 | key === undefined && index === undefined | |
| 1837 | ? ({ ok: false, message: NO_MEMBER } as const) | |
| 1838 | : await openMember(blob.value.body, blob.value.size, { ...(key === undefined ? {} : { key }), ...(index === undefined ? {} : { index }) }, (s, n) => range.call(scope.source, ref, path, s, n), download); | |
| 1839 | const explorer = await loadExplorer(scope.source, ref, path, true); | |
| 1840 | const back = html`<a href="${repoBase(repo)}/blob/${ref}/${path.split("/").map(encodeURIComponent).join("/")}">${filename}</a>`; | |
| 1841 | const crumbs = html`<nav class="crumbs mono" aria-label="Where this is"><span>${back}</span><span class="sep" aria-hidden="true">/</span><span aria-current="page">${member.ok ? member.label : "?"}</span></nav>`; | |
| 1842 | const pane = member.ok | |
| 1843 | ? html`${pageHead(html`<span class="mono">${member.label}</span>`, { lead: crumbs, sub: member.summary })}${member.content}` | |
| 1844 | : html`${pageHead(back, { sub: member.message })}`; | |
| 1845 | const response = codePage(ctx, scope, { | |
| 1846 | title: `${member.ok ? member.label : "Not found"} · ${filename} · ${repo.name}`, | |
| 1847 | ref, | |
| 1848 | explorer: explorerColumn(scope, explorer, { finder: true, actions: html`` }), | |
| 1849 | content: pane, | |
| 1850 | }); | |
| 1851 | if (!member.ok) return new Response(response.body, { status: 404, headers: response.headers }); | |
| 1852 | // A commit id never changes what it holds: the same cache headers the file's own page gets. | |
| 1853 | return immutableWhenAddressed(ctx.request, ctx.auth.user === undefined, repo, ref, response); | |
| 1854 | } | |
| 1855 | ||
| 1812 | 1856 | // GET /:owner/:name/blob/:ref/*path |
| 1813 | 1857 | export async function blobPage(ctx: Ctx): Promise<Response> { |
| 1814 | 1858 | const repo = loadRepo(ctx); |
| @@ -1831,6 +1875,13 @@ export async function blobPage(ctx: Ctx): Promise<Response> { | ||
| 1831 | 1875 | const page = pageText !== null && /^\d{1,6}$/.test(pageText) && Number(pageText) > 1 ? Number(pageText) : undefined; |
| 1832 | 1876 | const layerText = ctx.url.searchParams.get("layer"); |
| 1833 | 1877 | const layer = layerText !== null && /^\d{1,6}$/.test(layerText) ? Number(layerText) : undefined; |
| 1878 | // One first-level member of a big JSON file, by `?key=` or `?index=`. | |
| 1879 | const keyText = ctx.url.searchParams.get("key"); | |
| 1880 | const indexText = ctx.url.searchParams.get("index"); | |
| 1881 | if ((keyText !== null || indexText !== null) && isJsonPath(path)) { | |
| 1882 | const member = await jsonMemberPage(ctx, scope, repo, ref, path, keyText, indexText); | |
| 1883 | if (member !== undefined) return member; | |
| 1884 | } | |
| 1834 | 1885 | // The whole page as it last went out to a stranger, encoded (`kept_page.ts`). |
| 1835 | 1886 | const keptKey = kept ? keptPageKey(ctx, repo.id) : undefined; |
| 1836 | 1887 | const finished = keptKey === undefined ? undefined : keptPage(keptKey); |
src/web/render/json-big.ts +138 −17
| @@ -6,9 +6,9 @@ | ||
| 6 | 6 | * aligned text (`jsonOutlineText`); both read the same facts (`outlineFacts`). |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { html } from "../html.ts"; | |
| 9 | import { html, type Html } from "../html.ts"; | |
| 10 | 10 | import { fileSize } from "../ui.ts"; |
| 11 | import { outlineJson } from "./json-doc.ts"; | |
| 11 | import { type Outline, outlineJson, parseJson, stringify } from "./json-doc.ts"; | |
| 12 | 12 | import type { Rendered, RenderInput } from "./renderers.ts"; |
| 13 | 13 | |
| 14 | 14 | /** A fixed locale, so a count reads the same whatever the server's. */ |
| @@ -18,6 +18,8 @@ export interface OutlineRow { | ||
| 18 | 18 | readonly key: string; |
| 19 | 19 | readonly type: string; |
| 20 | 20 | readonly children: string; |
| 21 | /** Where this member opens on its own, relative to the file's page; absent when it cannot. */ | |
| 22 | readonly href?: string; | |
| 21 | 23 | } |
| 22 | 24 | |
| 23 | 25 | export interface OutlineFacts { |
| @@ -30,21 +32,26 @@ export interface OutlineFacts { | ||
| 30 | 32 | /** The outline of a prefix as plain data, or the sentence saying it has none. */ |
| 31 | 33 | export function outlineFacts(body: Uint8Array, size: number): OutlineFacts | string { |
| 32 | 34 | const outline = outlineJson(new TextDecoder().decode(body)); |
| 33 | const read = fileSize(body.length); | |
| 34 | 35 | const whole = fileSize(size); |
| 35 | 36 | if (outline === undefined) return `This file is ${whole}, too large to open as a form, and it does not open as an object or an array.`; |
| 37 | return describe(outline, `Read the first ${fileSize(body.length)} of ${whole}; the file was not parsed.`, whole, true); | |
| 38 | } | |
| 39 | ||
| 40 | /** An outline as plain data; `opens` makes each member a link to its own view. */ | |
| 41 | function describe(outline: Outline, lead: string, whole: string, opens: boolean): OutlineFacts { | |
| 36 | 42 | const object = outline.root === "object"; |
| 37 | 43 | const noun = object ? "keys" : "items"; |
| 38 | const said: string[] = [`Read the first ${read} of ${whole}; the file was not parsed.`]; | |
| 44 | const said: string[] = [lead]; | |
| 39 | 45 | if (outline.limited) said.push(`Showing the first ${count(outline.entries.length)} ${noun}; the rest are not listed.`); |
| 40 | 46 | else if (!outline.complete) said.push(`The ${noun} after these, and the end of the file, were past what was read and are not shown.`); |
| 41 | 47 | if (outline.entries.some((e) => e.atLeast === true)) said.push("A count marked “at least” is of a value the read cut short."); |
| 42 | 48 | return { |
| 43 | 49 | heading: object ? "Key" : "Item", |
| 44 | rows: outline.entries.map((e) => ({ | |
| 50 | rows: outline.entries.map((e, n) => ({ | |
| 45 | 51 | key: e.key, |
| 46 | 52 | type: e.type, |
| 47 | 53 | children: e.count === undefined ? "" : `${e.atLeast === true ? "at least " : ""}${count(e.count)}`, |
| 54 | ...(opens && (!object || !e.key.endsWith("…")) ? { href: object ? `?key=${encodeURIComponent(e.key)}` : `?index=${n}` } : {}), | |
| 48 | 55 | })), |
| 49 | 56 | said: said.join(" "), |
| 50 | 57 | summary: `${count(outline.entries.length)}${outline.complete ? "" : "+"} ${noun} · ${object ? "JSON object" : "JSON array"} · ${whole}`, |
| @@ -54,24 +61,131 @@ export function outlineFacts(body: Uint8Array, size: number): OutlineFacts | str | ||
| 54 | 61 | export function jsonOutline(input: RenderInput): Rendered { |
| 55 | 62 | const facts = outlineFacts(input.body, input.size); |
| 56 | 63 | if (typeof facts === "string") return { lines: 0, content: html``, fallback: facts }; |
| 57 | const rows = facts.rows.map( | |
| 58 | (r) => html`<tr><td class="mono">${r.key}</td><td>${r.type}</td><td class="num">${r.children}</td></tr>`, | |
| 59 | ); | |
| 60 | 64 | return { |
| 61 | 65 | lines: 0, |
| 62 | 66 | summary: facts.summary, |
| 63 | content: html`<table class="table" data-dense aria-label="First level of ${input.path.slice(input.path.lastIndexOf("/") + 1)}"> | |
| 64 | <thead><tr><th>${facts.heading}</th><th>Type</th><th class="num">Children</th></tr></thead> | |
| 65 | <tbody>${rows}</tbody> | |
| 66 | </table> | |
| 67 | content: html`${outlineTable(facts, `First level of ${input.path.slice(input.path.lastIndexOf("/") + 1)}`)} | |
| 67 | 68 | <p class="muted small">${facts.said} <a href="${input.download}">Download the whole file.</a></p>`, |
| 68 | 69 | }; |
| 69 | 70 | } |
| 70 | 71 | |
| 71 | /** The same outline as aligned text for the terminal, with where the whole file is. */ | |
| 72 | export function jsonOutlineText(body: Uint8Array, size: number, where: string): string { | |
| 73 | const facts = outlineFacts(body, size); | |
| 74 | if (typeof facts === "string") return `${facts}\n${where}\n`; | |
| 72 | function outlineTable(facts: OutlineFacts, label: string): Html { | |
| 73 | const rows = facts.rows.map((r) => { | |
| 74 | const key = r.href === undefined ? html`${r.key}` : html`<a href="${r.href}">${r.key}</a>`; | |
| 75 | return html`<tr><td class="mono">${key}</td><td>${r.type}</td><td class="num">${r.children}</td></tr>`; | |
| 76 | }); | |
| 77 | return html`<table class="table" data-dense aria-label="${label}"> | |
| 78 | <thead><tr><th>${facts.heading}</th><th>Type</th><th class="num">Children</th></tr></thead> | |
| 79 | <tbody>${rows}</tbody> | |
| 80 | </table>`; | |
| 81 | } | |
| 82 | ||
| 83 | /** | |
| 84 | * A member shown whole is pretty-printed up to this many bytes; a bigger one shows its first level. | |
| 85 | */ | |
| 86 | export const JSON_VALUE_CAP = 1_000_000; | |
| 87 | ||
| 88 | /** `length` bytes of the file from byte `start`: the only way the member is read. */ | |
| 89 | export type ByteRange = (start: number, length: number) => Promise<Uint8Array | undefined>; | |
| 90 | ||
| 91 | /** `text` is the same member as plain lines, for the terminal; `content` is the page's. */ | |
| 92 | export type Member = | |
| 93 | | { readonly ok: true; readonly label: string; readonly summary: string; readonly content: Html; readonly text: string } | |
| 94 | | { readonly ok: false; readonly message: string }; | |
| 95 | ||
| 96 | /** What a key or item the outline does not list is told, on the page and in the terminal alike. */ | |
| 97 | export const NO_MEMBER = "This file has no such key or item in its first level."; | |
| 98 | ||
| 99 | /** | |
| 100 | * `bytes` without a trailing, incomplete UTF-8 sequence (a read cut at a byte count can end inside | |
| 101 | * one). | |
| 102 | */ | |
| 103 | export function wholeUtf8(bytes: Uint8Array): Uint8Array { | |
| 104 | for (let back = 1; back <= Math.min(3, bytes.length); back++) { | |
| 105 | const b = bytes[bytes.length - back]!; | |
| 106 | if ((b & 0xc0) === 0x80) continue; // a continuation byte: look further back for its lead | |
| 107 | const need = b >= 0xf0 ? 4 : b >= 0xe0 ? 3 : b >= 0xc0 ? 2 : 1; | |
| 108 | return need > back ? bytes.subarray(0, bytes.length - back) : bytes; | |
| 109 | } | |
| 110 | return bytes; | |
| 111 | } | |
| 112 | ||
| 113 | type Want = { readonly key?: string; readonly index?: number }; | |
| 114 | ||
| 115 | function locate(outline: Outline, want: Want): number { | |
| 116 | return want.key !== undefined && outline.root === "object" | |
| 117 | ? outline.entries.findIndex((e) => e.key === want.key) | |
| 118 | : want.index !== undefined && outline.root === "array" && want.index < outline.entries.length | |
| 119 | ? want.index | |
| 120 | : -1; | |
| 121 | } | |
| 122 | ||
| 123 | /** Whether the outline of `prefix` lists this key or item: the check before any byte is read. */ | |
| 124 | export function hasMember(prefix: Uint8Array, want: Want): boolean { | |
| 125 | const outline = outlineJson(new TextDecoder("utf-8", { ignoreBOM: true }).decode(prefix)); | |
| 126 | return outline !== undefined && locate(outline, want) >= 0 && outline.spans[locate(outline, want)] !== undefined; | |
| 127 | } | |
| 128 | ||
| 129 | /** | |
| 130 | * One first-level member of a big document on its own. The member is found in the outline of | |
| 131 | * `prefix` (the same scan the outline page makes), its byte offset and length kept, and only that | |
| 132 | * range is read through `read`. A member in the outline is openable; a key not in it is refused. | |
| 133 | * A member over `JSON_VALUE_CAP`, or one the prefix ended inside, shows its own first level. | |
| 134 | */ | |
| 135 | export async function openMember( | |
| 136 | prefix: Uint8Array, | |
| 137 | size: number, | |
| 138 | want: Want, | |
| 139 | read: ByteRange, | |
| 140 | download: string, | |
| 141 | ): Promise<Member> { | |
| 142 | const text = new TextDecoder("utf-8", { ignoreBOM: true }).decode(prefix); | |
| 143 | const outline = outlineJson(text); | |
| 144 | const missing: Member = { ok: false, message: NO_MEMBER }; | |
| 145 | if (outline === undefined) return missing; | |
| 146 | const at = locate(outline, want); | |
| 147 | const entry = outline.entries[at]; | |
| 148 | const span = outline.spans[at]; | |
| 149 | if (entry === undefined || span === undefined) return missing; | |
| 150 | const enc = new TextEncoder(); | |
| 151 | const start = enc.encode(text.slice(0, span.start)).length; | |
| 152 | const known = span.end === undefined ? undefined : enc.encode(text.slice(span.start, span.end)).length; | |
| 153 | const wanted = Math.min(known ?? Number.POSITIVE_INFINITY, JSON_VALUE_CAP, size - start); | |
| 154 | const bytes = await read(start, wanted); | |
| 155 | if (bytes === undefined) return { ok: false, message: "This member could not be read." }; | |
| 156 | const body = new TextDecoder().decode(wholeUtf8(bytes)); | |
| 157 | const label = outline.root === "object" ? entry.key : `item ${at}`; | |
| 158 | const whole = known !== undefined && known <= JSON_VALUE_CAP; | |
| 159 | const link = html`<a href="${download}">Download the whole file.</a>`; | |
| 160 | if (whole) { | |
| 161 | const parsed = parseJson(body); | |
| 162 | const shown = parsed.ok ? stringify(parsed.value, { indent: " ", eol: "\n", finalNewline: false }) : body; | |
| 163 | return { | |
| 164 | ok: true, | |
| 165 | label, | |
| 166 | summary: `${entry.type} · ${fileSize(known)}`, | |
| 167 | text: `${shown}\n`, | |
| 168 | content: html`<div class="table-scroll" role="region" tabindex="0" aria-label="${label}"><pre class="mono">${shown}</pre></div><p class="muted small">Read ${fileSize(known)} of ${fileSize(size)}, by byte range; the rest of the file was not read. ${link}</p>`, | |
| 169 | }; | |
| 170 | } | |
| 171 | const inner = outlineJson(body); | |
| 172 | const lead = `Read the first ${fileSize(bytes.length)} of this ${known === undefined ? "member" : fileSize(known) + " member"} of ${fileSize(size)}; it was not parsed, and nothing past it was read.`; | |
| 173 | if (inner === undefined) { | |
| 174 | const said = `${lead} This ${entry.type} is over ${fileSize(JSON_VALUE_CAP)} and does not open as an object or an array, so none of it is shown.`; | |
| 175 | return { ok: true, label, summary: entry.type, text: `${said}\n`, content: html`<p class="muted">${said} ${link}</p>` }; | |
| 176 | } | |
| 177 | const facts = describe(inner, lead, fileSize(size), false); | |
| 178 | return { | |
| 179 | ok: true, | |
| 180 | label, | |
| 181 | summary: `${facts.summary.split(" · ")[0]} · ${entry.type}`, | |
| 182 | text: factsText(facts, ""), | |
| 183 | content: html`${outlineTable(facts, `First level of ${label}`)}<p class="muted small">${facts.said} ${link}</p>`, | |
| 184 | }; | |
| 185 | } | |
| 186 | ||
| 187 | /** Facts as aligned text, with a closing line saying where the whole file is. */ | |
| 188 | function factsText(facts: OutlineFacts, where: string): string { | |
| 75 | 189 | const w = (pick: (r: OutlineRow) => string, head: string): number => Math.max(head.length, ...facts.rows.map((r) => pick(r).length)); |
| 76 | 190 | const kw = w((r) => r.key, facts.heading); |
| 77 | 191 | const tw = w((r) => r.type, "Type"); |
| @@ -82,7 +196,14 @@ export function jsonOutlineText(body: Uint8Array, size: number, where: string): | ||
| 82 | 196 | line(facts.heading, "Type", "Children"), |
| 83 | 197 | ...facts.rows.map((r) => line(r.key, r.type, r.children)), |
| 84 | 198 | "", |
| 85 | `${facts.said} ${where}`, | |
| 199 | `${facts.said} ${where}`.trimEnd(), | |
| 86 | 200 | "", |
| 87 | 201 | ].join("\n"); |
| 88 | 202 | } |
| 203 | ||
| 204 | /** The same outline as aligned text for the terminal, with where the whole file is. */ | |
| 205 | export function jsonOutlineText(body: Uint8Array, size: number, where: string): string { | |
| 206 | const facts = outlineFacts(body, size); | |
| 207 | if (typeof facts === "string") return `${facts}\n${where}\n`; | |
| 208 | return factsText(facts, where); | |
| 209 | } |
src/web/render/json-doc.ts +17 −9
| @@ -361,6 +361,11 @@ export interface OutlineEntry { | ||
| 361 | 361 | export interface Outline { |
| 362 | 362 | readonly root: "object" | "array"; |
| 363 | 363 | readonly entries: readonly OutlineEntry[]; |
| 364 | /** | |
| 365 | * Where each entry's value sits in the text, as character offsets; `end` is absent when the text | |
| 366 | * ended inside it. | |
| 367 | */ | |
| 368 | readonly spans: readonly { readonly start: number; readonly end?: number }[]; | |
| 364 | 369 | /** Every member of the first level was listed, and the document ended. */ |
| 365 | 370 | readonly complete: boolean; |
| 366 | 371 | /** Listing stopped at `limit` members, not at the end of the text. */ |
| @@ -403,6 +408,7 @@ export function outlineJson(text: string, limit = 500): Outline | undefined { | ||
| 403 | 408 | const root = open === "{" ? "object" : "array"; |
| 404 | 409 | i++; |
| 405 | 410 | const entries: OutlineEntry[] = []; |
| 411 | const spans: { start: number; end?: number }[] = []; | |
| 406 | 412 | /** Skips the value at `i`; its type, child count and whether the text ended inside it. */ |
| 407 | 413 | const value = (): { t: Json["t"]; count?: number; cut: boolean } => { |
| 408 | 414 | const c = text[i]; |
| @@ -441,14 +447,14 @@ export function outlineJson(text: string, limit = 500): Outline | undefined { | ||
| 441 | 447 | const closer = root === "object" ? "}" : "]"; |
| 442 | 448 | for (;;) { |
| 443 | 449 | ws(); |
| 444 | if (i >= n) return { root, entries, complete: false, limited: false }; | |
| 445 | if (text[i] === closer) return { root, entries, complete: true, limited: false }; | |
| 446 | if (entries.length >= limit) return { root, entries, complete: false, limited: true }; | |
| 450 | if (i >= n) return { root, entries, spans, complete: false, limited: false }; | |
| 451 | if (text[i] === closer) return { root, entries, spans, complete: true, limited: false }; | |
| 452 | if (entries.length >= limit) return { root, entries, spans, complete: false, limited: true }; | |
| 447 | 453 | let key = `[${entries.length}]`; |
| 448 | 454 | if (root === "object") { |
| 449 | if (text[i] !== '"') return { root, entries, complete: false, limited: false }; | |
| 455 | if (text[i] !== '"') return { root, entries, spans, complete: false, limited: false }; | |
| 450 | 456 | const end = skipString(); |
| 451 | if (end > n) return { root, entries, complete: false, limited: false }; | |
| 457 | if (end > n) return { root, entries, spans, complete: false, limited: false }; | |
| 452 | 458 | const raw = text.slice(i, end); |
| 453 | 459 | try { |
| 454 | 460 | key = raw.length > 300 ? `${raw.slice(1, 200)}…` : String(JSON.parse(raw)); |
| @@ -457,15 +463,17 @@ export function outlineJson(text: string, limit = 500): Outline | undefined { | ||
| 457 | 463 | } |
| 458 | 464 | i = end; |
| 459 | 465 | ws(); |
| 460 | if (text[i] !== ":") return { root, entries, complete: false, limited: false }; | |
| 466 | if (text[i] !== ":") return { root, entries, spans, complete: false, limited: false }; | |
| 461 | 467 | i++; |
| 462 | 468 | ws(); |
| 463 | if (i >= n) return { root, entries, complete: false, limited: false }; | |
| 469 | if (i >= n) return { root, entries, spans, complete: false, limited: false }; | |
| 464 | 470 | } |
| 471 | const start = i; | |
| 465 | 472 | const v = value(); |
| 466 | if (v.cut && v.t !== "object" && v.t !== "array") return { root, entries, complete: false, limited: false }; | |
| 473 | if (v.cut && v.t !== "object" && v.t !== "array") return { root, entries, spans, complete: false, limited: false }; | |
| 467 | 474 | entries.push({ key, type: v.t, ...(v.count === undefined ? {} : { count: v.count }), ...(v.cut ? { atLeast: true } : {}) }); |
| 468 | if (v.cut) return { root, entries, complete: false, limited: false }; | |
| 475 | spans.push(v.cut ? { start } : { start, end: i }); | |
| 476 | if (v.cut) return { root, entries, spans, complete: false, limited: false }; | |
| 469 | 477 | ws(); |
| 470 | 478 | if (text[i] === ",") i++; |
| 471 | 479 | } |
test/json-big-page.test.ts +77
| @@ -7,6 +7,8 @@ import { join } from "node:path"; | ||
| 7 | 7 | import { createApp, type App } from "../src/app.ts"; |
| 8 | 8 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 9 | 9 | import { repoDir } from "../src/git/repos.ts"; |
| 10 | import { localSource } from "../src/view/local.ts"; | |
| 11 | import { getRepo } from "../src/core/repos.ts"; | |
| 10 | 12 | import { serve, type Servers } from "../src/http/server.ts"; |
| 11 | 13 | import { seedAdmin } from "./helpers/admin.ts"; |
| 12 | 14 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -68,4 +70,79 @@ describe("a big JSON file on the repository page", () => { | ||
| 68 | 70 | expect(text.includes(">zlast<")).toBe(false); |
| 69 | 71 | expect(text.includes("json-big")).toBe(false); |
| 70 | 72 | }); |
| 73 | ||
| 74 | test("a key opens on its own, read by byte range", async () => { | |
| 75 | const res = await fetch(`${servers.url}/alice/bigjson/blob/main/data.json?key=small`); | |
| 76 | const text = await res.text(); | |
| 77 | expect(res.status).toBe(200); | |
| 78 | expect(text.includes(""a": 1")).toBe(true); | |
| 79 | expect(text.includes("the rest of the file was not read")).toBe(true); | |
| 80 | expect(text.includes(">alpha<")).toBe(false); | |
| 81 | }); | |
| 82 | ||
| 83 | test("a member over the render cap shows its own first level", async () => { | |
| 84 | const res = await fetch(`${servers.url}/alice/bigjson/blob/main/data.json?key=alpha`); | |
| 85 | const text = await res.text(); | |
| 86 | expect(res.status).toBe(200); | |
| 87 | expect(text.includes("First level of alpha")).toBe(true); | |
| 88 | expect(text.includes("it was not parsed")).toBe(true); | |
| 89 | }); | |
| 90 | ||
| 91 | test("a key not in the outline is a 404 with a plain message, escaped", async () => { | |
| 92 | for (const key of ["nope", "zlast", "%3Cscript%3E"]) { | |
| 93 | const res = await fetch(`${servers.url}/alice/bigjson/blob/main/data.json?key=${key}`); | |
| 94 | const text = await res.text(); | |
| 95 | expect(res.status).toBe(404); | |
| 96 | expect(text.includes("no such key or item")).toBe(true); | |
| 97 | expect(text.includes("<script>")).toBe(false); | |
| 98 | } | |
| 99 | }); | |
| 100 | ||
| 101 | test("a member's page is cached for good at a commit id, and not at a branch name", async () => { | |
| 102 | const dir = repoDir(app.reposRoot, "alice", "bigjson"); | |
| 103 | const sha = Bun.spawnSync(["git", "rev-parse", "main"], { cwd: dir }).stdout.toString().trim(); | |
| 104 | const at = await fetch(`${servers.url}/alice/bigjson/blob/${sha}/data.json?key=small`); | |
| 105 | expect(at.headers.get("cache-control") ?? "").toContain("immutable"); | |
| 106 | const named = await fetch(`${servers.url}/alice/bigjson/blob/main/data.json?key=small`); | |
| 107 | expect(named.headers.get("cache-control") ?? "").not.toContain("immutable"); | |
| 108 | await at.text(); | |
| 109 | await named.text(); | |
| 110 | }); | |
| 111 | ||
| 112 | test("a long member scrolls inside a focusable region, and the path is a breadcrumb", async () => { | |
| 113 | const text = await (await fetch(`${servers.url}/alice/bigjson/blob/main/data.json?key=small`)).text(); | |
| 114 | expect(text.includes('class="table-scroll" role="region" tabindex="0"')).toBe(true); | |
| 115 | expect(text.includes('class="crumbs mono"')).toBe(true); | |
| 116 | expect(text.includes("›")).toBe(false); | |
| 117 | }); | |
| 118 | ||
| 119 | test("a real 20 MB blob: a late member is read through localSource.range, under 500 ms, equal to its bytes", async () => { | |
| 120 | const dir = repoDir(app.reposRoot, "alice", "bigjson"); | |
| 121 | const git = (args: string[], stdin: string): string => { | |
| 122 | const run = Bun.spawnSync(["git", ...args], { | |
| 123 | cwd: dir, | |
| 124 | stdin: new TextEncoder().encode(stdin), | |
| 125 | env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 126 | }); | |
| 127 | if (run.exitCode !== 0) throw new Error(run.stderr.toString()); | |
| 128 | return run.stdout.toString().trim(); | |
| 129 | }; | |
| 130 | const member = '{"late":"found","n":[1,2,3]}'; | |
| 131 | const doc = `{"pad":${list(350_000)},"late":${member}}`; | |
| 132 | expect(doc.length).toBeGreaterThan(20_000_000); | |
| 133 | const blob = git(["hash-object", "-w", "--stdin"], doc); | |
| 134 | const sha = Bun.spawnSync(["git", "rev-parse", "main"], { cwd: dir }).stdout.toString().trim(); | |
| 135 | const tree = git(["mktree"], `100644 blob ${blob}\thuge.json\n`); | |
| 136 | const commit = git(["commit-tree", tree, "-m", "huge"], ""); | |
| 137 | expect(sha.length).toBeGreaterThan(0); | |
| 138 | const repo = getRepo(app.db, "alice", "bigjson"); | |
| 139 | if (repo === undefined) throw new Error("no repo"); | |
| 140 | const source = localSource(app, repo); | |
| 141 | const start = new TextEncoder().encode(doc.slice(0, doc.indexOf('"late":{') + 7)).length; | |
| 142 | const t0 = performance.now(); | |
| 143 | const got = await source.range?.(commit, "huge.json", start, member.length); | |
| 144 | const ms = performance.now() - t0; | |
| 145 | expect(new TextDecoder().decode(got)).toBe(member); | |
| 146 | expect(ms).toBeLessThan(500); | |
| 147 | }); | |
| 71 | 148 | }); |
test/render-json-big.test.ts +71
| @@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test"; | ||
| 3 | 3 | import { outlineJson } from "../src/web/render/json-doc.ts"; |
| 4 | 4 | import { JSON_PREFIX_BYTES } from "../src/web/render/kinds.ts"; |
| 5 | 5 | import { config } from "../src/web/render/config.ts"; |
| 6 | import { JSON_VALUE_CAP, openMember, wholeUtf8 } from "../src/web/render/json-big.ts"; | |
| 6 | 7 | import { rendererFor } from "../src/web/render/renderers.ts"; |
| 7 | 8 | |
| 8 | 9 | const enc = (s: string): Uint8Array => new TextEncoder().encode(s); |
| @@ -89,3 +90,73 @@ describe("a small cut document", () => { | ||
| 89 | 90 | expect(r.content.toString()).not.toContain("<script>"); |
| 90 | 91 | }); |
| 91 | 92 | }); |
| 93 | ||
| 94 | describe("one member of a 20 MB document", () => { | |
| 95 | const text = '{"meta":{"name":"<b>x</b>","n":[1,2],"é":"ü"},' + generated(20).slice(1); | |
| 96 | const whole = enc(text); | |
| 97 | const prefix = whole.subarray(0, JSON_PREFIX_BYTES); | |
| 98 | /** A byte-range reader over the document that remembers every range it was asked for. */ | |
| 99 | const reader = () => { | |
| 100 | const asked: [number, number][] = []; | |
| 101 | return { asked, read: async (start: number, length: number) => (asked.push([start, length]), whole.slice(start, start + length)) }; | |
| 102 | }; | |
| 103 | const open = (want: { key?: string; index?: number }, r = reader()) => openMember(prefix, whole.length, want, r.read, "/d").then((m) => ({ m, r })); | |
| 104 | ||
| 105 | test("the outline links each key to its own view", () => { | |
| 106 | const out = draw(prefix, whole.length).content.toString(); | |
| 107 | expect(out).toContain('<a href="?key=meta">meta</a>'); | |
| 108 | expect(out).toContain('<a href="?key=alpha">alpha</a>'); | |
| 109 | }); | |
| 110 | test("a small member is read by its own byte range, inside a time budget, and nothing else is read", async () => { | |
| 111 | const at = text.indexOf('{"name"'); | |
| 112 | const end = text.indexOf("}", at) + 1; | |
| 113 | const t = performance.now(); | |
| 114 | const { m, r } = await open({ key: "meta" }); | |
| 115 | expect(performance.now() - t).toBeLessThan(500); | |
| 116 | expect(m.ok).toBe(true); | |
| 117 | expect(r.asked.length).toBe(1); | |
| 118 | const [start, length] = r.asked[0]!; | |
| 119 | expect(start).toBe(enc(text.slice(0, at)).length); | |
| 120 | expect(length).toBe(enc(text.slice(at, end)).length); | |
| 121 | if (!m.ok) throw new Error("unreachable"); | |
| 122 | const out = m.content.toString(); | |
| 123 | expect(out).toContain("<b>x</b>"); | |
| 124 | expect(out).not.toContain("<b>"); | |
| 125 | expect(out).toContain("ü"); | |
| 126 | }); | |
| 127 | test("a member over the render cap shows its first level, and reads no more than the cap from inside it", async () => { | |
| 128 | const { m, r } = await open({ key: "alpha" }); | |
| 129 | if (!m.ok) throw new Error("unreachable"); | |
| 130 | const [start, length] = r.asked[0]!; | |
| 131 | expect(length).toBeLessThanOrEqual(JSON_VALUE_CAP); | |
| 132 | expect(start).toBe(enc(text.slice(0, text.indexOf('[{"id"'))).length); | |
| 133 | const out = m.content.toString(); | |
| 134 | expect(out).toContain("<table"); | |
| 135 | expect(out).toContain("it was not parsed"); | |
| 136 | expect(out).toContain("the rest are not listed"); | |
| 137 | }); | |
| 138 | test("a key the outline does not hold is refused, and nothing is read", async () => { | |
| 139 | for (const want of [{ key: "nope" }, { key: "<script>" }, { index: 0 }, { key: "last" }]) { | |
| 140 | const { m, r } = await open(want); | |
| 141 | expect(m.ok).toBe(false); | |
| 142 | expect(r.asked.length).toBe(0); | |
| 143 | } | |
| 144 | }); | |
| 145 | test("an array's item opens by index", async () => { | |
| 146 | const arr = enc(`[{"a":1},[1,2,3],"s"]`); | |
| 147 | const m = await openMember(arr, 6_000_000, { index: 1 }, async (s, n) => arr.slice(s, s + n), "/d"); | |
| 148 | if (!m.ok) throw new Error("unreachable"); | |
| 149 | expect(m.content.toString()).toContain("[\n 1,\n 2,\n 3\n]"); | |
| 150 | expect((await openMember(arr, 6_000_000, { index: 3 }, async () => undefined, "/d")).ok).toBe(false); | |
| 151 | }); | |
| 152 | }); | |
| 153 | ||
| 154 | describe("a member cut inside a character", () => { | |
| 155 | test("wholeUtf8 drops a trailing incomplete sequence and keeps complete ones", () => { | |
| 156 | const e = new TextEncoder().encode("a€"); | |
| 157 | expect(new TextDecoder().decode(wholeUtf8(e.subarray(0, 2)))).toBe("a"); | |
| 158 | expect(new TextDecoder().decode(wholeUtf8(e.subarray(0, 3)))).toBe("a"); | |
| 159 | expect(new TextDecoder().decode(wholeUtf8(e))).toBe("a€"); | |
| 160 | expect(wholeUtf8(new TextEncoder().encode("ab")).length).toBe(2); | |
| 161 | }); | |
| 162 | }); |
test/spawn-pipes.test.ts +1 −1
| @@ -35,7 +35,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 35 | 35 | "src/git/combined.ts": { sites: ["stdout"], reason: "spawnSync buffers rev-parse stdout; stderr ignored" }, |
| 36 | 36 | "src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" }, |
| 37 | 37 | "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (three as text alongside exited, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" }, |
| 38 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed), and by blobTail until the blob ends, keeping a ring of its last bytes; stderr ignored" }, | |
| 38 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed), by blobRange until its range is in (then killed), and by blobTail until the blob ends, keeping a ring of its last bytes; stderr ignored" }, | |
| 39 | 39 | "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" }, |
| 40 | 40 | "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" }, |
| 41 | 41 | "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" }, |
test/tui-workspace-formats.test.ts +12 −1
| @@ -89,7 +89,7 @@ describe("the terminal workspace, by file type", () => { | ||
| 89 | 89 | return run.stdout.toString().trim(); |
| 90 | 90 | }; |
| 91 | 91 | const list = `[${Array(60_000).fill('{"id":1,"name":"padding padding"}').join(",")}]`; |
| 92 | const blob = git(["hash-object", "-w", "--stdin"], `{"alpha":${list},"beta":${list},"gamma":${list},"small":{"a":1}}`); | |
| 92 | const blob = git(["hash-object", "-w", "--stdin"], `{"small":{"a":1},"alpha":${list},"beta":${list},"gamma":${list}}`); | |
| 93 | 93 | const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${blob}\tdata.json\n`); |
| 94 | 94 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); |
| 95 | 95 | const h = boot(); |
| @@ -101,6 +101,17 @@ describe("the terminal workspace, by file type", () => { | ||
| 101 | 101 | expect(h.text()).toContain("array"); |
| 102 | 102 | expect(h.text()).toContain("60,000"); |
| 103 | 103 | expect(h.text()).toContain("the file was not parsed"); |
| 104 | // `m` names a key; it opens on its own, read by byte range. | |
| 105 | await h.press("m"); | |
| 106 | await h.type("small"); | |
| 107 | await h.press("enter"); | |
| 108 | expect(h.text()).toContain('"a": 1'); | |
| 109 | expect(h.text()).not.toContain("alpha"); | |
| 110 | // A key the outline does not list is told in the page's words. | |
| 111 | await h.press("m"); | |
| 112 | await h.type("nope"); | |
| 113 | await h.press("enter"); | |
| 114 | expect(h.text()).toContain("no such key or item"); | |
| 104 | 115 | } finally { |
| 105 | 116 | h.stop(); |
| 106 | 117 | } |
new above
offline