Public: anyone can read this; the crew can change it.
A JSON file over 5 MB opens as its first level with counts, and each key opens on its own #13993
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13964-a-json-file-over-5-mb-opens-as-its-first → main 6 commits fork 1 thread · 1 unresolved
13 files
changed +475
−6
docs/features/README.md +1
| @@ -305,6 +305,7 @@ Read with the guide page [Your apps](../guide-your-apps.md). | ||
| 305 | 305 | - [](form-keeps-input.md) |
| 306 | 306 | - [](vcf-view.md) |
| 307 | 307 | - [](csv-view.md) |
| 308 | - [](json-big.md) | |
| 308 | 309 | - [](timeline-cut-list.md) |
| 309 | 310 | - [](file-media.md) |
| 310 | 311 | - [](font.md) |
docs/features/json-big.md added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | # A big JSON file opens as its first level | |
| 2 | ||
| 3 | A `.json` file over the page's 1 MB read ceiling is not parsed and not turned | |
| 4 | into a form. The repository page and the terminal open it as an outline of its | |
| 5 | first level: each key (or, for an array, each item) with its type and how many | |
| 6 | children it has. | |
| 7 | ||
| 8 | - **The 4 MB read.** Only the first 4 MB of the file is read | |
| 9 | (`JSON_PREFIX_BYTES`), however big the file is. The outline is scanned from | |
| 10 | that prefix; nothing is parsed whole. The line under the outline says how much | |
| 11 | was read of how much. | |
| 12 | - **The 500-member limit.** At most 500 keys or items are listed | |
| 13 | (`outlineJson`'s `limit`). When the list stops there, the line says the rest | |
| 14 | are not listed. | |
| 15 | - **"At least".** A value the 4 MB read cut short has a count of the children | |
| 16 | seen so far, marked "at least": the real count is that or more. A cut value | |
| 17 | is the last row, because nothing after it was read. | |
| 18 | - **When there is no outline.** A file whose first byte is not `{` or `[`, or | |
| 19 | that holds a NUL byte, says so and offers the download. | |
| 20 | - **The terminal.** The workspace screen draws the same rows as aligned text, | |
| 21 | read-only, with where the whole file is (`jsonOutlineText`). | |
| 22 | ||
| 23 | What it does not do: opening a key on its own is not built. A person who needs a | |
| 24 | key's value downloads the file. | |
| 25 | ||
| 26 | Code: `src/web/render/json-big.ts` (facts, table, text), `json-doc.ts` | |
| 27 | (`outlineJson`), `kinds.ts` (`JSON_PREFIX_BYTES`), `src/view/local.ts` (the | |
| 28 | prefix read). Tests: `test/render-json-big.test.ts`, `test/json-big-page.test.ts`, | |
| 29 | `test/tui-workspace-formats.test.ts`. |
src/tui/screens/workspace.ts +7 −2
| @@ -50,7 +50,8 @@ import { canAdmin, canWrite, DECISIONS_BY, getRepoById, type Repo } from "../../ | ||
| 50 | 50 | import type { TreeEntry } from "../../git/read.ts"; |
| 51 | 51 | import { parseUnifiedDiff, type Diff } from "../../git/diff.ts"; |
| 52 | 52 | import { renderDiff as renderPatch } from "../ui/diff.ts"; |
| 53 | import { fileKind, isEnvExample, kindByName, HEAD_BYTES, NO_BYTES, RENDER_CAP, sniffType } from "../../web/render/kinds.ts"; | |
| 53 | import { jsonOutlineText } from "../../web/render/json-big.ts"; | |
| 54 | import { fileKind, isEnvExample, isJsonPath, kindByName, HEAD_BYTES, NO_BYTES, RENDER_CAP, sniffType } from "../../web/render/kinds.ts"; | |
| 54 | 55 | import { tailOf, tailSentence } from "../../web/log_tail.ts"; |
| 55 | 56 | import { downloadHref } from "../../web/pages/media.ts"; |
| 56 | 57 | import { isModelPath, modelFacts } from "../../web/render/model-facts.ts"; |
| @@ -1460,7 +1461,11 @@ export function workspaceScreen( | ||
| 1460 | 1461 | `Binary file, ${formatSize(read.value.size)}, ${sniffType(read.value.body.subarray(0, HEAD_BYTES))}.`; |
| 1461 | 1462 | } else { |
| 1462 | 1463 | raw = read.value.body; // .html included: source here, the page preview is web-only |
| 1463 | if (read.value.partial === true) | |
| 1464 | // A big JSON file arrives cut: its first level is drawn, as the page draws it. | |
| 1465 | if (read.value.partial === true && isJsonPath(t.path)) { | |
| 1466 | raw = new TextEncoder().encode(jsonOutlineText(read.value.body, read.value.size, downloadAt(t.path))); | |
| 1467 | cut = "First level of a file too big to open, read-only."; | |
| 1468 | } else if (read.value.partial === true) | |
| 1464 | 1469 | cut = `Showing the first ${formatSize(read.value.body.length)} of ${formatSize(read.value.size)}, read-only. The rest is in the raw file.`; |
| 1465 | 1470 | } |
| 1466 | 1471 | } |
src/view/local.ts +8 −1
| @@ -46,7 +46,7 @@ import { repoDir } from "../git/repos.ts"; | ||
| 46 | 46 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 47 | 47 | import type { Page } from "../origins/types.ts"; |
| 48 | 48 | import { tailable, TAIL_BYTES } from "../web/log_tail.ts"; |
| 49 | import { CSV_PREFIX_BYTES, HEAD_BYTES, isTablePath, RENDER_CAP, wholeLines } from "../web/render/kinds.ts"; | |
| 49 | import { CSV_PREFIX_BYTES, HEAD_BYTES, isJsonPath, isTablePath, JSON_PREFIX_BYTES, RENDER_CAP, wholeLines } from "../web/render/kinds.ts"; | |
| 50 | 50 | import type { |
| 51 | 51 | Blob, |
| 52 | 52 | Branch, |
| @@ -222,6 +222,13 @@ export function localSource( | ||
| 222 | 222 | if (prefix !== undefined && !prefix.subarray(0, 8000).includes(0)) |
| 223 | 223 | return ok({ kind: "text", body: wholeLines(prefix), size: blob.size, partial: true }); |
| 224 | 224 | } |
| 225 | // A JSON file over the ceiling is read as a prefix too: its outline is drawn, never parsed | |
| 226 | // whole. | |
| 227 | if (opts?.prefix === true && isJsonPath(path)) { | |
| 228 | const prefix = await blobHead(dir, ref, path, JSON_PREFIX_BYTES); | |
| 229 | if (prefix !== undefined && !prefix.subarray(0, 8000).includes(0)) | |
| 230 | return ok({ kind: "text", body: prefix, size: blob.size, partial: true }); | |
| 231 | } | |
| 225 | 232 | // Too large to read, but its first bytes still say what it is. |
| 226 | 233 | const head = await blobHead(dir, ref, path, HEAD_BYTES); |
| 227 | 234 | const tail = tailable(path) ? await blobTail(dir, ref, path, TAIL_BYTES, blob.size) : undefined; |
src/web/pages/repo.ts +3 −1
| @@ -103,7 +103,7 @@ import { fontJs } from "../render/font.ts"; | ||
| 103 | 103 | import { INLINE_MAX, keepGcode, keptGcode, layerCount } from "../render/gcode.ts"; |
| 104 | 104 | import { hasTable } from "../render/sqlite.ts"; |
| 105 | 105 | import { READ_CEILING } from "../render/psd-read.ts"; |
| 106 | import { CSV_PREFIX_BYTES, fileKind, HEAD_BYTES, isTablePath, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts"; | |
| 106 | import { CSV_PREFIX_BYTES, fileKind, HEAD_BYTES, isJsonPath, isTablePath, JSON_PREFIX_BYTES, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts"; | |
| 107 | 107 | import { RENDERERS, rendererFor } from "../render/renderers.ts"; |
| 108 | 108 | import { downloadHref, mediaHref } from "./media.ts"; |
| 109 | 109 | import { browseBase, localScope, type RepoScope } from "../scope.ts"; |
| @@ -1992,6 +1992,8 @@ export function capped(blob: Blob, path = ""): Blob { | ||
| 1992 | 1992 | return blob; |
| 1993 | 1993 | // A table file's body is already bounded by its prefix; the renderer draws a window of it. |
| 1994 | 1994 | if (blob.kind !== "too_large" && blob.body.length <= CSV_PREFIX_BYTES && isTablePath(path)) return blob; |
| 1995 | // A big JSON file's body is already its prefix; the renderer outlines it. | |
| 1996 | if (blob.kind === "text" && blob.partial === true && blob.body.length <= JSON_PREFIX_BYTES && isJsonPath(path)) return blob; | |
| 1995 | 1997 | return blob.kind !== "too_large" && blob.size > MAX_RENDER_BYTES |
| 1996 | 1998 | ? { kind: "too_large", size: blob.size, head: blob.body.slice(0, HEAD_BYTES) } |
| 1997 | 1999 | : blob; |
src/web/render/config.ts +3
| @@ -9,6 +9,7 @@ import { html, type Html } from "../html.ts"; | ||
| 9 | 9 | import { btn } from "../ui.ts"; |
| 10 | 10 | import { configFields, countFields, MAX_FIELDS } from "./config-form.ts"; |
| 11 | 11 | import { languageOf, parseConfig } from "./config-format.ts"; |
| 12 | import { jsonOutline } from "./json-big.ts"; | |
| 12 | 13 | import type { Rendered, RenderInput } from "./renderers.ts"; |
| 13 | 14 | |
| 14 | 15 | /** |
| @@ -33,6 +34,8 @@ function brokenConfig(language: string, text: string, line: number, column: numb | ||
| 33 | 34 | } |
| 34 | 35 | |
| 35 | 36 | export function config(input: RenderInput): Rendered { |
| 37 | // Only a JSON file over the read ceiling arrives cut: it is outlined, never parsed. | |
| 38 | if (input.partial === true) return jsonOutline(input); | |
| 36 | 39 | const text = new TextDecoder().decode(input.body); |
| 37 | 40 | const lines = text === "" ? 0 : text.replace(/\n$/, "").split("\n").length; |
| 38 | 41 | const language = languageOf(input.path); |
src/web/render/json-big.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | /** | |
| 2 | * A JSON file too big to parse, drawn as its first level: each key with its | |
| 3 | * type and how many children it has. Only a prefix of the file was read | |
| 4 | * (`JSON_PREFIX_BYTES`); the note under the outline says how much, and what | |
| 5 | * was left out. The page draws it as a table (`jsonOutline`), the terminal as | |
| 6 | * aligned text (`jsonOutlineText`); both read the same facts (`outlineFacts`). | |
| 7 | */ | |
| 8 | ||
| 9 | import { html } from "../html.ts"; | |
| 10 | import { fileSize } from "../ui.ts"; | |
| 11 | import { outlineJson } from "./json-doc.ts"; | |
| 12 | import type { Rendered, RenderInput } from "./renderers.ts"; | |
| 13 | ||
| 14 | /** A fixed locale, so a count reads the same whatever the server's. */ | |
| 15 | const count = (n: number): string => n.toLocaleString("en-US"); | |
| 16 | ||
| 17 | export interface OutlineRow { | |
| 18 | readonly key: string; | |
| 19 | readonly type: string; | |
| 20 | readonly children: string; | |
| 21 | } | |
| 22 | ||
| 23 | export interface OutlineFacts { | |
| 24 | readonly heading: string; | |
| 25 | readonly rows: readonly OutlineRow[]; | |
| 26 | readonly said: string; | |
| 27 | readonly summary: string; | |
| 28 | } | |
| 29 | ||
| 30 | /** The outline of a prefix as plain data, or the sentence saying it has none. */ | |
| 31 | export function outlineFacts(body: Uint8Array, size: number): OutlineFacts | string { | |
| 32 | const outline = outlineJson(new TextDecoder().decode(body)); | |
| 33 | const read = fileSize(body.length); | |
| 34 | const whole = fileSize(size); | |
| 35 | if (outline === undefined) return `This file is ${whole}, too large to open as a form, and it does not open as an object or an array.`; | |
| 36 | const object = outline.root === "object"; | |
| 37 | const noun = object ? "keys" : "items"; | |
| 38 | const said: string[] = [`Read the first ${read} of ${whole}; the file was not parsed.`]; | |
| 39 | if (outline.limited) said.push(`Showing the first ${count(outline.entries.length)} ${noun}; the rest are not listed.`); | |
| 40 | else if (!outline.complete) said.push(`The ${noun} after these, and the end of the file, were past what was read and are not shown.`); | |
| 41 | if (outline.entries.some((e) => e.atLeast === true)) said.push("A count marked “at least” is of a value the read cut short."); | |
| 42 | return { | |
| 43 | heading: object ? "Key" : "Item", | |
| 44 | rows: outline.entries.map((e) => ({ | |
| 45 | key: e.key, | |
| 46 | type: e.type, | |
| 47 | children: e.count === undefined ? "" : `${e.atLeast === true ? "at least " : ""}${count(e.count)}`, | |
| 48 | })), | |
| 49 | said: said.join(" "), | |
| 50 | summary: `${count(outline.entries.length)}${outline.complete ? "" : "+"} ${noun} · ${object ? "JSON object" : "JSON array"} · ${whole}`, | |
| 51 | }; | |
| 52 | } | |
| 53 | ||
| 54 | export function jsonOutline(input: RenderInput): Rendered { | |
| 55 | const facts = outlineFacts(input.body, input.size); | |
| 56 | if (typeof facts === "string") return { lines: 0, content: html``, fallback: facts }; | |
| 57 | const rows = facts.rows.map( | |
| 58 | (r) => html`<tr><td class="mono">${r.key}</td><td>${r.type}</td><td class="num">${r.children}</td></tr>`, | |
| 59 | ); | |
| 60 | return { | |
| 61 | lines: 0, | |
| 62 | summary: facts.summary, | |
| 63 | content: html`<table class="table" data-dense aria-label="First level of ${input.path.slice(input.path.lastIndexOf("/") + 1)}"> | |
| 64 | <thead><tr><th>${facts.heading}</th><th>Type</th><th class="num">Children</th></tr></thead> | |
| 65 | <tbody>${rows}</tbody> | |
| 66 | </table> | |
| 67 | <p class="muted small">${facts.said} <a href="${input.download}">Download the whole file.</a></p>`, | |
| 68 | }; | |
| 69 | } | |
| 70 | ||
| 71 | /** The same outline as aligned text for the terminal, with where the whole file is. */ | |
| 72 | export function jsonOutlineText(body: Uint8Array, size: number, where: string): string { | |
| 73 | const facts = outlineFacts(body, size); | |
| 74 | if (typeof facts === "string") return `${facts}\n${where}\n`; | |
| 75 | const w = (pick: (r: OutlineRow) => string, head: string): number => Math.max(head.length, ...facts.rows.map((r) => pick(r).length)); | |
| 76 | const kw = w((r) => r.key, facts.heading); | |
| 77 | const tw = w((r) => r.type, "Type"); | |
| 78 | const line = (a: string, b: string, c: string): string => `${a.padEnd(kw)} ${b.padEnd(tw)} ${c}`.trimEnd(); | |
| 79 | return [ | |
| 80 | facts.summary, | |
| 81 | "", | |
| 82 | line(facts.heading, "Type", "Children"), | |
| 83 | ...facts.rows.map((r) => line(r.key, r.type, r.children)), | |
| 84 | "", | |
| 85 | `${facts.said} ${where}`, | |
| 86 | "", | |
| 87 | ].join("\n"); | |
| 88 | } |
src/web/render/json-doc.ts +126
| @@ -344,3 +344,129 @@ export function lineDiff(before: string, after: string, context = 3): DiffLine[] | ||
| 344 | 344 | }); |
| 345 | 345 | return all.filter((_, k) => near[k]); |
| 346 | 346 | } |
| 347 | ||
| 348 | // ---- the outline of a document too big to parse ---- | |
| 349 | ||
| 350 | /** One member of a big document's first level: what it is and how many children it has. */ | |
| 351 | export interface OutlineEntry { | |
| 352 | /** The key, or `[n]` for an array's nth item. */ | |
| 353 | readonly key: string; | |
| 354 | readonly type: Json["t"]; | |
| 355 | /** Children of an object or array; undefined for a scalar. */ | |
| 356 | readonly count?: number; | |
| 357 | /** The text ended inside this value, so `count` is a floor. */ | |
| 358 | readonly atLeast?: boolean; | |
| 359 | } | |
| 360 | ||
| 361 | export interface Outline { | |
| 362 | readonly root: "object" | "array"; | |
| 363 | readonly entries: readonly OutlineEntry[]; | |
| 364 | /** Every member of the first level was listed, and the document ended. */ | |
| 365 | readonly complete: boolean; | |
| 366 | /** Listing stopped at `limit` members, not at the end of the text. */ | |
| 367 | readonly limited: boolean; | |
| 368 | } | |
| 369 | ||
| 370 | /** | |
| 371 | * The first level of a document, read in one pass over `text` with no | |
| 372 | * values built: each member's type and child count. It reads at most | |
| 373 | * `text.length` characters and keeps only `limit` entries, so what it costs | |
| 374 | * is set by the caller's bound, not by the document. It does not validate: a | |
| 375 | * document that is not JSON gives `undefined` only when it does not open as | |
| 376 | * an object or array. | |
| 377 | */ | |
| 378 | export function outlineJson(text: string, limit = 500): Outline | undefined { | |
| 379 | const n = text.length; | |
| 380 | let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; | |
| 381 | const ws = (): void => { | |
| 382 | for (;;) { | |
| 383 | const c = text.charCodeAt(i); | |
| 384 | if (c === 32 || c === 10 || c === 13 || c === 9) i++; | |
| 385 | else return; | |
| 386 | } | |
| 387 | }; | |
| 388 | /** Past the string that opens at `i`; n + 1 when the text ends inside it. */ | |
| 389 | const skipString = (): number => { | |
| 390 | let j = i + 1; | |
| 391 | for (;;) { | |
| 392 | const q = text.indexOf('"', j); | |
| 393 | if (q < 0) return n + 1; | |
| 394 | let back = 0; | |
| 395 | while (text.charCodeAt(q - 1 - back) === 92) back++; | |
| 396 | if (back % 2 === 0) return q + 1; | |
| 397 | j = q + 1; | |
| 398 | } | |
| 399 | }; | |
| 400 | ws(); | |
| 401 | const open = text[i]; | |
| 402 | if (open !== "{" && open !== "[") return undefined; | |
| 403 | const root = open === "{" ? "object" : "array"; | |
| 404 | i++; | |
| 405 | const entries: OutlineEntry[] = []; | |
| 406 | /** Skips the value at `i`; its type, child count and whether the text ended inside it. */ | |
| 407 | const value = (): { t: Json["t"]; count?: number; cut: boolean } => { | |
| 408 | const c = text[i]; | |
| 409 | if (c === '"') { | |
| 410 | i = skipString(); | |
| 411 | return { t: "string", cut: i > n }; | |
| 412 | } | |
| 413 | if (c === "{" || c === "[") { | |
| 414 | let depth = 0; | |
| 415 | let count = 0; | |
| 416 | let filled = false; | |
| 417 | while (i < n) { | |
| 418 | const ch = text.charCodeAt(i); | |
| 419 | if (ch === 34) { | |
| 420 | i = skipString(); | |
| 421 | filled ||= depth === 1; | |
| 422 | continue; | |
| 423 | } | |
| 424 | i++; | |
| 425 | if (ch === 123 || ch === 91) { | |
| 426 | if (depth === 1) filled = true; | |
| 427 | depth++; | |
| 428 | } else if (ch === 125 || ch === 93) { | |
| 429 | if (--depth === 0) return { t: c === "{" ? "object" : "array", count: count + (filled ? 1 : 0), cut: false }; | |
| 430 | } else if (ch === 44 && depth === 1) count++; | |
| 431 | else if (depth === 1 && ch !== 32 && ch !== 10 && ch !== 13 && ch !== 9 && ch !== 58) filled = true; | |
| 432 | } | |
| 433 | return { t: c === "{" ? "object" : "array", count: count + (filled ? 1 : 0), cut: true }; | |
| 434 | } | |
| 435 | const m = /^(?:true|false|null|-?[0-9][0-9.eE+-]*)/.exec(text.slice(i, i + 400)); | |
| 436 | const word = m?.[0] ?? ""; | |
| 437 | i += Math.max(word.length, 1); | |
| 438 | const t = word === "true" || word === "false" ? "boolean" : word === "null" ? "null" : "number"; | |
| 439 | return { t, cut: i > n }; | |
| 440 | }; | |
| 441 | const closer = root === "object" ? "}" : "]"; | |
| 442 | for (;;) { | |
| 443 | ws(); | |
| 444 | if (i >= n) return { root, entries, complete: false, limited: false }; | |
| 445 | if (text[i] === closer) return { root, entries, complete: true, limited: false }; | |
| 446 | if (entries.length >= limit) return { root, entries, complete: false, limited: true }; | |
| 447 | let key = `[${entries.length}]`; | |
| 448 | if (root === "object") { | |
| 449 | if (text[i] !== '"') return { root, entries, complete: false, limited: false }; | |
| 450 | const end = skipString(); | |
| 451 | if (end > n) return { root, entries, complete: false, limited: false }; | |
| 452 | const raw = text.slice(i, end); | |
| 453 | try { | |
| 454 | key = raw.length > 300 ? `${raw.slice(1, 200)}…` : String(JSON.parse(raw)); | |
| 455 | } catch { | |
| 456 | key = raw.slice(1, -1); | |
| 457 | } | |
| 458 | i = end; | |
| 459 | ws(); | |
| 460 | if (text[i] !== ":") return { root, entries, complete: false, limited: false }; | |
| 461 | i++; | |
| 462 | ws(); | |
| 463 | if (i >= n) return { root, entries, complete: false, limited: false }; | |
| 464 | } | |
| 465 | const v = value(); | |
| 466 | if (v.cut && v.t !== "object" && v.t !== "array") return { root, entries, complete: false, limited: false }; | |
| 467 | entries.push({ key, type: v.t, ...(v.count === undefined ? {} : { count: v.count }), ...(v.cut ? { atLeast: true } : {}) }); | |
| 468 | if (v.cut) return { root, entries, complete: false, limited: false }; | |
| 469 | ws(); | |
| 470 | if (text[i] === ",") i++; | |
| 471 | } | |
| 472 | } |
src/web/render/kinds.ts +9
| @@ -376,6 +376,15 @@ export const CSV_PREFIX_BYTES = 2 * MB; | ||
| 376 | 376 | /** Whether a name is a table file the page draws from a prefix. */ |
| 377 | 377 | export const isTablePath = (path: string): boolean => /\.(csv|tsv|jsonl|ndjson)$/i.test(path); |
| 378 | 378 | |
| 379 | /** | |
| 380 | * A `.json` file over the page's read ceiling is not parsed: the page reads its first | |
| 381 | * `JSON_PREFIX_BYTES` and draws the outline of its first level (`outlineJson`). | |
| 382 | */ | |
| 383 | export const JSON_PREFIX_BYTES = 4 * MB; | |
| 384 | ||
| 385 | /** Whether a name is a JSON document the page outlines when it is big. */ | |
| 386 | export const isJsonPath = (path: string): boolean => /\.json$/i.test(path); | |
| 387 | ||
| 379 | 388 | /** A prefix cut back to its last line end, so no half line is drawn; whole when it has none. */ |
| 380 | 389 | export function wholeLines(prefix: Uint8Array): Uint8Array { |
| 381 | 390 | const end = prefix.lastIndexOf(10); |
src/web/render/renderers.ts +3 −2
| @@ -33,7 +33,7 @@ import { feedXml } from "./feed-xml.ts"; | ||
| 33 | 33 | import { vcf } from "./vcf.ts"; |
| 34 | 34 | import { htmlPreview } from "./html_preview.ts"; |
| 35 | 35 | import { image } from "./image.ts"; |
| 36 | import { fileKind, HEAD_BYTES, kindByName, NO_BYTES, RENDER_CAP, type FileKind } from "./kinds.ts"; | |
| 36 | import { fileKind, HEAD_BYTES, isJsonPath, kindByName, NO_BYTES, RENDER_CAP, type FileKind } from "./kinds.ts"; | |
| 37 | 37 | import { log, drawsLog } from "./log.ts"; |
| 38 | 38 | import { makefile } from "./makefile.ts"; |
| 39 | 39 | import { markdown } from "./markdown.ts"; |
| @@ -208,7 +208,8 @@ export function rendererFor( | ||
| 208 | 208 | : kindByName(path); |
| 209 | 209 | // A table's cap is lifted only for a body the source already cut to a prefix. |
| 210 | 210 | const prefixed = kind === "csv" && blob.kind === "text" && blob.partial === true; |
| 211 | if (blob.size > RENDER_CAP[kind] && !prefixed) return undefined; | |
| 211 | const outlined = kind === "config" && blob.kind === "text" && blob.partial === true && isJsonPath(path); | |
| 212 | if (blob.size > RENDER_CAP[kind] && !prefixed && !outlined) return undefined; | |
| 212 | 213 | if (MEDIA_KINDS.has(kind) && !where.media) return undefined; |
| 213 | 214 | if (blob.kind === "too_large" && !MEDIA_KINDS.has(kind) && kind !== "psd") return undefined; |
| 214 | 215 | // A kind may draw only some of its files: `config` draws JSON, YAML and |
test/json-big-page.test.ts added +71
| @@ -0,0 +1,71 @@ | ||
| 1 | /** A real .json file over 5 MB, in a real repository, opens as its first level on the page. */ | |
| 2 | ||
| 3 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { repoDir } from "../src/git/repos.ts"; | |
| 10 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 11 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 12 | import { quietLogs } from "./helpers/globals.ts"; | |
| 13 | ||
| 14 | quietLogs(); | |
| 15 | ||
| 16 | const item = '{"id":1,"name":"padding padding padding","tags":["x","y"]}'; | |
| 17 | const list = (n: number): string => `[${Array(n).fill(item).join(",")}]`; | |
| 18 | /** Over 5 MB: three keys of long lists, then a key past the 4 MB read. */ | |
| 19 | const BIG = `{"small":{"a":1,"b":2},"alpha":${list(40_000)},"beta":${list(40_000)},"gamma":${list(40_000)},"tail":${list(1000)},"zlast":true}`; | |
| 20 | ||
| 21 | describe("a big JSON file on the repository page", () => { | |
| 22 | let dataDir = ""; | |
| 23 | let app: App; | |
| 24 | let servers: Servers; | |
| 25 | ||
| 26 | beforeAll(async () => { | |
| 27 | dataDir = await mkdtemp(join(tmpdir(), "forge-json-big-")); | |
| 28 | app = createApp({ dataDir, port: 0 }); | |
| 29 | servers = await serve(app); | |
| 30 | const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-json-big-passphrase" }); | |
| 31 | if (!a.ok) throw new Error("signup failed"); | |
| 32 | const created = await createRepo(app, a.value, { name: "bigjson", visibility: "public" }); | |
| 33 | if (!created.ok) throw new Error("repo failed"); | |
| 34 | const made = await commitToBranch(app, a.value, created.value, { message: "start", files: [{ path: "README.md", content: "# b\n" }] }); | |
| 35 | if (!made.ok) throw new Error("commit failed"); | |
| 36 | const dir = repoDir(app.reposRoot, "alice", "bigjson"); | |
| 37 | const git = (args: string[], stdin?: string): string => { | |
| 38 | const run = Bun.spawnSync(["git", ...args], { | |
| 39 | cwd: dir, | |
| 40 | stdin: stdin === undefined ? "ignore" : new TextEncoder().encode(stdin), | |
| 41 | env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 42 | }); | |
| 43 | if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`); | |
| 44 | return run.stdout.toString().trim(); | |
| 45 | }; | |
| 46 | const blob = git(["hash-object", "-w", "--stdin"], BIG); | |
| 47 | const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${blob}\tdata.json\n`); | |
| 48 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); | |
| 49 | }, 60_000); | |
| 50 | ||
| 51 | afterAll(async () => { | |
| 52 | await servers?.stop(); | |
| 53 | app?.close(); | |
| 54 | await rm(dataDir, { recursive: true, force: true }); | |
| 55 | }); | |
| 56 | ||
| 57 | test("is over 5 MB", () => expect(BIG.length).toBeGreaterThan(5_000_000)); | |
| 58 | ||
| 59 | test("opens as its first level, bare, with a floor on the cut count and a say of what was left out", async () => { | |
| 60 | const res = await fetch(`${servers.url}/alice/bigjson/blob/main/data.json`); | |
| 61 | const text = await res.text(); | |
| 62 | expect(res.status).toBe(200); | |
| 63 | expect(text.includes(">alpha<")).toBe(true); | |
| 64 | expect(text.includes(">small<")).toBe(true); | |
| 65 | expect(text.includes("40,000")).toBe(true); | |
| 66 | expect(text.includes("at least")).toBe(true); | |
| 67 | expect(text.includes("the file was not parsed")).toBe(true); | |
| 68 | expect(text.includes(">zlast<")).toBe(false); | |
| 69 | expect(text.includes("json-big")).toBe(false); | |
| 70 | }); | |
| 71 | }); |
test/render-json-big.test.ts added +91
| @@ -0,0 +1,91 @@ | ||
| 1 | /** A JSON file over the read ceiling opens as its first level, never parsed whole. */ | |
| 2 | import { describe, expect, test } from "bun:test"; | |
| 3 | import { outlineJson } from "../src/web/render/json-doc.ts"; | |
| 4 | import { JSON_PREFIX_BYTES } from "../src/web/render/kinds.ts"; | |
| 5 | import { config } from "../src/web/render/config.ts"; | |
| 6 | import { rendererFor } from "../src/web/render/renderers.ts"; | |
| 7 | ||
| 8 | const enc = (s: string): Uint8Array => new TextEncoder().encode(s); | |
| 9 | const draw = (body: Uint8Array, size: number) => | |
| 10 | config({ path: "r/big.json", size, body, partial: true, media: "/m", download: "/d" }); | |
| 11 | ||
| 12 | /** About `mb` megabytes: a few keys, each holding a long list. */ | |
| 13 | function generated(mb: number): string { | |
| 14 | const item = '{"id":1,"name":"a \\"quoted\\" ] } name","tags":["x","y"],"ok":true,"n":null,"v":1.5e3}'; | |
| 15 | const per = Math.ceil((mb * 1_000_000) / 6 / (item.length + 1)); | |
| 16 | const list = `[${Array(per).fill(item).join(",")}]`; | |
| 17 | return `{"alpha":${list},"beta":${list},"gamma":{"a":1,"b":[1,2],"c":{}},"delta":"text","epsilon":${list},"zeta":${list},"eta":${list},"theta":${list},"last":false}`; | |
| 18 | } | |
| 19 | ||
| 20 | describe("outlineJson", () => { | |
| 21 | test("lists a first level with types and child counts", () => { | |
| 22 | const o = outlineJson('{"a":[1,2,3],"b":{"x":1,"y":{"z":[]}},"c":"s","d":12,"e":true,"f":null,"g":[],"h":{}}')!; | |
| 23 | expect(o.complete).toBe(true); | |
| 24 | expect(o.entries).toEqual([ | |
| 25 | { key: "a", type: "array", count: 3 }, | |
| 26 | { key: "b", type: "object", count: 2 }, | |
| 27 | { key: "c", type: "string" }, | |
| 28 | { key: "d", type: "number" }, | |
| 29 | { key: "e", type: "boolean" }, | |
| 30 | { key: "f", type: "null" }, | |
| 31 | { key: "g", type: "array", count: 0 }, | |
| 32 | { key: "h", type: "object", count: 0 }, | |
| 33 | ]); | |
| 34 | }); | |
| 35 | test("brackets and commas inside strings do not count", () => { | |
| 36 | const o = outlineJson('{"a":["]","}\\\\",",\\"",3]}')!; | |
| 37 | expect(o.entries[0]).toEqual({ key: "a", type: "array", count: 4 }); | |
| 38 | }); | |
| 39 | test("an array root lists its items", () => { | |
| 40 | const o = outlineJson('[1,{"a":1},[1,2]]')!; | |
| 41 | expect(o.root).toBe("array"); | |
| 42 | expect(o.entries.map((e) => [e.key, e.type, e.count])).toEqual([["[0]", "number", undefined], ["[1]", "object", 1], ["[2]", "array", 2]]); | |
| 43 | }); | |
| 44 | test("a cut value has a floor for a count and ends the list", () => { | |
| 45 | const o = outlineJson('{"a":[1],"b":[1,2,3,4')!; | |
| 46 | expect(o.complete).toBe(false); | |
| 47 | expect(o.entries[1]).toEqual({ key: "b", type: "array", count: 4, atLeast: true }); | |
| 48 | }); | |
| 49 | test("stops at the limit", () => { | |
| 50 | const o = outlineJson(`[${Array(50).fill(1).join(",")}]`, 10)!; | |
| 51 | expect(o.entries.length).toBe(10); | |
| 52 | expect(o.limited).toBe(true); | |
| 53 | }); | |
| 54 | test("a scalar or empty text has no outline", () => { | |
| 55 | expect(outlineJson("12")).toBeUndefined(); | |
| 56 | expect(outlineJson("")).toBeUndefined(); | |
| 57 | }); | |
| 58 | }); | |
| 59 | ||
| 60 | describe("a 20 MB document", () => { | |
| 61 | const text = generated(20); | |
| 62 | const whole = enc(text); | |
| 63 | test("is big enough to matter", () => expect(whole.length).toBeGreaterThan(20_000_000)); | |
| 64 | test("outlines its prefix inside a time budget, and says what it left out", () => { | |
| 65 | const prefix = whole.subarray(0, JSON_PREFIX_BYTES); | |
| 66 | const t = performance.now(); | |
| 67 | const r = draw(prefix, whole.length); | |
| 68 | const ms = performance.now() - t; | |
| 69 | expect(ms).toBeLessThan(500); | |
| 70 | const out = r.content.toString(); | |
| 71 | expect(out).toContain(">alpha<"); | |
| 72 | expect(out).toContain("array"); | |
| 73 | expect(out).toContain("at least"); | |
| 74 | expect(out).toContain("the file was not parsed"); | |
| 75 | expect(out).toContain("past what was read and are not shown"); | |
| 76 | expect(out).not.toContain(">last<"); | |
| 77 | }); | |
| 78 | test("the page hands a partial JSON blob to the outline, and a plain one to the form", () => { | |
| 79 | const blob = { kind: "text" as const, body: whole.subarray(0, 1000), size: whole.length, partial: true }; | |
| 80 | expect(rendererFor("r/big.json", blob)).toBeDefined(); | |
| 81 | expect(rendererFor("r/big.json", { kind: "text", body: whole.subarray(0, 1000), size: whole.length })).toBeUndefined(); | |
| 82 | }); | |
| 83 | }); | |
| 84 | ||
| 85 | describe("a small cut document", () => { | |
| 86 | test("escapes keys", () => { | |
| 87 | const r = draw(enc('{"<script>":[1]}'), 6_000_000); | |
| 88 | expect(r.content.toString()).toContain("<script>"); | |
| 89 | expect(r.content.toString()).not.toContain("<script>"); | |
| 90 | }); | |
| 91 | }); |
test/tui-workspace-formats.test.ts +36
| @@ -70,6 +70,42 @@ describe("the terminal workspace, by file type", () => { | ||
| 70 | 70 | } |
| 71 | 71 | }); |
| 72 | 72 | |
| 73 | test("a .json over 5 MB opens as its first level, read-only, as the page draws it", async () => { | |
| 74 | const made = await createRepo(forge, alice, { name: "hugejson", visibility: "public", description: "big" }); | |
| 75 | if (!made.ok) throw new Error("repo failed"); | |
| 76 | const seeded = await commitToBranch(forge, alice, made.value, { | |
| 77 | message: "start", | |
| 78 | files: [{ path: "README.md", content: "# big\n" }], | |
| 79 | }); | |
| 80 | if (!seeded.ok) throw new Error("commit failed"); | |
| 81 | const dir = repoDir(forge.reposRoot, "alice", "hugejson"); | |
| 82 | const git = (args: string[], stdin?: string): string => { | |
| 83 | const run = Bun.spawnSync(["git", ...args], { | |
| 84 | cwd: dir, | |
| 85 | stdin: stdin === undefined ? "ignore" : new TextEncoder().encode(stdin), | |
| 86 | env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 87 | }); | |
| 88 | if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`); | |
| 89 | return run.stdout.toString().trim(); | |
| 90 | }; | |
| 91 | const list = `[${Array(60_000).fill('{"id":1,"name":"padding padding"}').join(",")}]`; | |
| 92 | const blob = git(["hash-object", "-w", "--stdin"], `{"alpha":${list},"beta":${list},"gamma":${list},"small":{"a":1}}`); | |
| 93 | const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${blob}\tdata.json\n`); | |
| 94 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); | |
| 95 | const h = boot(); | |
| 96 | try { | |
| 97 | await h.settle(); | |
| 98 | await open(h, "hugejson"); | |
| 99 | await openFile(h, "data.json"); | |
| 100 | expect(h.text()).toContain("alpha"); | |
| 101 | expect(h.text()).toContain("array"); | |
| 102 | expect(h.text()).toContain("60,000"); | |
| 103 | expect(h.text()).toContain("the file was not parsed"); | |
| 104 | } finally { | |
| 105 | h.stop(); | |
| 106 | } | |
| 107 | }); | |
| 108 | ||
| 73 | 109 | test("a .csv opens as an aligned table with a pinned header, and s flips to the source", async () => { |
| 74 | 110 | const made = await createRepo(forge, alice, { |
| 75 | 111 | name: "ledger", |
new above
offline
Principles 6 and 7: a card is a box around a table. Render the table bare; hairline only where it ends. Also
json-bigclasses are undefined in CSS.