Public: anyone can read this; the crew can change it.
A CSV, TSV or JSON-lines file over 5 MB opens its first 1,000 rows and says how many it did not show #13494
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13465-a-csv-tsv-or-json-lines-file-over-5-mb-o → main 8 commits fork 5 threads · 5 unresolved
10 files
changed +308
−21
docs/features/csv-view.md +8
| @@ -30,3 +30,11 @@ In the terminal, on a `.csv`/`.tsv` open in the workspace, `o` orders the rows: | ||
| 30 | 30 | - A line that does not parse is one row, `Line N: not valid JSON`, or `Line N: not a JSON object` for valid JSON that is not an object (N counts physical lines). The rest of the file still renders. |
| 31 | 31 | - The header line says `JSON Lines` in place of the delimiter. A file with no non-blank lines falls back to the source view. |
| 32 | 32 | - At most 200 columns are drawn; the page says how many there are. |
| 33 | ||
| 34 | ## Files over 5 MB | |
| 35 | ||
| 36 | A table file over the render cap is not refused: a reader that draws (the page, the terminal workspace) asks the source for a prefix (`blob(ref, path, { prefix: true })`); the source reads its first bytes (`CSV_PREFIX_BYTES`) cut at a whole line and marks the blob `partial`. Only a `partial` blob lifts the table's `RENDER_CAP`; an editor or any other reader asks for no prefix and gets `too_large` as before, so a cut file is never saved. | |
| 37 | - The page's cut inside a quoted field drops that record: one pass finds the last line end outside any quote (`safeLines`). | |
| 38 | - The note under the table is one sentence: `Showing N of about M rows. The rest is in the raw file.` M is the prefix's row density over the whole file size, so it is an estimate; the header line says `about M rows`. | |
| 39 | - Sorting and `?all=1` work over the prefix only. Raw and Download still give the whole file. | |
| 40 | - The terminal workspace opens the same prefix for a `.csv`/`.tsv`, says so in a toast (`Showing the first X of Y, read-only. The rest is in the raw file.`) and refuses `e` and the external editor on it. |
src/tui/screens/workspace.ts +24 −3
| @@ -790,6 +790,11 @@ class Tab { | ||
| 790 | 790 | note = ""; |
| 791 | 791 | noteTitle = "Not shown"; |
| 792 | 792 | noteDetail = ""; |
| 793 | /** | |
| 794 | * Non-empty when `raw` is only the start of a larger file: the sentence that says so. It is never | |
| 795 | * edited. | |
| 796 | */ | |
| 797 | cut = ""; | |
| 793 | 798 | raw: Uint8Array | null = null; |
| 794 | 799 | plain: string[] = []; |
| 795 | 800 | lines: Span[][] = []; |
| @@ -1120,6 +1125,7 @@ interface CacheEntry { | ||
| 1120 | 1125 | readonly noteTitle: string; |
| 1121 | 1126 | /** A last line under the note: where to get the file. */ |
| 1122 | 1127 | readonly noteDetail: string; |
| 1128 | readonly cut?: string; | |
| 1123 | 1129 | readonly raw: Uint8Array | null; |
| 1124 | 1130 | readonly picture: Uint8Array | null; |
| 1125 | 1131 | readonly sheet: Table | null; |
| @@ -1396,7 +1402,7 @@ export function workspaceScreen( | ||
| 1396 | 1402 | if (entry === undefined) { |
| 1397 | 1403 | t.loading = true; |
| 1398 | 1404 | ctx.invalidate(); |
| 1399 | const read = await src.blob(ref, t.path); | |
| 1405 | const read = await src.blob(ref, t.path, { prefix: true }); | |
| 1400 | 1406 | if (stale(mine)) return; |
| 1401 | 1407 | let note = ""; |
| 1402 | 1408 | let noteTitle = "Not shown"; |
| @@ -1409,6 +1415,7 @@ export function workspaceScreen( | ||
| 1409 | 1415 | noteDetail = downloadAt(t.path); |
| 1410 | 1416 | }; |
| 1411 | 1417 | let raw: Uint8Array | null = null; |
| 1418 | let cut = ""; | |
| 1412 | 1419 | let picture: Uint8Array | null = null; |
| 1413 | 1420 | let sheet: Table | null = null; |
| 1414 | 1421 | let db: Uint8Array | null = null; |
| @@ -1450,9 +1457,13 @@ export function workspaceScreen( | ||
| 1450 | 1457 | archiveNote(t.path, read.value.body, read.value.size, formatSize) ?? |
| 1451 | 1458 | (kindByName(t.path) === "psd" ? psdNote(read.value.body, read.value.size, formatSize) : undefined) ?? |
| 1452 | 1459 | `Binary file, ${formatSize(read.value.size)}, ${sniffType(read.value.body.subarray(0, HEAD_BYTES))}.`; |
| 1453 | } else raw = read.value.body; // .html included: source here, the page preview is web-only | |
| 1460 | } else { | |
| 1461 | raw = read.value.body; // .html included: source here, the page preview is web-only | |
| 1462 | if (read.value.partial === true) | |
| 1463 | cut = `Showing the first ${formatSize(read.value.body.length)} of ${formatSize(read.value.size)}, read-only. The rest is in the raw file.`; | |
| 1464 | } | |
| 1454 | 1465 | } |
| 1455 | entry = { note, noteTitle, noteDetail, raw, picture, sheet, db }; | |
| 1466 | entry = { note, noteTitle, noteDetail, cut, raw, picture, sheet, db }; | |
| 1456 | 1467 | remember(key, entry); |
| 1457 | 1468 | } |
| 1458 | 1469 | if (stale(mine)) return; |
| @@ -1461,6 +1472,8 @@ export function workspaceScreen( | ||
| 1461 | 1472 | t.noteTitle = entry.noteTitle; |
| 1462 | 1473 | t.noteDetail = entry.noteDetail; |
| 1463 | 1474 | t.raw = entry.raw; |
| 1475 | t.cut = entry.cut ?? ""; | |
| 1476 | if (t.cut !== "") ctx.toast(t.cut, "attn"); | |
| 1464 | 1477 | t.sheet = entry.sheet; |
| 1465 | 1478 | t.db = entry.db; |
| 1466 | 1479 | t.dbAt = { table: 0, page: 1 }; |
| @@ -2056,6 +2069,10 @@ export function workspaceScreen( | ||
| 2056 | 2069 | readOnlyToast(); |
| 2057 | 2070 | return; |
| 2058 | 2071 | } |
| 2072 | if (t.cut !== "") { | |
| 2073 | ctx.toast(t.cut, "attn"); | |
| 2074 | return; | |
| 2075 | } | |
| 2059 | 2076 | if (t.raw === null) { |
| 2060 | 2077 | ctx.toast( |
| 2061 | 2078 | t.image !== null |
| @@ -2866,6 +2883,10 @@ export function workspaceScreen( | ||
| 2866 | 2883 | return; |
| 2867 | 2884 | } |
| 2868 | 2885 | const t = tab(); |
| 2886 | if (t !== undefined && t.cut !== "") { | |
| 2887 | ctx.toast(t.cut, "attn"); | |
| 2888 | return; | |
| 2889 | } | |
| 2869 | 2890 | if (t === undefined || t.raw === null) { |
| 2870 | 2891 | ctx.toast( |
| 2871 | 2892 | t === undefined ? "no file open" : "that file is not text", |
src/view/local.ts +12 −3
| @@ -46,7 +46,7 @@ import { repoDir } from "../git/repos.ts"; | ||
| 46 | 46 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 47 | 47 | import type { Page } from "../origins/types.ts"; |
| 48 | 48 | import { tailable, TAIL_BYTES } from "../web/log_tail.ts"; |
| 49 | import { HEAD_BYTES, RENDER_CAP } from "../web/render/kinds.ts"; | |
| 49 | import { CSV_PREFIX_BYTES, HEAD_BYTES, isTablePath, RENDER_CAP, wholeLines } from "../web/render/kinds.ts"; | |
| 50 | 50 | import type { |
| 51 | 51 | Blob, |
| 52 | 52 | Branch, |
| @@ -208,11 +208,20 @@ export function localSource( | ||
| 208 | 208 | return listed.ok ? ok(listed.value) : err("not_found"); |
| 209 | 209 | }, |
| 210 | 210 | |
| 211 | async blob(ref: string, path: string): Promise<Result<Blob, SourceError>> { | |
| 211 | async blob(ref: string, path: string, opts?: { readonly prefix?: boolean }): Promise<Result<Blob, SourceError>> { | |
| 212 | 212 | if (!isSafeRef(ref) || !isSafeTreePath(path)) return err("not_found"); |
| 213 | const blob = await blobForRender(dir, ref, path, blobCeiling(path)); | |
| 213 | // Only a reader that draws (the page, the terminal) asks for a prefix; an editor never gets | |
| 214 | // one. | |
| 215 | const table = opts?.prefix === true && isTablePath(path); | |
| 216 | const blob = await blobForRender(dir, ref, path, table ? CSV_PREFIX_BYTES : blobCeiling(path)); | |
| 214 | 217 | if (blob.kind === "missing") return err("not_found"); |
| 215 | 218 | if (blob.kind !== "too_large") return ok(blob); |
| 219 | // A table file over the prefix is read as its first bytes only, cut at a line end. | |
| 220 | if (table) { | |
| 221 | const prefix = await blobHead(dir, ref, path, CSV_PREFIX_BYTES); | |
| 222 | if (prefix !== undefined && !prefix.subarray(0, 8000).includes(0)) | |
| 223 | return ok({ kind: "text", body: wholeLines(prefix), size: blob.size, partial: true }); | |
| 224 | } | |
| 216 | 225 | // Too large to read, but its first bytes still say what it is. |
| 217 | 226 | const head = await blobHead(dir, ref, path, HEAD_BYTES); |
| 218 | 227 | const tail = tailable(path) ? await blobTail(dir, ref, path, TAIL_BYTES, blob.size) : undefined; |
src/view/source.ts +11 −2
| @@ -115,7 +115,16 @@ export interface Tag { | ||
| 115 | 115 | * than one to invent. |
| 116 | 116 | */ |
| 117 | 117 | export type Blob = |
| 118 | | { readonly kind: "text"; readonly body: Uint8Array; readonly size: number } | |
| 118 | | { | |
| 119 | readonly kind: "text"; | |
| 120 | readonly body: Uint8Array; | |
| 121 | readonly size: number; | |
| 122 | /** | |
| 123 | * The body is only the start of a larger file, cut at a line end (a table file over its | |
| 124 | * prefix). | |
| 125 | */ | |
| 126 | readonly partial?: boolean; | |
| 127 | } | |
| 119 | 128 | | { |
| 120 | 129 | readonly kind: "binary"; |
| 121 | 130 | readonly body: Uint8Array; |
| @@ -249,7 +258,7 @@ export interface RepoSource { | ||
| 249 | 258 | tree(ref: string, path: string): Promise<Result<TreeEntry[], SourceError>>; |
| 250 | 259 | /** Every file path under `ref`, flat, for a fuzzy finder. Capped and cached. */ |
| 251 | 260 | paths(ref: string): Promise<Result<PathListing, SourceError>>; |
| 252 | blob(ref: string, path: string): Promise<Result<Blob, SourceError>>; | |
| 261 | blob(ref: string, path: string, opts?: { readonly prefix?: boolean }): Promise<Result<Blob, SourceError>>; | |
| 253 | 262 | readme( |
| 254 | 263 | ref: string, |
| 255 | 264 | ): Promise<Result<{ path: string; markdown: string } | null, SourceError>>; |
src/web/pages/repo.ts +5 −2
| @@ -103,7 +103,7 @@ import { fontJs } from "../render/font.ts"; | ||
| 103 | 103 | import { INLINE_MAX, keepGcode, keptGcode, layerCount } from "../render/gcode.ts"; |
| 104 | 104 | import { hasTable } from "../render/sqlite.ts"; |
| 105 | 105 | import { READ_CEILING } from "../render/psd-read.ts"; |
| 106 | import { fileKind, HEAD_BYTES, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts"; | |
| 106 | import { CSV_PREFIX_BYTES, fileKind, HEAD_BYTES, isTablePath, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts"; | |
| 107 | 107 | import { RENDERERS, rendererFor } from "../render/renderers.ts"; |
| 108 | 108 | import { downloadHref, mediaHref } from "./media.ts"; |
| 109 | 109 | import { browseBase, localScope, type RepoScope } from "../scope.ts"; |
| @@ -1241,6 +1241,7 @@ export function renderBlob( | ||
| 1241 | 1241 | path: o.path, |
| 1242 | 1242 | size: blob.size, |
| 1243 | 1243 | body: renderer.body, |
| 1244 | ...(blob.kind === "text" && blob.partial === true ? { partial: true } : {}), | |
| 1244 | 1245 | media: mediaHref(base, o.ref, o.path), |
| 1245 | 1246 | download: downloadHref(base, o.ref, o.path), |
| 1246 | 1247 | pathHref: `${base}/blob/${encoded}/${o.path}`, |
| @@ -1848,7 +1849,7 @@ export async function blobPage(ctx: Ctx): Promise<Response> { | ||
| 1848 | 1849 | const hit = cachedBlobPage(repo.id, ref, path, viewKey(sort), all); |
| 1849 | 1850 | if (hit !== undefined) return keptHit(hit); |
| 1850 | 1851 | } |
| 1851 | const blob = await git.read(() => scope.source.blob(ref, path)); | |
| 1852 | const blob = await git.read(() => scope.source.blob(ref, path, { prefix: true })); | |
| 1852 | 1853 | if (!blob.ok) return notFoundPage(ctx); |
| 1853 | 1854 | // A stranger's file page reads its folders and ref names through what other strangers' pages kept |
| 1854 | 1855 | // (#9514). |
| @@ -1985,6 +1986,8 @@ export function capped(blob: Blob, path = ""): Blob { | ||
| 1985 | 1986 | return blob; |
| 1986 | 1987 | if (blob.kind !== "too_large" && blob.size <= READ_CEILING && fileKind(path, blob.body.subarray(0, HEAD_BYTES)) === "psd") |
| 1987 | 1988 | return blob; |
| 1989 | // A table file's body is already bounded by its prefix; the renderer draws a window of it. | |
| 1990 | if (blob.kind !== "too_large" && blob.body.length <= CSV_PREFIX_BYTES && isTablePath(path)) return blob; | |
| 1988 | 1991 | return blob.kind !== "too_large" && blob.size > MAX_RENDER_BYTES |
| 1989 | 1992 | ? { kind: "too_large", size: blob.size, head: blob.body.slice(0, HEAD_BYTES) } |
| 1990 | 1993 | : blob; |
src/web/render/csv.ts +31 −10
| @@ -50,23 +50,41 @@ export function sortColumn(text: string, path: string, sort: string | undefined) | ||
| 50 | 50 | return head.rows[0]!.slice(0, COLUMN_CAP).includes(sort) ? sort : undefined; |
| 51 | 51 | } |
| 52 | 52 | |
| 53 | /** | |
| 54 | * A prefix cut back to its last line end outside any quoted field (quotes counted in one pass; an | |
| 55 | * escaped quote is two), so a cut inside a field drops that record instead of failing the table. | |
| 56 | */ | |
| 57 | export function safeLines(bytes: Uint8Array): Uint8Array { | |
| 58 | let open = false; | |
| 59 | let end = 0; | |
| 60 | for (let i = 0; i < bytes.length; i++) { | |
| 61 | const b = bytes[i]!; | |
| 62 | if (b === 34) open = !open; | |
| 63 | else if (b === 10 && !open) end = i + 1; | |
| 64 | } | |
| 65 | return bytes.subarray(0, end === 0 ? bytes.length : end); | |
| 66 | } | |
| 67 | ||
| 53 | 68 | export function csv(input: RenderInput): Rendered { |
outdated
Up to 64 full re-parses of a 2 MB prefix is slow in the worst case. Find the last safe line boundary in one pass. |
||
| 54 | 69 | const want = input.all ? ROW_CAP : ROW_WINDOW; |
| 70 | const partial = input.partial === true; | |
| 71 | const lines = isJsonlPath(input.path); | |
| 72 | // A prefix is cut where no quoted field is open, in one pass, so it parses whole. | |
| 73 | const bytes = partial && !lines ? safeLines(input.body) : input.body; | |
| 74 | const text = new TextDecoder().decode(bytes); | |
| 55 | 75 | // A sort needs every row the page may draw, so the window is cut after it; a sort that names no |
| 56 | 76 | // column is no sort and reads no more than a page without one. |
| 57 | const text = new TextDecoder().decode(input.body); | |
| 58 | 77 | const sort = sortColumn(text, input.path, input.sort); |
| 59 | 78 | const keep = sort === undefined ? want : ROW_CAP; |
| 60 | const lines = isJsonlPath(input.path); | |
| 61 | const parsed = lines | |
| 62 | ? parseJsonl(text, keep, COLUMN_CAP) | |
| 63 | : parseCsv(text, { rows: keep + 1, columns: COLUMN_CAP }); | |
| 79 | const parsed = lines ? parseJsonl(text, keep, COLUMN_CAP) : parseCsv(text, { rows: keep + 1, columns: COLUMN_CAP }); | |
| 64 | 80 | if ("error" in parsed) return { content: html``, lines: 0, fallback: parsed.error }; |
| 65 | const total = parsed.records - 1; | |
| 81 | // The rows in the prefix are exact; the file's are the prefix's density over its whole size. | |
| 82 | const inPrefix = parsed.records - 1; | |
| 83 | const total = partial ? Math.max(inPrefix, Math.round((inPrefix * input.size) / Math.max(1, bytes.length))) : inPrefix; | |
| 66 | 84 | const width = Math.min(parsed.columns, COLUMN_CAP); |
| 67 | 85 | const numeric = numericColumns(parsed, width); |
| 68 | 86 | // A wide table shrinks its row window so the page stays bounded. |
| 69 | const shown = Math.min(total, want, Math.floor(CELL_CAP / width)); | |
| 87 | const shown = Math.min(inPrefix, want, Math.floor(CELL_CAP / width)); | |
| 70 | 88 | const [head, ...unsorted] = parsed.rows; |
| 71 | 89 | // `?sort=` must name a header cell exactly; anything else is ignored. |
| 72 | 90 | const column = sort === undefined ? -1 : head!.slice(0, width).indexOf(sort); |
| @@ -89,8 +107,11 @@ export function csv(input: RenderInput): Rendered { | ||
| 89 | 107 | parsed.columns > width |
| 90 | 108 | ? `Showing ${width.toLocaleString()} of ${parsed.columns.toLocaleString()} columns. ` |
| 91 | 109 | : ""; |
| 92 | const more = | |
| 93 | shown >= total && wide === "" | |
| 110 | const count = (n: number): string => n.toLocaleString(); | |
| 111 | const ofRows = partial ? (inPrefix > shown ? `of about ${count(total)}` : `of more than ${count(shown)}`) : `of ${count(total)}`; | |
| 112 | const more = partial | |
| 113 | ? `<p class="muted small csv-more">${wide}Showing ${count(shown)} ${ofRows} rows. The rest is in the raw file.</p>` | |
| 114 | : shown >= total && wide === "" | |
| 94 | 115 | ? "" |
| 95 | 116 | : `<p class="muted small csv-more">${wide}${ |
| 96 | 117 | shown >= total ? "" : `Showing ${shown.toLocaleString()} of ${total.toLocaleString()} rows. ` |
| @@ -119,7 +140,7 @@ export function csv(input: RenderInput): Rendered { | ||
| 119 | 140 | return { |
| 120 | 141 | content, |
| 121 | 142 | lines: 0, |
| 122 | summary: `${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${lines ? "JSON Lines" : `${NAMES[parsed.delimiter]}-separated`} · ${fileSize(input.size)}`, | |
| 143 | summary: `${partial ? "about " : ""}${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${lines ? "JSON Lines" : `${NAMES[parsed.delimiter]}-separated`} · ${fileSize(input.size)}`, | |
| 123 | 144 | }; |
| 124 | 145 | } |
| 125 | 146 |
src/web/render/kinds.ts +13
| @@ -91,6 +91,7 @@ export const RENDER_CAP: Readonly<Record<FileKind, number>> = { | ||
| 91 | 91 | html: 2 * MB, |
| 92 | 92 | audio: 500 * MB, |
| 93 | 93 | video: 4_000 * MB, |
| 94 | // Past it only a source that cuts a prefix (`Blob.partial`) is drawn, from that prefix. | |
| 94 | 95 | csv: 10 * MB, |
outdated
A 4000 MB cap applies to every source, not only the one that cuts a prefix. Scope it to partial bodies. |
||
| 95 | 96 | // Under the source's own 5 MB read ceiling: a larger file is never read. |
| 96 | 97 | sqlite: 5 * MB, |
| @@ -369,6 +370,18 @@ const isStepHead = (head: Uint8Array): boolean => | ||
| 369 | 370 | /** For a caller that has only a name: the kind its extension gives. */ |
| 370 | 371 | export const NO_BYTES: Uint8Array = new Uint8Array(0); |
| 371 | 372 | |
| 373 | /** The bytes of a CSV, TSV or JSON-lines file the page reads; the rest is counted by estimate. */ | |
| 374 | export const CSV_PREFIX_BYTES = 2 * MB; | |
| 375 | ||
| 376 | /** Whether a name is a table file the page draws from a prefix. */ | |
| 377 | export const isTablePath = (path: string): boolean => /\.(csv|tsv|jsonl|ndjson)$/i.test(path); | |
| 378 | ||
| 379 | /** A prefix cut back to its last line end, so no half line is drawn; whole when it has none. */ | |
| 380 | export function wholeLines(prefix: Uint8Array): Uint8Array { | |
| 381 | const end = prefix.lastIndexOf(10); | |
| 382 | return end === -1 ? prefix : prefix.subarray(0, end + 1); | |
| 383 | } | |
| 384 | ||
| 372 | 385 | /** How many leading bytes the registry needs to decide. */ |
| 373 | 386 | export const HEAD_BYTES = 4096; |
| 374 | 387 |
src/web/render/renderers.ts +5 −1
| @@ -79,6 +79,8 @@ export interface RenderInput { | ||
| 79 | 79 | * these. |
| 80 | 80 | */ |
| 81 | 81 | readonly exists?: (path: string) => boolean; |
| 82 | /** The body is only the start of the file, cut at a line end: the row count is an estimate. */ | |
| 83 | readonly partial?: boolean; | |
| 82 | 84 | /** `?all=1`: draw past the default window, up to the renderer's cap. */ |
| 83 | 85 | readonly all?: boolean; |
| 84 | 86 | /** |
| @@ -204,7 +206,9 @@ export function rendererFor( | ||
| 204 | 206 | : blob.head !== undefined |
| 205 | 207 | ? fileKind(path, blob.head) |
| 206 | 208 | : kindByName(path); |
| 207 | if (blob.size > RENDER_CAP[kind]) return undefined; | |
| 209 | // A table's cap is lifted only for a body the source already cut to a prefix. | |
| 210 | const prefixed = kind === "csv" && blob.kind === "text" && blob.partial === true; | |
| 211 | if (blob.size > RENDER_CAP[kind] && !prefixed) return undefined; | |
| 208 | 212 | if (MEDIA_KINDS.has(kind) && !where.media) return undefined; |
| 209 | 213 | if (blob.kind === "too_large" && !MEDIA_KINDS.has(kind) && kind !== "psd") return undefined; |
| 210 | 214 | // A kind may draw only some of its files: `config` draws JSON, YAML and |
test/render-csv-prefix.test.ts added +160
| @@ -0,0 +1,160 @@ | ||
| 1 | /** | |
| 2 | * A table file over the prefix opens its first 1,000 rows from a bounded read, and says what it cut | |
| 3 | * (#13465). | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 11 | import type { Repo } from "../src/core/repos.ts"; | |
| 12 | import { repoDir } from "../src/git/repos.ts"; | |
| 13 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 14 | import { rendererFor } from "../src/web/render/renderers.ts"; | |
| 15 | import { localSource } from "../src/view/local.ts"; | |
| 16 | import { csv, safeLines } from "../src/web/render/csv.ts"; | |
| 17 | import { CSV_PREFIX_BYTES, RENDER_CAP, wholeLines } from "../src/web/render/kinds.ts"; | |
| 18 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 19 | import { quietLogs } from "./helpers/globals.ts"; | |
| 20 | ||
| 21 | quietLogs(); | |
| 22 | ||
| 23 | const enc = (s: string): Uint8Array => new TextEncoder().encode(s); | |
| 24 | ||
| 25 | /** About 20 MB of rows, built as one buffer. */ | |
| 26 | const generate = (make: (i: number) => string, bytes: number): Uint8Array => { | |
| 27 | const parts: string[] = []; | |
| 28 | let size = 0; | |
| 29 | for (let i = 0; size < bytes; i++) { | |
| 30 | const line = make(i); | |
| 31 | parts.push(line); | |
| 32 | size += line.length; | |
| 33 | } | |
| 34 | return enc(parts.join("")); | |
| 35 | }; | |
| 36 | const CSV = generate((i) => (i === 0 ? "id,name,score\n" : `${i},user-${i},${(i * 7) % 100}\n`), 20_000_000); | |
| 37 | const JSONL = generate((i) => `{"id":${i},"name":"user-${i}","tags":["a","b"]}\n`, 20_000_000); | |
| 38 | ||
| 39 | describe("the prefix cut", () => { | |
| 40 | test("drops the final partial line", () => { | |
| 41 | expect(new TextDecoder().decode(wholeLines(enc("a,b\n1,2\n3,")))).toBe("a,b\n1,2\n"); | |
| 42 | expect(wholeLines(enc("no newline")).length).toBe(10); | |
| 43 | }); | |
| 44 | ||
| 45 | test("a partial body says 'of about N' with the estimate from the prefix density", () => { | |
| 46 | const rows = "a,b\n" + "1,2\n".repeat(5_000); | |
| 47 | const text = enc(rows); | |
| 48 | const out = csv({ path: "d.csv", size: text.length * 10, body: text, partial: true, media: "/m", download: "/d" }); | |
| 49 | const page = String(out.content); | |
| 50 | expect(page).toContain("Showing 1,000 of about 50,000 rows"); | |
| 51 | expect(out.summary).toContain("about 50,000 rows"); | |
| 52 | expect(page).not.toContain("?all=1"); | |
| 53 | }); | |
| 54 | ||
| 55 | test("a prefix that ends inside the window says 'more than'", () => { | |
| 56 | const text = enc("a,b\n" + "1,2\n".repeat(100)); | |
| 57 | const out = csv({ path: "d.csv", size: 50_000_000, body: text, partial: true, media: "/m", download: "/d" }); | |
| 58 | expect(String(out.content)).toContain("Showing 100 of more than 100 rows"); | |
| 59 | const wide = enc("a,b\n" + "1,2\n".repeat(1_000)); | |
| 60 | const exact = csv({ path: "d.csv", size: 50_000_000, body: wide, partial: true, media: "/m", download: "/d" }); | |
| 61 | expect(String(exact.content)).toContain("of more than 1,000 rows"); | |
| 62 | }); | |
| 63 | ||
| 64 | test("the note is one sentence: no file size, no bytes read", () => { | |
| 65 | const text = enc("a,b\n" + "1,2\n".repeat(5_000)); | |
| 66 | const page = String(csv({ path: "d.csv", size: text.length * 10, body: text, partial: true, media: "/m", download: "/d" }).content); | |
| 67 | expect(page).toContain("Showing 1,000 of about 50,000 rows. The rest is in the raw file."); | |
| 68 | expect(page).not.toContain("only its first"); | |
| 69 | }); | |
| 70 | ||
| 71 | test("the cap is lifted only for a body the source cut", () => { | |
| 72 | const big = RENDER_CAP.csv + 1; | |
| 73 | const body = enc("a,b\n1,2\n"); | |
| 74 | expect(rendererFor("d.csv", { kind: "text", body, size: big })).toBeUndefined(); | |
| 75 | expect(rendererFor("d.csv", { kind: "text", body, size: big, partial: true })).toBeDefined(); | |
| 76 | }); | |
| 77 | ||
| 78 | test("safeLines cuts at the last line end outside a quoted field, in one pass", () => { | |
| 79 | const dec = (b: Uint8Array) => new TextDecoder().decode(b); | |
| 80 | expect(dec(safeLines(enc('a\n"x\ny"\n"o\np')))).toBe('a\n"x\ny"\n'); | |
| 81 | expect(dec(safeLines(enc('a\n"q ""x"" \n"\nz')))).toBe('a\n"q ""x"" \n"\n'); | |
| 82 | }); | |
| 83 | ||
| 84 | test("a cut inside a quoted field still draws the rows before it", () => { | |
| 85 | const text = enc('a,b\n1,2\n3,"open\nstill open'); | |
| 86 | const out = csv({ path: "d.csv", size: 9_000_000, body: text, partial: true, media: "/m", download: "/d" }); | |
| 87 | expect(out.fallback).toBeUndefined(); | |
| 88 | expect(String(out.content)).toContain(`<td class="num">1</td>`); | |
| 89 | }); | |
| 90 | }); | |
| 91 | ||
| 92 | describe("a 20 MB table file", () => { | |
| 93 | let dataDir: string; | |
| 94 | let app: App; | |
| 95 | let servers: Servers; | |
| 96 | let base: string; | |
| 97 | let repo: Repo; | |
| 98 | ||
| 99 | beforeAll(async () => { | |
| 100 | dataDir = await mkdtemp(join(tmpdir(), "forge-csv-prefix-")); | |
| 101 | app = createApp({ dataDir, port: 0 }); | |
| 102 | servers = await serve(app); | |
| 103 | base = servers.url; | |
| 104 | const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-prefix-passphrase" }); | |
| 105 | if (!a.ok) throw new Error("signup failed"); | |
| 106 | const created = await createRepo(app, a.value, { name: "data", visibility: "public" }); | |
| 107 | if (!created.ok) throw new Error("repo failed"); | |
| 108 | repo = created.value; | |
| 109 | const made = await commitToBranch(app, a.value, repo, { message: "start", files: [{ path: "README.md", content: "# d\n" }] }); | |
| 110 | if (!made.ok) throw new Error("commit failed"); | |
| 111 | const dir = repoDir(app.reposRoot, "alice", "data"); | |
| 112 | const git = (args: string[], stdin?: Uint8Array | string): string => { | |
| 113 | const run = Bun.spawnSync(["git", ...args], { | |
| 114 | cwd: dir, | |
| 115 | stdin: typeof stdin === "string" ? enc(stdin) : (stdin ?? "ignore"), | |
| 116 | env: { ...process.env, GIT_AUTHOR_NAME: "alice", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "alice", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 117 | }); | |
| 118 | if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`); | |
| 119 | return run.stdout.toString().trim(); | |
| 120 | }; | |
| 121 | const lines = [git(["ls-tree", "main"])]; | |
| 122 | for (const [name, body] of [["big.csv", CSV], ["big.jsonl", JSONL]] as const) | |
| 123 | lines.push(`100644 blob ${git(["hash-object", "-w", "--stdin"], body)}\t${name}`); | |
| 124 | const tree = git(["mktree"], lines.join("\n") + "\n"); | |
| 125 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); | |
| 126 | }, 60_000); | |
| 127 | ||
| 128 | afterAll(async () => { | |
| 129 | await servers?.stop(); | |
| 130 | app?.close(); | |
| 131 | await rm(dataDir, { recursive: true, force: true }); | |
| 132 | }); | |
| 133 | ||
| 134 | test("the source hands over a bounded prefix ending on a line", async () => { | |
| 135 | const read = await localSource(app, repo).blob("main", "big.csv", { prefix: true }); | |
| 136 | if (!read.ok || read.value.kind !== "text") throw new Error("expected a text prefix"); | |
| 137 | expect(read.value.partial).toBe(true); | |
| 138 | expect(read.value.size).toBe(CSV.length); | |
| 139 | expect(read.value.body.length).toBeLessThanOrEqual(CSV_PREFIX_BYTES); | |
| 140 | expect(read.value.body[read.value.body.length - 1]).toBe(10); | |
| 141 | }); | |
| 142 | ||
| 143 | test("a reader that asks for no prefix (an editor) still gets too_large", async () => { | |
| 144 | const read = await localSource(app, repo).blob("main", "big.csv"); | |
| 145 | expect(read.ok && read.value.kind).toBe("too_large"); | |
| 146 | }); | |
| 147 | ||
| 148 | for (const name of ["big.csv", "big.jsonl"]) { | |
| 149 | test(`${name} opens its first 1,000 rows, says it cut the rest, and `, async () => { | |
| 150 | const res = await fetch(`${base}/alice/data/blob/main/${name}`); | |
| 151 | const text = await res.text(); | |
| 152 | // Bytes, not clocks: the page is a window of a bounded prefix, never the 20 MB file. | |
| 153 | expect(text.length).toBeLessThan(2_000_000); | |
| 154 | expect(res.status).toBe(200); | |
| 155 | expect(text).toMatch(/Showing 1,000 of about [\d,]+ rows/); | |
| 156 | expect(text).toContain("user-999"); | |
| 157 | expect(text).not.toContain("user-1001<"); | |
Non-blocking: the test title ends in 'and ' with nothing after it. Finish it, e.g. 'and shows no row past the window'. |
||
| 158 | }); | |
| 159 | } | |
| 160 | }); |
test/tui-workspace-formats.test.ts +39
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | import { Database } from "bun:sqlite"; |
| 9 | 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 10 | 10 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 11 | import { repoDir } from "../src/git/repos.ts"; | |
| 11 | 12 | import type { App as Forge } from "../src/app.ts"; |
| 12 | 13 | import type { User } from "../src/core/users.ts"; |
| 13 | 14 | import { zip } from "./helpers/zip.ts"; |
| @@ -31,6 +32,44 @@ describe("the terminal workspace, by file type", () => { | ||
| 31 | 32 | |
| 32 | 33 | afterAll(() => fx.close()); |
| 33 | 34 | |
| 35 | test("a .csv over the page's size limit opens its first rows, read-only, with the page's note", async () => { | |
| 36 | const made = await createRepo(forge, alice, { name: "hugetable", visibility: "public", description: "big" }); | |
| 37 | if (!made.ok) throw new Error("repo failed"); | |
| 38 | const seeded = await commitToBranch(forge, alice, made.value, { | |
| 39 | message: "start", | |
| 40 | files: [{ path: "README.md", content: "# big\n" }], | |
| 41 | }); | |
| 42 | if (!seeded.ok) throw new Error("commit failed"); | |
| 43 | // Over the 2 MB a commit through the app takes: written with git itself, as a push would. | |
| 44 | const dir = repoDir(forge.reposRoot, "alice", "hugetable"); | |
| 45 | const git = (args: string[], stdin?: string): string => { | |
| 46 | const run = Bun.spawnSync(["git", ...args], { | |
| 47 | cwd: dir, | |
| 48 | stdin: stdin === undefined ? "ignore" : new TextEncoder().encode(stdin), | |
| 49 | env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 50 | }); | |
| 51 | if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`); | |
| 52 | return run.stdout.toString().trim(); | |
| 53 | }; | |
| 54 | const rows = Array.from({ length: 200_000 }, (_, i) => `${i + 1},item ${i + 1},padding-padding-padding`); | |
| 55 | const blob = git(["hash-object", "-w", "--stdin"], ["id,name,pad", ...rows, ""].join("\n")); | |
| 56 | const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${blob}\tbig.csv\n`); | |
| 57 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); | |
| 58 | const h = boot(); | |
| 59 | try { | |
| 60 | await h.settle(); | |
| 61 | await open(h, "hugetable"); | |
| 62 | await openFile(h, "big.csv"); | |
| 63 | expect(h.text()).toContain("id"); | |
| 64 | expect(h.text()).toContain("item 1"); | |
| 65 | expect(h.text()).toContain("read-only"); | |
| 66 | await h.press("e"); | |
| 67 | expect(h.text()).toContain("read-only"); | |
| 68 | } finally { | |
| 69 | h.stop(); | |
| 70 | } | |
| 71 | }); | |
| 72 | ||
| 34 | 73 | test("a .csv opens as an aligned table with a pinned header, and s flips to the source", async () => { |
| 35 | 74 | const made = await createRepo(forge, alice, { |
| 36 | 75 | name: "ledger", |
new above
offline
Both doors: the terminal must open the prefix too. Add the src/tui/ change and drop this line.