Public: anyone can read this; the crew can change it.

A CSV, TSV or JSON-lines file over 5 MB opens its first 1,000 rows and says how many it did not show #13494

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13465-a-csv-tsv-or-json-lines-file-over-5-mb-o → main 8 commits fork 5 threads · 5 unresolved

  • Merged. scout/13465-a-csv-tsv-or-json-lines-file-over-5-mb-o was merged into main as 5d89b28.

Review

Sign in to review this change.
10 files changed +308 −21
docs/features/csv-view.md +8
@@ -30,3 +30,11 @@ In the terminal, on a `.csv`/`.tsv` open in the workspace, `o` orders the rows:
3030 - A line that does not parse is one row, `Line N: not valid JSON`, or `Line N: not a JSON object` for valid JSON that is not an object (N counts physical lines). The rest of the file still renders.
3131 - The header line says `JSON Lines` in place of the delimiter. A file with no non-blank lines falls back to the source view.
3232 - At most 200 columns are drawn; the page says how many there are.
33
34 ## Files over 5 MB
35
36 A table file over the render cap is not refused: a reader that draws (the page, the terminal workspace) asks the source for a prefix (`blob(ref, path, { prefix: true })`); the source reads its first bytes (`CSV_PREFIX_BYTES`) cut at a whole line and marks the blob `partial`. Only a `partial` blob lifts the table's `RENDER_CAP`; an editor or any other reader asks for no prefix and gets `too_large` as before, so a cut file is never saved.
37 - The page's cut inside a quoted field drops that record: one pass finds the last line end outside any quote (`safeLines`).
outdated

Both doors: the terminal must open the prefix too. Add the src/tui/ change and drop this line.

38 - The note under the table is one sentence: `Showing N of about M rows. The rest is in the raw file.` M is the prefix's row density over the whole file size, so it is an estimate; the header line says `about M rows`.
39 - Sorting and `?all=1` work over the prefix only. Raw and Download still give the whole file.
40 - The terminal workspace opens the same prefix for a `.csv`/`.tsv`, says so in a toast (`Showing the first X of Y, read-only. The rest is in the raw file.`) and refuses `e` and the external editor on it.
src/tui/screens/workspace.ts +24 −3
@@ -790,6 +790,11 @@ class Tab {
790790 note = "";
791791 noteTitle = "Not shown";
792792 noteDetail = "";
793 /**
794 * Non-empty when `raw` is only the start of a larger file: the sentence that says so. It is never
795 * edited.
796 */
797 cut = "";
793798 raw: Uint8Array | null = null;
794799 plain: string[] = [];
795800 lines: Span[][] = [];
@@ -1120,6 +1125,7 @@ interface CacheEntry {
11201125 readonly noteTitle: string;
11211126 /** A last line under the note: where to get the file. */
11221127 readonly noteDetail: string;
1128 readonly cut?: string;
11231129 readonly raw: Uint8Array | null;
11241130 readonly picture: Uint8Array | null;
11251131 readonly sheet: Table | null;
@@ -1396,7 +1402,7 @@ export function workspaceScreen(
13961402 if (entry === undefined) {
13971403 t.loading = true;
13981404 ctx.invalidate();
1399 const read = await src.blob(ref, t.path);
1405 const read = await src.blob(ref, t.path, { prefix: true });
14001406 if (stale(mine)) return;
14011407 let note = "";
14021408 let noteTitle = "Not shown";
@@ -1409,6 +1415,7 @@ export function workspaceScreen(
14091415 noteDetail = downloadAt(t.path);
14101416 };
14111417 let raw: Uint8Array | null = null;
1418 let cut = "";
14121419 let picture: Uint8Array | null = null;
14131420 let sheet: Table | null = null;
14141421 let db: Uint8Array | null = null;
@@ -1450,9 +1457,13 @@ export function workspaceScreen(
14501457 archiveNote(t.path, read.value.body, read.value.size, formatSize) ??
14511458 (kindByName(t.path) === "psd" ? psdNote(read.value.body, read.value.size, formatSize) : undefined) ??
14521459 `Binary file, ${formatSize(read.value.size)}, ${sniffType(read.value.body.subarray(0, HEAD_BYTES))}.`;
1453 } else raw = read.value.body; // .html included: source here, the page preview is web-only
1460 } else {
1461 raw = read.value.body; // .html included: source here, the page preview is web-only
1462 if (read.value.partial === true)
1463 cut = `Showing the first ${formatSize(read.value.body.length)} of ${formatSize(read.value.size)}, read-only. The rest is in the raw file.`;
1464 }
14541465 }
1455 entry = { note, noteTitle, noteDetail, raw, picture, sheet, db };
1466 entry = { note, noteTitle, noteDetail, cut, raw, picture, sheet, db };
14561467 remember(key, entry);
14571468 }
14581469 if (stale(mine)) return;
@@ -1461,6 +1472,8 @@ export function workspaceScreen(
14611472 t.noteTitle = entry.noteTitle;
14621473 t.noteDetail = entry.noteDetail;
14631474 t.raw = entry.raw;
1475 t.cut = entry.cut ?? "";
1476 if (t.cut !== "") ctx.toast(t.cut, "attn");
14641477 t.sheet = entry.sheet;
14651478 t.db = entry.db;
14661479 t.dbAt = { table: 0, page: 1 };
@@ -2056,6 +2069,10 @@ export function workspaceScreen(
20562069 readOnlyToast();
20572070 return;
20582071 }
2072 if (t.cut !== "") {
2073 ctx.toast(t.cut, "attn");
2074 return;
2075 }
20592076 if (t.raw === null) {
20602077 ctx.toast(
20612078 t.image !== null
@@ -2866,6 +2883,10 @@ export function workspaceScreen(
28662883 return;
28672884 }
28682885 const t = tab();
2886 if (t !== undefined && t.cut !== "") {
2887 ctx.toast(t.cut, "attn");
2888 return;
2889 }
28692890 if (t === undefined || t.raw === null) {
28702891 ctx.toast(
28712892 t === undefined ? "no file open" : "that file is not text",
src/view/local.ts +12 −3
@@ -46,7 +46,7 @@ import { repoDir } from "../git/repos.ts";
4646 import { err, ok, type Result } from "../kernel/result.ts";
4747 import type { Page } from "../origins/types.ts";
4848 import { tailable, TAIL_BYTES } from "../web/log_tail.ts";
49 import { HEAD_BYTES, RENDER_CAP } from "../web/render/kinds.ts";
49 import { CSV_PREFIX_BYTES, HEAD_BYTES, isTablePath, RENDER_CAP, wholeLines } from "../web/render/kinds.ts";
5050 import type {
5151 Blob,
5252 Branch,
@@ -208,11 +208,20 @@ export function localSource(
208208 return listed.ok ? ok(listed.value) : err("not_found");
209209 },
210210
211 async blob(ref: string, path: string): Promise<Result<Blob, SourceError>> {
211 async blob(ref: string, path: string, opts?: { readonly prefix?: boolean }): Promise<Result<Blob, SourceError>> {
212212 if (!isSafeRef(ref) || !isSafeTreePath(path)) return err("not_found");
213 const blob = await blobForRender(dir, ref, path, blobCeiling(path));
213 // Only a reader that draws (the page, the terminal) asks for a prefix; an editor never gets
214 // one.
215 const table = opts?.prefix === true && isTablePath(path);
216 const blob = await blobForRender(dir, ref, path, table ? CSV_PREFIX_BYTES : blobCeiling(path));
214217 if (blob.kind === "missing") return err("not_found");
215218 if (blob.kind !== "too_large") return ok(blob);
219 // A table file over the prefix is read as its first bytes only, cut at a line end.
220 if (table) {
221 const prefix = await blobHead(dir, ref, path, CSV_PREFIX_BYTES);
222 if (prefix !== undefined && !prefix.subarray(0, 8000).includes(0))
223 return ok({ kind: "text", body: wholeLines(prefix), size: blob.size, partial: true });
224 }
216225 // Too large to read, but its first bytes still say what it is.
217226 const head = await blobHead(dir, ref, path, HEAD_BYTES);
218227 const tail = tailable(path) ? await blobTail(dir, ref, path, TAIL_BYTES, blob.size) : undefined;
src/view/source.ts +11 −2
@@ -115,7 +115,16 @@ export interface Tag {
115115 * than one to invent.
116116 */
117117 export type Blob =
118 | { readonly kind: "text"; readonly body: Uint8Array; readonly size: number }
118 | {
119 readonly kind: "text";
120 readonly body: Uint8Array;
121 readonly size: number;
122 /**
123 * The body is only the start of a larger file, cut at a line end (a table file over its
124 * prefix).
125 */
126 readonly partial?: boolean;
127 }
119128 | {
120129 readonly kind: "binary";
121130 readonly body: Uint8Array;
@@ -249,7 +258,7 @@ export interface RepoSource {
249258 tree(ref: string, path: string): Promise<Result<TreeEntry[], SourceError>>;
250259 /** Every file path under `ref`, flat, for a fuzzy finder. Capped and cached. */
251260 paths(ref: string): Promise<Result<PathListing, SourceError>>;
252 blob(ref: string, path: string): Promise<Result<Blob, SourceError>>;
261 blob(ref: string, path: string, opts?: { readonly prefix?: boolean }): Promise<Result<Blob, SourceError>>;
253262 readme(
254263 ref: string,
255264 ): Promise<Result<{ path: string; markdown: string } | null, SourceError>>;
src/web/pages/repo.ts +5 −2
@@ -103,7 +103,7 @@ import { fontJs } from "../render/font.ts";
103103 import { INLINE_MAX, keepGcode, keptGcode, layerCount } from "../render/gcode.ts";
104104 import { hasTable } from "../render/sqlite.ts";
105105 import { READ_CEILING } from "../render/psd-read.ts";
106 import { fileKind, HEAD_BYTES, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts";
106 import { CSV_PREFIX_BYTES, fileKind, HEAD_BYTES, isTablePath, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts";
107107 import { RENDERERS, rendererFor } from "../render/renderers.ts";
108108 import { downloadHref, mediaHref } from "./media.ts";
109109 import { browseBase, localScope, type RepoScope } from "../scope.ts";
@@ -1241,6 +1241,7 @@ export function renderBlob(
12411241 path: o.path,
12421242 size: blob.size,
12431243 body: renderer.body,
1244 ...(blob.kind === "text" && blob.partial === true ? { partial: true } : {}),
12441245 media: mediaHref(base, o.ref, o.path),
12451246 download: downloadHref(base, o.ref, o.path),
12461247 pathHref: `${base}/blob/${encoded}/${o.path}`,
@@ -1848,7 +1849,7 @@ export async function blobPage(ctx: Ctx): Promise<Response> {
18481849 const hit = cachedBlobPage(repo.id, ref, path, viewKey(sort), all);
18491850 if (hit !== undefined) return keptHit(hit);
18501851 }
1851 const blob = await git.read(() => scope.source.blob(ref, path));
1852 const blob = await git.read(() => scope.source.blob(ref, path, { prefix: true }));
18521853 if (!blob.ok) return notFoundPage(ctx);
18531854 // A stranger's file page reads its folders and ref names through what other strangers' pages kept
18541855 // (#9514).
@@ -1985,6 +1986,8 @@ export function capped(blob: Blob, path = ""): Blob {
19851986 return blob;
19861987 if (blob.kind !== "too_large" && blob.size <= READ_CEILING && fileKind(path, blob.body.subarray(0, HEAD_BYTES)) === "psd")
19871988 return blob;
1989 // A table file's body is already bounded by its prefix; the renderer draws a window of it.
1990 if (blob.kind !== "too_large" && blob.body.length <= CSV_PREFIX_BYTES && isTablePath(path)) return blob;
19881991 return blob.kind !== "too_large" && blob.size > MAX_RENDER_BYTES
19891992 ? { kind: "too_large", size: blob.size, head: blob.body.slice(0, HEAD_BYTES) }
19901993 : blob;
src/web/render/csv.ts +31 −10
@@ -50,23 +50,41 @@ export function sortColumn(text: string, path: string, sort: string | undefined)
5050 return head.rows[0]!.slice(0, COLUMN_CAP).includes(sort) ? sort : undefined;
5151 }
5252
53 /**
54 * A prefix cut back to its last line end outside any quoted field (quotes counted in one pass; an
55 * escaped quote is two), so a cut inside a field drops that record instead of failing the table.
56 */
57 export function safeLines(bytes: Uint8Array): Uint8Array {
58 let open = false;
59 let end = 0;
60 for (let i = 0; i < bytes.length; i++) {
61 const b = bytes[i]!;
62 if (b === 34) open = !open;
63 else if (b === 10 && !open) end = i + 1;
64 }
65 return bytes.subarray(0, end === 0 ? bytes.length : end);
66 }
67
5368 export function csv(input: RenderInput): Rendered {
outdated

Up to 64 full re-parses of a 2 MB prefix is slow in the worst case. Find the last safe line boundary in one pass.

5469 const want = input.all ? ROW_CAP : ROW_WINDOW;
70 const partial = input.partial === true;
71 const lines = isJsonlPath(input.path);
72 // A prefix is cut where no quoted field is open, in one pass, so it parses whole.
73 const bytes = partial && !lines ? safeLines(input.body) : input.body;
74 const text = new TextDecoder().decode(bytes);
5575 // A sort needs every row the page may draw, so the window is cut after it; a sort that names no
5676 // column is no sort and reads no more than a page without one.
57 const text = new TextDecoder().decode(input.body);
5877 const sort = sortColumn(text, input.path, input.sort);
5978 const keep = sort === undefined ? want : ROW_CAP;
60 const lines = isJsonlPath(input.path);
61 const parsed = lines
62 ? parseJsonl(text, keep, COLUMN_CAP)
63 : parseCsv(text, { rows: keep + 1, columns: COLUMN_CAP });
79 const parsed = lines ? parseJsonl(text, keep, COLUMN_CAP) : parseCsv(text, { rows: keep + 1, columns: COLUMN_CAP });
6480 if ("error" in parsed) return { content: html``, lines: 0, fallback: parsed.error };
65 const total = parsed.records - 1;
81 // The rows in the prefix are exact; the file's are the prefix's density over its whole size.
82 const inPrefix = parsed.records - 1;
83 const total = partial ? Math.max(inPrefix, Math.round((inPrefix * input.size) / Math.max(1, bytes.length))) : inPrefix;
6684 const width = Math.min(parsed.columns, COLUMN_CAP);
6785 const numeric = numericColumns(parsed, width);
6886 // A wide table shrinks its row window so the page stays bounded.
69 const shown = Math.min(total, want, Math.floor(CELL_CAP / width));
87 const shown = Math.min(inPrefix, want, Math.floor(CELL_CAP / width));
7088 const [head, ...unsorted] = parsed.rows;
7189 // `?sort=` must name a header cell exactly; anything else is ignored.
7290 const column = sort === undefined ? -1 : head!.slice(0, width).indexOf(sort);
@@ -89,8 +107,11 @@ export function csv(input: RenderInput): Rendered {
89107 parsed.columns > width
90108 ? `Showing ${width.toLocaleString()} of ${parsed.columns.toLocaleString()} columns. `
91109 : "";
92 const more =
93 shown >= total && wide === ""
110 const count = (n: number): string => n.toLocaleString();
111 const ofRows = partial ? (inPrefix > shown ? `of about ${count(total)}` : `of more than ${count(shown)}`) : `of ${count(total)}`;
112 const more = partial
113 ? `<p class="muted small csv-more">${wide}Showing ${count(shown)} ${ofRows} rows. The rest is in the raw file.</p>`
114 : shown >= total && wide === ""
94115 ? ""
95116 : `<p class="muted small csv-more">${wide}${
96117 shown >= total ? "" : `Showing ${shown.toLocaleString()} of ${total.toLocaleString()} rows. `
@@ -119,7 +140,7 @@ export function csv(input: RenderInput): Rendered {
119140 return {
120141 content,
121142 lines: 0,
122 summary: `${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${lines ? "JSON Lines" : `${NAMES[parsed.delimiter]}-separated`} · ${fileSize(input.size)}`,
143 summary: `${partial ? "about " : ""}${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${lines ? "JSON Lines" : `${NAMES[parsed.delimiter]}-separated`} · ${fileSize(input.size)}`,
123144 };
124145 }
125146
src/web/render/kinds.ts +13
@@ -91,6 +91,7 @@ export const RENDER_CAP: Readonly<Record<FileKind, number>> = {
9191 html: 2 * MB,
9292 audio: 500 * MB,
9393 video: 4_000 * MB,
94 // Past it only a source that cuts a prefix (`Blob.partial`) is drawn, from that prefix.
9495 csv: 10 * MB,
outdated

A 4000 MB cap applies to every source, not only the one that cuts a prefix. Scope it to partial bodies.

9596 // Under the source's own 5 MB read ceiling: a larger file is never read.
9697 sqlite: 5 * MB,
@@ -369,6 +370,18 @@ const isStepHead = (head: Uint8Array): boolean =>
369370 /** For a caller that has only a name: the kind its extension gives. */
370371 export const NO_BYTES: Uint8Array = new Uint8Array(0);
371372
373 /** The bytes of a CSV, TSV or JSON-lines file the page reads; the rest is counted by estimate. */
374 export const CSV_PREFIX_BYTES = 2 * MB;
375
376 /** Whether a name is a table file the page draws from a prefix. */
377 export const isTablePath = (path: string): boolean => /\.(csv|tsv|jsonl|ndjson)$/i.test(path);
378
379 /** A prefix cut back to its last line end, so no half line is drawn; whole when it has none. */
380 export function wholeLines(prefix: Uint8Array): Uint8Array {
381 const end = prefix.lastIndexOf(10);
382 return end === -1 ? prefix : prefix.subarray(0, end + 1);
383 }
384
372385 /** How many leading bytes the registry needs to decide. */
373386 export const HEAD_BYTES = 4096;
374387
src/web/render/renderers.ts +5 −1
@@ -79,6 +79,8 @@ export interface RenderInput {
7979 * these.
8080 */
8181 readonly exists?: (path: string) => boolean;
82 /** The body is only the start of the file, cut at a line end: the row count is an estimate. */
83 readonly partial?: boolean;
8284 /** `?all=1`: draw past the default window, up to the renderer's cap. */
8385 readonly all?: boolean;
8486 /**
@@ -204,7 +206,9 @@ export function rendererFor(
204206 : blob.head !== undefined
205207 ? fileKind(path, blob.head)
206208 : kindByName(path);
207 if (blob.size > RENDER_CAP[kind]) return undefined;
209 // A table's cap is lifted only for a body the source already cut to a prefix.
210 const prefixed = kind === "csv" && blob.kind === "text" && blob.partial === true;
211 if (blob.size > RENDER_CAP[kind] && !prefixed) return undefined;
208212 if (MEDIA_KINDS.has(kind) && !where.media) return undefined;
209213 if (blob.kind === "too_large" && !MEDIA_KINDS.has(kind) && kind !== "psd") return undefined;
210214 // A kind may draw only some of its files: `config` draws JSON, YAML and
test/render-csv-prefix.test.ts added +160
@@ -0,0 +1,160 @@
1 /**
2 * A table file over the prefix opens its first 1,000 rows from a bounded read, and says what it cut
3 * (#13465).
4 */
5 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
11 import type { Repo } from "../src/core/repos.ts";
12 import { repoDir } from "../src/git/repos.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import { rendererFor } from "../src/web/render/renderers.ts";
15 import { localSource } from "../src/view/local.ts";
16 import { csv, safeLines } from "../src/web/render/csv.ts";
17 import { CSV_PREFIX_BYTES, RENDER_CAP, wholeLines } from "../src/web/render/kinds.ts";
18 import { seedAdmin } from "./helpers/admin.ts";
19 import { quietLogs } from "./helpers/globals.ts";
20
21 quietLogs();
22
23 const enc = (s: string): Uint8Array => new TextEncoder().encode(s);
24
25 /** About 20 MB of rows, built as one buffer. */
26 const generate = (make: (i: number) => string, bytes: number): Uint8Array => {
27 const parts: string[] = [];
28 let size = 0;
29 for (let i = 0; size < bytes; i++) {
30 const line = make(i);
31 parts.push(line);
32 size += line.length;
33 }
34 return enc(parts.join(""));
35 };
36 const CSV = generate((i) => (i === 0 ? "id,name,score\n" : `${i},user-${i},${(i * 7) % 100}\n`), 20_000_000);
37 const JSONL = generate((i) => `{"id":${i},"name":"user-${i}","tags":["a","b"]}\n`, 20_000_000);
38
39 describe("the prefix cut", () => {
40 test("drops the final partial line", () => {
41 expect(new TextDecoder().decode(wholeLines(enc("a,b\n1,2\n3,")))).toBe("a,b\n1,2\n");
42 expect(wholeLines(enc("no newline")).length).toBe(10);
43 });
44
45 test("a partial body says 'of about N' with the estimate from the prefix density", () => {
46 const rows = "a,b\n" + "1,2\n".repeat(5_000);
47 const text = enc(rows);
48 const out = csv({ path: "d.csv", size: text.length * 10, body: text, partial: true, media: "/m", download: "/d" });
49 const page = String(out.content);
50 expect(page).toContain("Showing 1,000 of about 50,000 rows");
51 expect(out.summary).toContain("about 50,000 rows");
52 expect(page).not.toContain("?all=1");
53 });
54
55 test("a prefix that ends inside the window says 'more than'", () => {
56 const text = enc("a,b\n" + "1,2\n".repeat(100));
57 const out = csv({ path: "d.csv", size: 50_000_000, body: text, partial: true, media: "/m", download: "/d" });
58 expect(String(out.content)).toContain("Showing 100 of more than 100 rows");
59 const wide = enc("a,b\n" + "1,2\n".repeat(1_000));
60 const exact = csv({ path: "d.csv", size: 50_000_000, body: wide, partial: true, media: "/m", download: "/d" });
61 expect(String(exact.content)).toContain("of more than 1,000 rows");
62 });
63
64 test("the note is one sentence: no file size, no bytes read", () => {
65 const text = enc("a,b\n" + "1,2\n".repeat(5_000));
66 const page = String(csv({ path: "d.csv", size: text.length * 10, body: text, partial: true, media: "/m", download: "/d" }).content);
67 expect(page).toContain("Showing 1,000 of about 50,000 rows. The rest is in the raw file.");
68 expect(page).not.toContain("only its first");
69 });
70
71 test("the cap is lifted only for a body the source cut", () => {
72 const big = RENDER_CAP.csv + 1;
73 const body = enc("a,b\n1,2\n");
74 expect(rendererFor("d.csv", { kind: "text", body, size: big })).toBeUndefined();
75 expect(rendererFor("d.csv", { kind: "text", body, size: big, partial: true })).toBeDefined();
76 });
77
78 test("safeLines cuts at the last line end outside a quoted field, in one pass", () => {
79 const dec = (b: Uint8Array) => new TextDecoder().decode(b);
80 expect(dec(safeLines(enc('a\n"x\ny"\n"o\np')))).toBe('a\n"x\ny"\n');
81 expect(dec(safeLines(enc('a\n"q ""x"" \n"\nz')))).toBe('a\n"q ""x"" \n"\n');
82 });
83
84 test("a cut inside a quoted field still draws the rows before it", () => {
85 const text = enc('a,b\n1,2\n3,"open\nstill open');
86 const out = csv({ path: "d.csv", size: 9_000_000, body: text, partial: true, media: "/m", download: "/d" });
87 expect(out.fallback).toBeUndefined();
88 expect(String(out.content)).toContain(`<td class="num">1</td>`);
89 });
90 });
91
92 describe("a 20 MB table file", () => {
93 let dataDir: string;
94 let app: App;
95 let servers: Servers;
96 let base: string;
97 let repo: Repo;
98
99 beforeAll(async () => {
100 dataDir = await mkdtemp(join(tmpdir(), "forge-csv-prefix-"));
101 app = createApp({ dataDir, port: 0 });
102 servers = await serve(app);
103 base = servers.url;
104 const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-prefix-passphrase" });
105 if (!a.ok) throw new Error("signup failed");
106 const created = await createRepo(app, a.value, { name: "data", visibility: "public" });
107 if (!created.ok) throw new Error("repo failed");
108 repo = created.value;
109 const made = await commitToBranch(app, a.value, repo, { message: "start", files: [{ path: "README.md", content: "# d\n" }] });
110 if (!made.ok) throw new Error("commit failed");
111 const dir = repoDir(app.reposRoot, "alice", "data");
112 const git = (args: string[], stdin?: Uint8Array | string): string => {
113 const run = Bun.spawnSync(["git", ...args], {
114 cwd: dir,
115 stdin: typeof stdin === "string" ? enc(stdin) : (stdin ?? "ignore"),
116 env: { ...process.env, GIT_AUTHOR_NAME: "alice", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "alice", GIT_COMMITTER_EMAIL: "[email protected]" },
117 });
118 if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`);
119 return run.stdout.toString().trim();
120 };
121 const lines = [git(["ls-tree", "main"])];
122 for (const [name, body] of [["big.csv", CSV], ["big.jsonl", JSONL]] as const)
123 lines.push(`100644 blob ${git(["hash-object", "-w", "--stdin"], body)}\t${name}`);
124 const tree = git(["mktree"], lines.join("\n") + "\n");
125 git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]);
126 }, 60_000);
127
128 afterAll(async () => {
129 await servers?.stop();
130 app?.close();
131 await rm(dataDir, { recursive: true, force: true });
132 });
133
134 test("the source hands over a bounded prefix ending on a line", async () => {
135 const read = await localSource(app, repo).blob("main", "big.csv", { prefix: true });
136 if (!read.ok || read.value.kind !== "text") throw new Error("expected a text prefix");
137 expect(read.value.partial).toBe(true);
138 expect(read.value.size).toBe(CSV.length);
139 expect(read.value.body.length).toBeLessThanOrEqual(CSV_PREFIX_BYTES);
140 expect(read.value.body[read.value.body.length - 1]).toBe(10);
141 });
142
143 test("a reader that asks for no prefix (an editor) still gets too_large", async () => {
144 const read = await localSource(app, repo).blob("main", "big.csv");
145 expect(read.ok && read.value.kind).toBe("too_large");
146 });
147
148 for (const name of ["big.csv", "big.jsonl"]) {
149 test(`${name} opens its first 1,000 rows, says it cut the rest, and `, async () => {
150 const res = await fetch(`${base}/alice/data/blob/main/${name}`);
151 const text = await res.text();
152 // Bytes, not clocks: the page is a window of a bounded prefix, never the 20 MB file.
153 expect(text.length).toBeLessThan(2_000_000);
154 expect(res.status).toBe(200);
155 expect(text).toMatch(/Showing 1,000 of about [\d,]+ rows/);
156 expect(text).toContain("user-999");
157 expect(text).not.toContain("user-1001<");

Non-blocking: the test title ends in 'and ' with nothing after it. Finish it, e.g. 'and shows no row past the window'.

158 });
159 }
160 });
test/tui-workspace-formats.test.ts +39
@@ -8,6 +8,7 @@
88 import { Database } from "bun:sqlite";
99 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
1010 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
11 import { repoDir } from "../src/git/repos.ts";
1112 import type { App as Forge } from "../src/app.ts";
1213 import type { User } from "../src/core/users.ts";
1314 import { zip } from "./helpers/zip.ts";
@@ -31,6 +32,44 @@ describe("the terminal workspace, by file type", () => {
3132
3233 afterAll(() => fx.close());
3334
35 test("a .csv over the page's size limit opens its first rows, read-only, with the page's note", async () => {
36 const made = await createRepo(forge, alice, { name: "hugetable", visibility: "public", description: "big" });
37 if (!made.ok) throw new Error("repo failed");
38 const seeded = await commitToBranch(forge, alice, made.value, {
39 message: "start",
40 files: [{ path: "README.md", content: "# big\n" }],
41 });
42 if (!seeded.ok) throw new Error("commit failed");
43 // Over the 2 MB a commit through the app takes: written with git itself, as a push would.
44 const dir = repoDir(forge.reposRoot, "alice", "hugetable");
45 const git = (args: string[], stdin?: string): string => {
46 const run = Bun.spawnSync(["git", ...args], {
47 cwd: dir,
48 stdin: stdin === undefined ? "ignore" : new TextEncoder().encode(stdin),
49 env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" },
50 });
51 if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`);
52 return run.stdout.toString().trim();
53 };
54 const rows = Array.from({ length: 200_000 }, (_, i) => `${i + 1},item ${i + 1},padding-padding-padding`);
55 const blob = git(["hash-object", "-w", "--stdin"], ["id,name,pad", ...rows, ""].join("\n"));
56 const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${blob}\tbig.csv\n`);
57 git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]);
58 const h = boot();
59 try {
60 await h.settle();
61 await open(h, "hugetable");
62 await openFile(h, "big.csv");
63 expect(h.text()).toContain("id");
64 expect(h.text()).toContain("item 1");
65 expect(h.text()).toContain("read-only");
66 await h.press("e");
67 expect(h.text()).toContain("read-only");
68 } finally {
69 h.stop();
70 }
71 });
72
3473 test("a .csv opens as an aligned table with a pinned header, and s flips to the source", async () => {
3574 const made = await createRepo(forge, alice, {
3675 name: "ledger",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs