Public: anyone can read this; the crew can change it.
A JSON Lines file (.jsonl, .ndjson) renders as a table, as CSV does #10322
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10285-a-json-lines-file-jsonl-ndjson-renders-a → main 5 commits fork 3 threads · 3 unresolved
6 files
changed +129
−6
docs/features/csv-view.md +9
| @@ -6,3 +6,12 @@ The first 1,000 rows are drawn on the server with a sticky header and (on phones | ||
| 6 | 6 | `?all=1` ("Show all N rows", no JavaScript needed) draws up to 20,000. The header line shows rows × columns and the delimiter. |
| 7 | 7 | A small script adds sort (header click or Enter) and a filter box over the rows drawn; no library. |
| 8 | 8 | A file that is not valid CSV shows the source with a one-line reason. `?view=source`, Raw and Download work as for any file. |
| 9 | ||
| 10 | ## JSON Lines | |
| 11 | ||
| 12 | `.jsonl`/`.ndjson` files use the same table (`src/web/render/jsonl-parse.ts` makes the rows; `csv.ts` draws them). | |
| 13 | - Columns are the union of the keys of the first 1,000 objects, in first-seen order; a key first seen later is not a column. | |
| 14 | - Strings show as they are, `null` as empty, nested values as compact JSON; everything is escaped. | |
| 15 | - A line that does not parse is one row, `Line N: not valid JSON`, or `Line N: not a JSON object` for valid JSON that is not an object (N counts physical lines). The rest of the file still renders. | |
| 16 | - The header line says `JSON Lines` in place of the delimiter. A file with no non-blank lines falls back to the source view. | |
| 17 | - At most 200 columns are drawn; the page says how many there are. |
src/web/render/csv.ts +9 −6
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | import { escapeHtml, html, raw } from "../html.ts"; |
| 8 | 8 | import { fileSize } from "../ui.ts"; |
| 9 | 9 | import { numericColumns, parseCsv, type Delimiter } from "./csv-parse.ts"; |
| 10 | import { parseJsonl } from "./jsonl-parse.ts"; | |
| 10 | 11 | import type { Rendered, RenderInput } from "./renderers.ts"; |
| 11 | 12 | |
| 12 | 13 | /** Rows drawn by default, and by `?all=1` at most. */ |
| @@ -35,10 +36,11 @@ const cells = (tag: "th" | "td", row: string[], width: number, numeric: boolean[ | ||
| 35 | 36 | |
| 36 | 37 | export function csv(input: RenderInput): Rendered { |
| 37 | 38 | const want = input.all ? ROW_CAP : ROW_WINDOW; |
| 38 | const parsed = parseCsv(new TextDecoder().decode(input.body), { | |
| 39 | rows: want + 1, | |
| 40 | columns: COLUMN_CAP, | |
| 41 | }); | |
| 39 | const text = new TextDecoder().decode(input.body); | |
| 40 | const lines = isJsonlPath(input.path); | |
| 41 | const parsed = lines | |
| 42 | ? parseJsonl(text, want, COLUMN_CAP) | |
| 43 | : parseCsv(text, { rows: want + 1, columns: COLUMN_CAP }); | |
| 42 | 44 | if ("error" in parsed) return { content: html``, lines: 0, fallback: parsed.error }; |
| 43 | 45 | const total = parsed.records - 1; |
| 44 | 46 | const width = Math.min(parsed.columns, COLUMN_CAP); |
| @@ -83,12 +85,13 @@ export function csv(input: RenderInput): Rendered { | ||
| 83 | 85 | return { |
| 84 | 86 | content, |
| 85 | 87 | lines: 0, |
| 86 | summary: `${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${NAMES[parsed.delimiter]}-separated · ${fileSize(input.size)}`, | |
| 88 | summary: `${total.toLocaleString()} rows × ${parsed.columns.toLocaleString()} columns · ${lines ? "JSON Lines" : `${NAMES[parsed.delimiter]}-separated`} · ${fileSize(input.size)}`, | |
| 87 | 89 | }; |
| 88 | 90 | } |
| 89 | 91 | |
| 90 | 92 | /** Whether a path can hold a table, so only those pages carry the script below. */ |
| 91 | export const isCsvPath = (path: string): boolean => /\.(csv|tsv)$/i.test(path); | |
| 93 | export const isCsvPath = (path: string): boolean => /\.(csv|tsv|jsonl|ndjson)$/i.test(path); | |
| 94 | const isJsonlPath = (path: string): boolean => /\.(jsonl|ndjson)$/i.test(path); | |
| 92 | 95 | |
| 93 | 96 | /** Sort on a header click, filter on typing. Trusted static code. */ |
| 94 | 97 | export const csvJs = /* js */ ` |
src/web/render/jsonl-parse.ts added +64
| @@ -0,0 +1,64 @@ | ||
| 1 | /** | |
| 2 | * JSON Lines (.jsonl, .ndjson) as the same plain table the CSV parser makes: | |
| 3 | * one object per line, the columns the union of the keys of the first | |
| 4 | * thousand objects in first-seen order. A line that does not parse is one row | |
| 5 | * that says so, with its line number; the rest of the file still reads. | |
| 6 | */ | |
| 7 | ||
| 8 | import type { Parsed } from "./csv-parse.ts"; | |
| 9 | ||
| 10 | /** Objects whose keys make the columns. */ | |
| 11 | export const KEY_WINDOW = 1_000; | |
| 12 | ||
| 13 | /** A cell: strings as they are, null empty, nested values as compact JSON. */ | |
| 14 | const cell = (v: unknown): string => | |
| 15 | v === null || v === undefined ? "" : typeof v === "string" ? v : typeof v === "object" ? JSON.stringify(v) : String(v); | |
| 16 | ||
| 17 | type Line = { readonly n: number; readonly value: Record<string, unknown> } | { readonly n: number; readonly bad: string }; | |
| 18 | ||
| 19 | export function parseJsonl(input: string, keepRows: number, maxColumns = Infinity): Parsed { | |
| 20 | const text = input.charCodeAt(0) === 0xfeff ? input.slice(1) : input; | |
| 21 | const keys = new Map<string, number>(); | |
| 22 | const kept: Line[] = []; | |
| 23 | let objects = 0; | |
| 24 | let records = 0; | |
| 25 | let n = 0; | |
| 26 | let pos = 0; | |
| 27 | while (pos <= text.length) { | |
| 28 | let end = text.indexOf("\n", pos); | |
| 29 | if (end === -1) end = text.length; | |
| 30 | n++; | |
| 31 | const raw = text.slice(pos, end).trim(); | |
| 32 | pos = end + 1; | |
| 33 | if (raw === "") continue; | |
| 34 | records++; | |
| 35 | const keeping = kept.length < keepRows; | |
| 36 | const needKeys = objects < KEY_WINDOW; | |
| 37 | if (!keeping && !needKeys) continue; | |
| 38 | let line: Line; | |
| 39 | try { | |
| 40 | const v: unknown = JSON.parse(raw); | |
| 41 | line = | |
| 42 | v !== null && typeof v === "object" && !Array.isArray(v) | |
| 43 | ? { n, value: v as Record<string, unknown> } | |
| 44 | : { n, bad: `Line ${n}: not a JSON object` }; | |
| 45 | } catch { | |
| 46 | line = { n, bad: `Line ${n}: not valid JSON` }; | |
| 47 | } | |
| 48 | if ("value" in line && needKeys) { | |
| 49 | objects++; | |
| 50 | for (const k of Object.keys(line.value)) if (!keys.has(k)) keys.set(k, keys.size); | |
| 51 | } | |
| 52 | if (keeping) kept.push(line); | |
| 53 | } | |
| 54 | if (records === 0) return { error: "The file has no rows." }; | |
| 55 | // Only the columns the page can draw are built; `columns` stays the true count. | |
| 56 | const all = keys.size > 0 ? [...keys.keys()] : ["line"]; | |
| 57 | const header = all.slice(0, maxColumns); | |
| 58 | const rows: string[][] = [header]; | |
| 59 | for (const line of kept) { | |
| 60 | if ("bad" in line) rows.push([line.bad]); | |
Medium: |
||
| 61 | else rows.push(header.map((k) => (Object.hasOwn(line.value, k) ? cell(line.value[k]) : ""))); | |
| 62 | } | |
| 63 | return { delimiter: ",", rows, columns: all.length, records: records + 1 }; | |
| 64 | } |
src/web/render/kinds.ts +2
| @@ -121,6 +121,8 @@ const MEDIA: Readonly<Record<string, Media>> = { | ||
| 121 | 121 | const TEXTUAL: Readonly<Record<string, FileKind>> = { |
| 122 | 122 | csv: "csv", |
| 123 | 123 | tsv: "csv", |
| 124 | jsonl: "csv", | |
| 125 | ndjson: "csv", | |
| 124 | 126 | gltf: "model3d", |
| 125 | 127 | json: "config", |
| 126 | 128 | jsonc: "config", |
test/render-csv.test.ts +43
| @@ -2,6 +2,7 @@ | ||
| 2 | 2 | import { describe, expect, test } from "bun:test"; |
| 3 | 3 | import { parseCsv, numericColumns, type Table } from "../src/web/render/csv-parse.ts"; |
| 4 | 4 | import { COLUMN_CAP, csv, csvJs, isCsvPath, ROW_CAP } from "../src/web/render/csv.ts"; |
| 5 | import { parseJsonl } from "../src/web/render/jsonl-parse.ts"; | |
| 5 | 6 | import { fileKind } from "../src/web/render/kinds.ts"; |
| 6 | 7 | import { renderBlob } from "../src/web/pages/repo.ts"; |
| 7 | 8 | import type { RepoScope } from "../src/web/scope.ts"; |
| @@ -148,3 +149,45 @@ describe("csv on the blob page", () => { | ||
| 148 | 149 | expect(isCsvPath("a/b.ts")).toBe(false); |
| 149 | 150 | }); |
| 150 | 151 | }); |
| 152 | ||
| 153 | describe("json lines as a table", () => { | |
| 154 | const lines = (text: string) => | |
| 155 | csv({ path: "d.jsonl", size: text.length, body: enc(text), media: "/m", pathHref: "/o/r/blob/main/d.jsonl" }); | |
| 156 | const text = | |
| 157 | '{"a":1,"b":"x"}\n{"c":{"k":[1,2]},"a":"<script>alert(1)</script>"}\nnot json\n\n{"b":null}\n'; | |
| 158 | test("kinds: jsonl and ndjson are csv, and carry the table script", () => { | |
| 159 | expect(fileKind("x.jsonl", enc("{}\n"))).toBe("csv"); | |
| 160 | expect(fileKind("x.ndjson", enc("{}\n"))).toBe("csv"); | |
| 161 | expect(isCsvPath("x.NDJSON")).toBe(true); | |
| 162 | }); | |
| 163 | test("mixed keys in first-seen order, nested as compact JSON, a bad line as a row, script escaped", () => { | |
| 164 | const r = lines(text); | |
| 165 | const out = String(r.content); | |
| 166 | expect(out).toMatch(/<th>a<\/th><th>b<\/th><th>c<\/th>/); | |
| 167 | expect(out).toContain("{"k":[1,2]}"); | |
| 168 | expect(out).toContain("Line 3: not valid JSON"); | |
| 169 | expect(out).toContain("<script>alert(1)</script>"); | |
| 170 | expect(out).not.toContain("<script>alert"); | |
| 171 | expect(r.summary).toContain("4 rows × 3 columns · JSON Lines"); | |
| 172 | }); | |
| 173 | test("a non-object line and an empty file", () => { | |
outdated
Add a case with >1,000 objects where a key first appears after the window, asserting it is not a column. |
||
| 174 | expect(String(lines('{"a":1}\n[1]\n').content)).toContain("Line 2: not a JSON object"); | |
| 175 | expect(lines("\n\n").fallback).toBe("The file has no rows."); | |
| 176 | }); | |
| 177 | test("a key first seen after the 1,000th object is not a column; huge key sets are capped", () => { | |
| 178 | const rows = Array.from({ length: 1_000 }, (_, i) => `{"a":${i}}`); | |
| 179 | const out = String(lines([...rows, '{"late":1}'].join("\n")).content); | |
| 180 | expect(out).toContain(">a</th>"); | |
| 181 | expect(out).not.toContain("late"); | |
| 182 | const wide = lines(JSON.stringify(Object.fromEntries(Array.from({ length: COLUMN_CAP + 5 }, (_, i) => [`k${i}`, i])))); | |
| 183 | expect(String(wide.content)).toContain(`Showing ${COLUMN_CAP} of ${COLUMN_CAP + 5} columns`); | |
| 184 | expect(String(wide.content)).not.toContain(`<th>k${COLUMN_CAP}</th>`); | |
| 185 | }); | |
| 186 | test("a missing key is an empty cell, never an inherited property", () => { | |
| 187 | const p = parseJsonl('{"constructor":1,"toString":2,"__proto__":3}\n{"a":1}\n', 10); | |
| 188 | if ("error" in p) throw new Error(p.error); | |
| 189 | expect(p.rows[0]).toEqual(["constructor", "toString", "__proto__", "a"]); | |
| 190 | expect(p.rows[1]).toEqual(["1", "2", "3", ""]); | |
| 191 | expect(p.rows[2]).toEqual(["", "", "", "1"]); | |
| 192 | }); | |
| 193 | }); |
test/render-kinds.test.ts +2
| @@ -39,6 +39,8 @@ const SAMPLES: Record<string, [Uint8Array, FileKind, string]> = { | ||
| 39 | 39 | glb: [enc("glTF\x02\0\0\0"), "model3d", "model/gltf-binary"], |
| 40 | 40 | csv: [enc("a,b\n1,2\n"), "csv", "application/octet-stream"], |
| 41 | 41 | tsv: [enc("a\tb\n"), "csv", "application/octet-stream"], |
| 42 | jsonl: [enc("{\"a\":1}\n"), "csv", "application/octet-stream"], | |
| 43 | ndjson: [enc("{\"a\":1}\n"), "csv", "application/octet-stream"], | |
| 42 | 44 | gltf: [enc('{"asset":{}}'), "model3d", "application/octet-stream"], |
| 43 | 45 | json: [enc("{}"), "config", "application/octet-stream"], |
| 44 | 46 | jsonc: [enc("{}"), "config", "application/octet-stream"], |
Header and every row are built at full key count; cap at the column limit here (keep the true count in
columns).