Public: anyone can read this; the crew can change it.
A 20 MB generated CSV opens its first 1,000 rows with bounded memory, proved by a test under a time budget #13952
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13934-a-20-mb-generated-csv-opens-its-first-1 → main 1 commit fork 1 thread · 1 unresolved
1 file
changed +80
−0
test/render-csv-prefix.test.ts +80
| @@ -145,6 +145,86 @@ describe("a 20 MB table file", () => { | ||
| 145 | 145 | expect(read.ok && read.value.kind).toBe("too_large"); |
| 146 | 146 | }); |
| 147 | 147 | |
| 148 | // Bytes, not clocks: every byte a spawned git hands back is counted, so a renderer that read | |
| 149 | // past the prefix fails on the count however fast the machine is. The clock is only a generous | |
| 150 | // backstop. | |
| 151 | const BYTES_SLACK = 256 * 1024; | |
| 152 | const HEAP_BOUND = 16 * 1024 * 1024; | |
| 153 | const TIME_BUDGET_MS = 3_000; | |
| 154 | const used = (): number => { | |
| 155 | Bun.gc(true); | |
| 156 | const m = process.memoryUsage(); | |
| 157 | return m.heapUsed + m.external; | |
| 158 | }; | |
| 159 | const counted = async <T>(run: () => Promise<T>): Promise<{ value: T; piped: number }> => { | |
| 160 | const real = Bun.spawn; | |
| 161 | let piped = 0; | |
| 162 | (Bun as { spawn: unknown }).spawn = ((...args: Parameters<typeof Bun.spawn>) => { | |
| 163 | const proc = real(...args); | |
| 164 | return new Proxy(proc, { | |
| 165 | get(target, key) { | |
| 166 | const v = Reflect.get(target, key, target); | |
| 167 | if (key !== "stdout" || v === null || typeof v !== "object") return typeof v === "function" ? v.bind(target) : v; | |
| 168 | return new Proxy(v as ReadableStream<Uint8Array>, { | |
| 169 | get(stream, k) { | |
| 170 | const f = Reflect.get(stream, k, stream); | |
| 171 | if (k !== "getReader") return typeof f === "function" ? f.bind(stream) : f; | |
| 172 | return () => { | |
| 173 | const r = (f as () => ReadableStreamDefaultReader<Uint8Array>).call(stream); | |
| 174 | const read = r.read.bind(r); | |
| 175 | r.read = (async () => { | |
| 176 | const next = await read(); | |
| 177 | if (next.value !== undefined) piped += next.value.length; | |
| 178 | return next; | |
| 179 | }) as typeof r.read; | |
| 180 | return r; | |
| 181 | }; | |
| 182 | }, | |
| 183 | }); | |
| 184 | }, | |
| 185 | }); | |
| 186 | }) as typeof Bun.spawn; | |
| 187 | try { | |
| 188 | return { value: await run(), piped }; | |
| 189 | } finally { | |
| 190 | (Bun as { spawn: unknown }).spawn = real; | |
| 191 | } | |
| 192 | }; | |
| 193 | ||
| 194 | for (const [name, whole] of [["big.csv", CSV], ["big.jsonl", JSONL]] as const) { | |
| 195 | test(`${name} renders from the prefix: bytes read, heap growth and time bounded`, async () => { | |
| 196 | expect(whole.length).toBeGreaterThanOrEqual(20_000_000); | |
| 197 | await fetch(`${base}/alice/data/blob/main/${name}`).then((r) => r.text()); // warm | |
| 198 | const before = used(); | |
| 199 | const t0 = performance.now(); | |
| 200 | const { value, piped } = await counted(async () => { | |
| 201 | const read = await localSource(app, repo).blob("main", name, { prefix: true }); | |
| 202 | if (!read.ok || read.value.kind !== "text") throw new Error("expected a text prefix"); | |
| 203 | const out = csv({ path: name, size: read.value.size, body: read.value.body, partial: read.value.partial === true, media: "/m", download: "/d" }); | |
| 204 | return { out, got: read.value.body.length }; | |
| 205 | }); | |
| 206 | const ms = performance.now() - t0; | |
| 207 | const grew = used() - before; | |
| 208 | expect(String(value.out.content)).toMatch(/Showing 1,000 of about [\d,]+ rows/); | |
| 209 | expect(value.got).toBeLessThanOrEqual(CSV_PREFIX_BYTES); | |
| 210 | // Never the whole file: at most the prefix off the pipe, plus a chunk or two in flight. | |
| 211 | expect(piped).toBeLessThanOrEqual(CSV_PREFIX_BYTES + BYTES_SLACK); | |
| 212 | expect(piped).toBeLessThan(whole.length / 4); | |
| 213 | expect(grew).toBeLessThan(HEAP_BOUND); | |
| 214 | expect(ms).toBeLessThan(TIME_BUDGET_MS); | |
| 215 | }); | |
| 216 | } | |
| 217 | ||
| 218 | test("the byte counter sees a whole-file read, so the bound above can fail", async () => { | |
| 219 | const dir = repoDir(app.reposRoot, "alice", "data"); | |
| 220 | const { piped } = await counted(async () => { | |
| 221 | const p = Bun.spawn(["git", "cat-file", "blob", "main:big.csv"], { cwd: dir, stdout: "pipe", stderr: "ignore" }); | |
| 222 | const r = p.stdout.getReader(); | |
| 223 | while (!(await r.read()).done); | |
| 224 | }); | |
| 225 | expect(piped).toBeGreaterThanOrEqual(CSV.length); | |
| 226 | }); | |
| 227 | ||
| 148 | 228 | for (const name of ["big.csv", "big.jsonl"]) { |
| 149 | 229 | test(`${name} opens its first 1,000 rows, says it cut the rest, and `, async () => { |
| 150 | 230 | const res = await fetch(`${base}/alice/data/blob/main/${name}`); |
new above
offline
Swapping Bun.spawn globally is safe only while tests in this file run serially. Move it to a helper if reused.