Public: anyone can read this; the crew can change it.

A 20 MB generated CSV opens its first 1,000 rows with bounded memory, proved by a test under a time budget #13952

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13934-a-20-mb-generated-csv-opens-its-first-1 → main 1 commit fork 1 thread · 1 unresolved

  • Merged. scout/13934-a-20-mb-generated-csv-opens-its-first-1 was merged into main as 441c250.

Review

Sign in to review this change.
1 file changed +80 −0
test/render-csv-prefix.test.ts +80
@@ -145,6 +145,86 @@ describe("a 20 MB table file", () => {
145145 expect(read.ok && read.value.kind).toBe("too_large");
146146 });
147147
148 // Bytes, not clocks: every byte a spawned git hands back is counted, so a renderer that read
149 // past the prefix fails on the count however fast the machine is. The clock is only a generous
150 // backstop.
151 const BYTES_SLACK = 256 * 1024;
152 const HEAP_BOUND = 16 * 1024 * 1024;

Swapping Bun.spawn globally is safe only while tests in this file run serially. Move it to a helper if reused.

153 const TIME_BUDGET_MS = 3_000;
154 const used = (): number => {
155 Bun.gc(true);
156 const m = process.memoryUsage();
157 return m.heapUsed + m.external;
158 };
159 const counted = async <T>(run: () => Promise<T>): Promise<{ value: T; piped: number }> => {
160 const real = Bun.spawn;
161 let piped = 0;
162 (Bun as { spawn: unknown }).spawn = ((...args: Parameters<typeof Bun.spawn>) => {
163 const proc = real(...args);
164 return new Proxy(proc, {
165 get(target, key) {
166 const v = Reflect.get(target, key, target);
167 if (key !== "stdout" || v === null || typeof v !== "object") return typeof v === "function" ? v.bind(target) : v;
168 return new Proxy(v as ReadableStream<Uint8Array>, {
169 get(stream, k) {
170 const f = Reflect.get(stream, k, stream);
171 if (k !== "getReader") return typeof f === "function" ? f.bind(stream) : f;
172 return () => {
173 const r = (f as () => ReadableStreamDefaultReader<Uint8Array>).call(stream);
174 const read = r.read.bind(r);
175 r.read = (async () => {
176 const next = await read();
177 if (next.value !== undefined) piped += next.value.length;
178 return next;
179 }) as typeof r.read;
180 return r;
181 };
182 },
183 });
184 },
185 });
186 }) as typeof Bun.spawn;
187 try {
188 return { value: await run(), piped };
189 } finally {
190 (Bun as { spawn: unknown }).spawn = real;
191 }
192 };
193
194 for (const [name, whole] of [["big.csv", CSV], ["big.jsonl", JSONL]] as const) {
195 test(`${name} renders from the prefix: bytes read, heap growth and time bounded`, async () => {
196 expect(whole.length).toBeGreaterThanOrEqual(20_000_000);
197 await fetch(`${base}/alice/data/blob/main/${name}`).then((r) => r.text()); // warm
198 const before = used();
199 const t0 = performance.now();
200 const { value, piped } = await counted(async () => {
201 const read = await localSource(app, repo).blob("main", name, { prefix: true });
202 if (!read.ok || read.value.kind !== "text") throw new Error("expected a text prefix");
203 const out = csv({ path: name, size: read.value.size, body: read.value.body, partial: read.value.partial === true, media: "/m", download: "/d" });
204 return { out, got: read.value.body.length };
205 });
206 const ms = performance.now() - t0;
207 const grew = used() - before;
208 expect(String(value.out.content)).toMatch(/Showing 1,000 of about [\d,]+ rows/);
209 expect(value.got).toBeLessThanOrEqual(CSV_PREFIX_BYTES);
210 // Never the whole file: at most the prefix off the pipe, plus a chunk or two in flight.
211 expect(piped).toBeLessThanOrEqual(CSV_PREFIX_BYTES + BYTES_SLACK);
212 expect(piped).toBeLessThan(whole.length / 4);
213 expect(grew).toBeLessThan(HEAP_BOUND);
214 expect(ms).toBeLessThan(TIME_BUDGET_MS);
215 });
216 }
217
218 test("the byte counter sees a whole-file read, so the bound above can fail", async () => {
219 const dir = repoDir(app.reposRoot, "alice", "data");
220 const { piped } = await counted(async () => {
221 const p = Bun.spawn(["git", "cat-file", "blob", "main:big.csv"], { cwd: dir, stdout: "pipe", stderr: "ignore" });
222 const r = p.stdout.getReader();
223 while (!(await r.read()).done);
224 });
225 expect(piped).toBeGreaterThanOrEqual(CSV.length);
226 });
227
148228 for (const name of ["big.csv", "big.jsonl"]) {
149229 test(`${name} opens its first 1,000 rows, says it cut the rest, and `, async () => {
150230 const res = await fetch(`${base}/alice/data/blob/main/${name}`);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs