Public: anyone can read this; the crew can change it.

A data query says how many rows it read and how long it took, on the page and in the MCP answer #13409

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13385-a-data-query-says-how-many-rows-it-read → main 5 commits fork 2 threads · 2 unresolved

  • Merged. scout/13385-a-data-query-says-how-many-rows-it-read was merged into main as 57b4841.

Review

Sign in to review this change.
9 files changed +46 −17
docs/features/data-browser.md +2
@@ -21,3 +21,5 @@ API and MCP by repository (`data/repo_scope.ts`, `http/api/data.ts`): `GET /api/
2121 ## On the web (#7580)
2222
2323 Three doors, one body (`src/web/pages/data_view.ts`): `/data` (every file you may see), `/admin/data` (the forge's own database, admins; the Forge data button on `/data`) and `/:owner/:name/data` (an app's Data tab for its writers: environments → files → tables → rows; a reader gets the 404). Rows sort by header (`sort`, `dir`), filter by text on one column or all (`q`, `col`), JSON cells are pretty-printed, and a copy button per row puts it on the clipboard as JSON. Sealed columns show `••••••` and cannot be sorted or searched. The SQL box is a GET form (`sql`, capped at 4,000 characters), so a query is an address to share; it shows the rows and the elapsed ms. The sorted column carries `aria-sort` and a muted arrow from CSS; a wide grid scrolls sideways only. Not done: saved queries, export, editing rows.
24
25 Cost of a query: `query` also returns `rowsRead` (rows the query produced, the one that tripped a cap included) and `elapsedMs` (at least 1). The page prints them as one muted line under the table (`N rows read · X ms`), the terminal's `:` result shows the same in its header line, and the MCP `data` answer with `sql` carries both fields. They count rows the query produced, not rows SQLite scanned.
src/data/query.ts +10 −3
@@ -23,6 +23,10 @@ export type QueryResult =
2323 readonly columns: readonly string[];
2424 readonly rows: readonly (readonly (string | number | null)[])[];
2525 readonly truncated: boolean;
26 /** Rows the query produced before it stopped, the row that tripped a cap included. */
27 readonly rowsRead: number;
28 /** Milliseconds from start to answer, at least 1. */
29 readonly elapsedMs: number;
2630 }
2731 | {
2832 readonly ok: false;
@@ -55,6 +59,7 @@ export function query(
5559 message: "Busy with other queries: try again in a moment.",
5660 });
5761 running++;
62 const t0 = performance.now();
5863 return new Promise((resolve) => {
5964 const child = Bun.spawn([process.execPath, new URL("./query_child.ts", import.meta.url).pathname], {
6065 stdin: "pipe",
@@ -94,14 +99,16 @@ export function query(
9499 void new Response(child.stdout).text().then(
95100 (text) => {
96101 try {
97 const m = JSON.parse(text) as { ok: boolean; columns?: string[]; rows?: (string | number | null)[][]; truncated?: boolean; message?: string };
102 const m = JSON.parse(text) as { ok: boolean; columns?: string[]; rows?: (string | number | null)[][]; truncated?: boolean; rowsRead?: number; message?: string };
98103 done(
99104 m.ok
100 ? { ok: true, columns: m.columns ?? [], rows: m.rows ?? [], truncated: m.truncated === true }
105 ? { ok: true, columns: m.columns ?? [], rows: m.rows ?? [], truncated: m.truncated === true,
106 rowsRead: m.rowsRead ?? (m.rows ?? []).length, elapsedMs: Math.max(1, Math.round(performance.now() - t0)) }
101107 : { ok: false, error: "sql", message: String(m.message) },
102108 );
103109 } catch {
104 // The child died before it answered (a heap limit takes the process's allocator with it): the query failed.
110 // The child died before it answered (a heap limit takes the process's allocator with it):
111 // the query failed.
105112 done({ ok: false, error: "sql", message: "The query used too much memory or stopped." });
106113 }
107114 },
src/data/query_worker.ts +11 −6
@@ -1,9 +1,10 @@
11 /**
22 * Runs one query in a process of its own (`query_child.ts`), so that a slow one can be stopped by
33 * ending the process and a greedy one by the heap limit that process sets for itself: SQLite's heap
4 * limit is the whole process's, so it must never be set in the forge's (2026-11-13, 0.7.585: a data query
5 * set it on a thread and every connection of the forge failed with "out of memory"). Secret columns are shadowed by temporary views that
6 * show the mask, so an alias or a subquery cannot read them.
4 * limit is the whole process's, so it must never be set in the forge's (2026-11-13, 0.7.585: a data
5 * query set it on a thread and every connection of the forge failed with "out of memory"). Secret
6 * columns are shadowed by temporary views that show the mask, so an alias or a subquery cannot read
7 * them.
78 */
89
910 import { Database } from "bun:sqlite";
@@ -16,14 +17,16 @@ export interface Job {
1617 readonly limit: number;
1718 /** Bytes of result text kept before the rest is dropped. */
1819 readonly bytes: number;
19 /** Memory SQLite may hold for this query: the child's own, the process-wide limit is its alone. */
20 /**
21 * Memory SQLite may hold for this query: the child's own, the process-wide limit is its alone.
22 */
2023 readonly heap: number;
2124 }
2225
2326 const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
2427
2528 export type Message =
26 | { ok: true; columns: string[]; rows: (string | number | null)[][]; truncated: boolean }
29 | { ok: true; columns: string[]; rows: (string | number | null)[][]; truncated: boolean; rowsRead: number }
2730 | { ok: false; message: string };
2831
2932 export function runJob(job: Job): Message {
@@ -66,7 +69,9 @@ export function runJob(job: Job): Message {
6669 const out: (string | number | null)[][] = [];
6770 let truncated = false;
6871 let size = 0;
72 let rowsRead = 0;
6973 for (const row of stmt.values() as unknown[][]) {
74 rowsRead++;
7075 const cells = row.map(cell);
7176 size += cells.reduce<number>((n, v) => n + (typeof v === "string" ? v.length : 8), 0);
7277 if (out.length >= job.limit || size > job.bytes) {
@@ -75,7 +80,7 @@ export function runJob(job: Job): Message {
7580 }
7681 out.push(cells);
7782 }
78 return { ok: true, columns: [...columns], rows: out, truncated };
83 return { ok: true, columns: [...columns], rows: out, truncated, rowsRead };
7984 } catch (error) {
8085 return { ok: false, message: error instanceof Error ? error.message : String(error) };
8186 } finally {
src/tui/screens/data.ts +1 −1
@@ -59,7 +59,7 @@ function resultScreen(ctx: Ctx, sql: string, r: Extract<QueryResult, { ok: true
5959 keys: [binding(["j", "k"], "move"), binding("y", "yank row as JSON"), binding("esc", "back")],
6060 render(area, s) {
6161 const c = ctx.theme.c;
62 const note = `${r.rows.length} rows${r.truncated ? " (cut)" : ""} · ${sql}`;
62 const note = `${r.rowsRead} rows read${r.truncated ? " (cut)" : ""} · ${r.elapsedMs} ms · ${sql}`;
6363 s.text(area.x + 1, area.y, truncate(note, area.w - 2), style(c.mutedFg, c.bg), area.x + area.w);
6464 s.text(area.x + 1, area.y + 1, truncate(` ${r.columns.join(" | ")}`, area.w - 2), style(c.mutedFg, c.bg), area.x + area.w);
6565 const h = Math.max(1, area.h - 3);
src/web/pages/data_view.ts +2 −2
@@ -226,8 +226,8 @@ export async function dataBody(view: DataView, q: URLSearchParams): Promise<Html
226226 const ms = Math.max(1, Math.round(performance.now() - t0));
227227 out = result.ok
228228 ? html`${out}
229 <p class="muted small">${String(result.rows.length)} rows${result.truncated ? " (cut)" : ""} · ${String(ms)} ms</p>
230 ${result.rows.length === 0 ? empty({ title: "No rows" }) : grid(result.columns, result.rows, "Query result", (c) => html`<th scope="col">${c}</th>`, false)}`
229 ${result.rows.length === 0 ? empty({ title: "No rows" }) : grid(result.columns, result.rows, "Query result", (c) => html`<th scope="col">${c}</th>`, false)}
outdated

The issue asks for the line under the table, and this renders it above the grid. Move it after the grid/empty block. Also, rowsRead equals rows.length here, so "rows read" is a misleading label (docs say rows returned, not scanned). Use honest wording or count what was actually read.

230 <p class="muted small">${String(result.rowsRead)} rows read${result.truncated ? " (cut)" : ""} · ${String(result.elapsedMs)} ms</p>`
231231 : html`${out}<p role="alert" class="muted">${result.message} · ${String(ms)} ms</p>`;
232232 }
233233 return out;
test/api-data.test.ts +7 −2
@@ -1,4 +1,7 @@
1 /** `/api/v1/repos/:o/:n/data…` and MCP `data` with `repo`: the web's permissions, read-only SQL, masking, the 1,000-row cap. */
1 /**
2 * `/api/v1/repos/:o/:n/data…` and MCP `data` with `repo`: the web's permissions, read-only SQL,
3 * masking, the 1,000-row cap.
4 */
25 import { afterAll, beforeAll, expect, test } from "bun:test";
36 import { Database } from "bun:sqlite";
47 import { mkdirSync } from "node:fs";
@@ -100,7 +103,8 @@ test("permission parity with the web: a reader, a stranger and a guest get 404s"
100103 }),
101104 ).then(async () => {
102105 expect(listSources(app.db, dataDir, undefined)).toEqual([]);
103 // A reader's token and a signed-out caller meet the same refusal: 404 for the token, 401 for no identity.
106 // A reader's token and a signed-out caller meet the same refusal: 404 for the token, 401 for no
107 // identity.
104108 expect((await post(tok["bob"]!, "/production/app.db/query", "SELECT 1")).status).toBe(404);
105109 expect((await post("x", "/production/app.db/query", "SELECT 1")).status).toBe(401);
106110 const m = await mcp(tok["bob"]!, { repo: "alice/notes" });
@@ -123,6 +127,7 @@ test("masking: secret columns show dots in rows and queries, over REST and MCP",
123127 expect(q).not.toContain("tok-secret-1");
124128 const m = await mcp(tok["alice"]!, { repo: "alice/notes", env: "production", file: "app.db", sql: "SELECT * FROM users" });
125129 expect(JSON.stringify(m.body)).not.toContain("tok-secret-1");
130 expect(JSON.stringify(m.body)).toMatch(/rowsRead.*elapsedMs|elapsedMs.*rowsRead/);
126131 });
127132
128133 test("read-only: writes are refused and change nothing; a read token may query", async () => {
test/data-door.test.ts +3 −1
@@ -166,7 +166,9 @@ describe("the data door", () => {
166166 test("the SQL box runs a read-only query from the address and shows rows and ms", async () => {
167167 const ok = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("SELECT name FROM users ORDER BY id")}`, "alice")).text();
168168 expect(ok).toContain("ann");
169 expect(ok).toMatch(/\d+ ms/);
169 expect(ok).toMatch(/2 rows read · \d+ ms/);

This checks the text but not its place. Add an assertion that the line comes after the table, so a test fails when it is above it.

170 expect(ok.indexOf("</table>")).toBeGreaterThan(-1);
171 expect(ok.indexOf("rows read")).toBeGreaterThan(ok.lastIndexOf("</table>"));
170172 const sealed = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("SELECT api_token FROM users")}`, "alice")).text();
171173 expect(sealed).not.toContain("tok-ann-1");
172174 const bad = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("DELETE FROM users")}`, "alice")).text();
test/data-query-limits.test.ts +9 −1
@@ -120,8 +120,16 @@ heapTest("after a query that hits the heap limit, a connection of the forge's ow
120120 "SELECT length(hex(zeroblob(100000000))) || length(hex(zeroblob(100000000)))",
121121 );
122122 expect(r.ok).toBe(false);
123 // New pages need new allocations: with a process-wide limit left behind, this throws "out of memory".
123 // New pages need new allocations: with a process-wide limit left behind, this throws "out of
124 // memory".
124125 expect(main.query<{ s: number }, []>("SELECT sum(length(b)) AS s FROM m WHERE a >= 140000").get()?.s).toBeGreaterThan(0);
125126 expect(() => main.query("INSERT INTO m (b) VALUES ('x')").run()).not.toThrow();
126127 main.close();
127128 }, 60_000);
129
130 test("a query says how many rows it read and how long it took", async () => {
131 const r = await query(src, "SELECT id FROM t");
132 if (!r.ok) throw new Error(r.message);
133 expect(r.rowsRead).toBe(1);
134 expect(r.elapsedMs).toBeGreaterThanOrEqual(1);
135 });
test/tui-data.test.ts +1 −1
@@ -266,7 +266,7 @@ test(": caps rows as the page does and says so; a bad query is one plain sentenc
266266 const { tui } = await fresh("alice");
267267 await open_(tui, 4);
268268 await askSql(tui, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 1500) SELECT i FROM n");
269 expect(tui.screen.toText()).toContain("1000 rows (cut)");
269 expect(tui.screen.toText()).toContain("1001 rows read (cut)");
270270 press(tui, "esc");
271271 await askSql(tui, "SELECT nope FROM users");
272272 const text = tui.screen.toText();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs