Public: anyone can read this; the crew can change it.
A data query says how many rows it read and how long it took, on the page and in the MCP answer #13409
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13385-a-data-query-says-how-many-rows-it-read → main 5 commits fork 2 threads · 2 unresolved
9 files
changed +46
−17
docs/features/data-browser.md +2
| @@ -21,3 +21,5 @@ API and MCP by repository (`data/repo_scope.ts`, `http/api/data.ts`): `GET /api/ | ||
| 21 | 21 | ## On the web (#7580) |
| 22 | 22 | |
| 23 | 23 | Three doors, one body (`src/web/pages/data_view.ts`): `/data` (every file you may see), `/admin/data` (the forge's own database, admins; the Forge data button on `/data`) and `/:owner/:name/data` (an app's Data tab for its writers: environments → files → tables → rows; a reader gets the 404). Rows sort by header (`sort`, `dir`), filter by text on one column or all (`q`, `col`), JSON cells are pretty-printed, and a copy button per row puts it on the clipboard as JSON. Sealed columns show `••••••` and cannot be sorted or searched. The SQL box is a GET form (`sql`, capped at 4,000 characters), so a query is an address to share; it shows the rows and the elapsed ms. The sorted column carries `aria-sort` and a muted arrow from CSS; a wide grid scrolls sideways only. Not done: saved queries, export, editing rows. |
| 24 | ||
| 25 | Cost of a query: `query` also returns `rowsRead` (rows the query produced, the one that tripped a cap included) and `elapsedMs` (at least 1). The page prints them as one muted line under the table (`N rows read · X ms`), the terminal's `:` result shows the same in its header line, and the MCP `data` answer with `sql` carries both fields. They count rows the query produced, not rows SQLite scanned. |
src/data/query.ts +10 −3
| @@ -23,6 +23,10 @@ export type QueryResult = | ||
| 23 | 23 | readonly columns: readonly string[]; |
| 24 | 24 | readonly rows: readonly (readonly (string | number | null)[])[]; |
| 25 | 25 | readonly truncated: boolean; |
| 26 | /** Rows the query produced before it stopped, the row that tripped a cap included. */ | |
| 27 | readonly rowsRead: number; | |
| 28 | /** Milliseconds from start to answer, at least 1. */ | |
| 29 | readonly elapsedMs: number; | |
| 26 | 30 | } |
| 27 | 31 | | { |
| 28 | 32 | readonly ok: false; |
| @@ -55,6 +59,7 @@ export function query( | ||
| 55 | 59 | message: "Busy with other queries: try again in a moment.", |
| 56 | 60 | }); |
| 57 | 61 | running++; |
| 62 | const t0 = performance.now(); | |
| 58 | 63 | return new Promise((resolve) => { |
| 59 | 64 | const child = Bun.spawn([process.execPath, new URL("./query_child.ts", import.meta.url).pathname], { |
| 60 | 65 | stdin: "pipe", |
| @@ -94,14 +99,16 @@ export function query( | ||
| 94 | 99 | void new Response(child.stdout).text().then( |
| 95 | 100 | (text) => { |
| 96 | 101 | try { |
| 97 | const m = JSON.parse(text) as { ok: boolean; columns?: string[]; rows?: (string | number | null)[][]; truncated?: boolean; message?: string }; | |
| 102 | const m = JSON.parse(text) as { ok: boolean; columns?: string[]; rows?: (string | number | null)[][]; truncated?: boolean; rowsRead?: number; message?: string }; | |
| 98 | 103 | done( |
| 99 | 104 | m.ok |
| 100 | ? { ok: true, columns: m.columns ?? [], rows: m.rows ?? [], truncated: m.truncated === true } | |
| 105 | ? { ok: true, columns: m.columns ?? [], rows: m.rows ?? [], truncated: m.truncated === true, | |
| 106 | rowsRead: m.rowsRead ?? (m.rows ?? []).length, elapsedMs: Math.max(1, Math.round(performance.now() - t0)) } | |
| 101 | 107 | : { ok: false, error: "sql", message: String(m.message) }, |
| 102 | 108 | ); |
| 103 | 109 | } catch { |
| 104 | // The child died before it answered (a heap limit takes the process's allocator with it): the query failed. | |
| 110 | // The child died before it answered (a heap limit takes the process's allocator with it): | |
| 111 | // the query failed. | |
| 105 | 112 | done({ ok: false, error: "sql", message: "The query used too much memory or stopped." }); |
| 106 | 113 | } |
| 107 | 114 | }, |
src/data/query_worker.ts +11 −6
| @@ -1,9 +1,10 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * Runs one query in a process of its own (`query_child.ts`), so that a slow one can be stopped by |
| 3 | 3 | * ending the process and a greedy one by the heap limit that process sets for itself: SQLite's heap |
| 4 | * limit is the whole process's, so it must never be set in the forge's (2026-11-13, 0.7.585: a data query | |
| 5 | * set it on a thread and every connection of the forge failed with "out of memory"). Secret columns are shadowed by temporary views that | |
| 6 | * show the mask, so an alias or a subquery cannot read them. | |
| 4 | * limit is the whole process's, so it must never be set in the forge's (2026-11-13, 0.7.585: a data | |
| 5 | * query set it on a thread and every connection of the forge failed with "out of memory"). Secret | |
| 6 | * columns are shadowed by temporary views that show the mask, so an alias or a subquery cannot read | |
| 7 | * them. | |
| 7 | 8 | */ |
| 8 | 9 | |
| 9 | 10 | import { Database } from "bun:sqlite"; |
| @@ -16,14 +17,16 @@ export interface Job { | ||
| 16 | 17 | readonly limit: number; |
| 17 | 18 | /** Bytes of result text kept before the rest is dropped. */ |
| 18 | 19 | readonly bytes: number; |
| 19 | /** Memory SQLite may hold for this query: the child's own, the process-wide limit is its alone. */ | |
| 20 | /** | |
| 21 | * Memory SQLite may hold for this query: the child's own, the process-wide limit is its alone. | |
| 22 | */ | |
| 20 | 23 | readonly heap: number; |
| 21 | 24 | } |
| 22 | 25 | |
| 23 | 26 | const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`; |
| 24 | 27 | |
| 25 | 28 | export type Message = |
| 26 | | { ok: true; columns: string[]; rows: (string | number | null)[][]; truncated: boolean } | |
| 29 | | { ok: true; columns: string[]; rows: (string | number | null)[][]; truncated: boolean; rowsRead: number } | |
| 27 | 30 | | { ok: false; message: string }; |
| 28 | 31 | |
| 29 | 32 | export function runJob(job: Job): Message { |
| @@ -66,7 +69,9 @@ export function runJob(job: Job): Message { | ||
| 66 | 69 | const out: (string | number | null)[][] = []; |
| 67 | 70 | let truncated = false; |
| 68 | 71 | let size = 0; |
| 72 | let rowsRead = 0; | |
| 69 | 73 | for (const row of stmt.values() as unknown[][]) { |
| 74 | rowsRead++; | |
| 70 | 75 | const cells = row.map(cell); |
| 71 | 76 | size += cells.reduce<number>((n, v) => n + (typeof v === "string" ? v.length : 8), 0); |
| 72 | 77 | if (out.length >= job.limit || size > job.bytes) { |
| @@ -75,7 +80,7 @@ export function runJob(job: Job): Message { | ||
| 75 | 80 | } |
| 76 | 81 | out.push(cells); |
| 77 | 82 | } |
| 78 | return { ok: true, columns: [...columns], rows: out, truncated }; | |
| 83 | return { ok: true, columns: [...columns], rows: out, truncated, rowsRead }; | |
| 79 | 84 | } catch (error) { |
| 80 | 85 | return { ok: false, message: error instanceof Error ? error.message : String(error) }; |
| 81 | 86 | } finally { |
src/tui/screens/data.ts +1 −1
| @@ -59,7 +59,7 @@ function resultScreen(ctx: Ctx, sql: string, r: Extract<QueryResult, { ok: true | ||
| 59 | 59 | keys: [binding(["j", "k"], "move"), binding("y", "yank row as JSON"), binding("esc", "back")], |
| 60 | 60 | render(area, s) { |
| 61 | 61 | const c = ctx.theme.c; |
| 62 | const note = `${r.rows.length} rows${r.truncated ? " (cut)" : ""} · ${sql}`; | |
| 62 | const note = `${r.rowsRead} rows read${r.truncated ? " (cut)" : ""} · ${r.elapsedMs} ms · ${sql}`; | |
| 63 | 63 | s.text(area.x + 1, area.y, truncate(note, area.w - 2), style(c.mutedFg, c.bg), area.x + area.w); |
| 64 | 64 | s.text(area.x + 1, area.y + 1, truncate(` ${r.columns.join(" | ")}`, area.w - 2), style(c.mutedFg, c.bg), area.x + area.w); |
| 65 | 65 | const h = Math.max(1, area.h - 3); |
src/web/pages/data_view.ts +2 −2
| @@ -226,8 +226,8 @@ export async function dataBody(view: DataView, q: URLSearchParams): Promise<Html | ||
| 226 | 226 | const ms = Math.max(1, Math.round(performance.now() - t0)); |
| 227 | 227 | out = result.ok |
| 228 | 228 | ? html`${out} |
| 229 | <p class="muted small">${String(result.rows.length)} rows${result.truncated ? " (cut)" : ""} · ${String(ms)} ms</p> | |
| 230 | ${result.rows.length === 0 ? empty({ title: "No rows" }) : grid(result.columns, result.rows, "Query result", (c) => html`<th scope="col">${c}</th>`, false)}` | |
| 229 | ${result.rows.length === 0 ? empty({ title: "No rows" }) : grid(result.columns, result.rows, "Query result", (c) => html`<th scope="col">${c}</th>`, false)} | |
| 230 | <p class="muted small">${String(result.rowsRead)} rows read${result.truncated ? " (cut)" : ""} · ${String(result.elapsedMs)} ms</p>` | |
| 231 | 231 | : html`${out}<p role="alert" class="muted">${result.message} · ${String(ms)} ms</p>`; |
| 232 | 232 | } |
| 233 | 233 | return out; |
test/api-data.test.ts +7 −2
| @@ -1,4 +1,7 @@ | ||
| 1 | /** `/api/v1/repos/:o/:n/data…` and MCP `data` with `repo`: the web's permissions, read-only SQL, masking, the 1,000-row cap. */ | |
| 1 | /** | |
| 2 | * `/api/v1/repos/:o/:n/data…` and MCP `data` with `repo`: the web's permissions, read-only SQL, | |
| 3 | * masking, the 1,000-row cap. | |
| 4 | */ | |
| 2 | 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; |
| 3 | 6 | import { Database } from "bun:sqlite"; |
| 4 | 7 | import { mkdirSync } from "node:fs"; |
| @@ -100,7 +103,8 @@ test("permission parity with the web: a reader, a stranger and a guest get 404s" | ||
| 100 | 103 | }), |
| 101 | 104 | ).then(async () => { |
| 102 | 105 | expect(listSources(app.db, dataDir, undefined)).toEqual([]); |
| 103 | // A reader's token and a signed-out caller meet the same refusal: 404 for the token, 401 for no identity. | |
| 106 | // A reader's token and a signed-out caller meet the same refusal: 404 for the token, 401 for no | |
| 107 | // identity. | |
| 104 | 108 | expect((await post(tok["bob"]!, "/production/app.db/query", "SELECT 1")).status).toBe(404); |
| 105 | 109 | expect((await post("x", "/production/app.db/query", "SELECT 1")).status).toBe(401); |
| 106 | 110 | const m = await mcp(tok["bob"]!, { repo: "alice/notes" }); |
| @@ -123,6 +127,7 @@ test("masking: secret columns show dots in rows and queries, over REST and MCP", | ||
| 123 | 127 | expect(q).not.toContain("tok-secret-1"); |
| 124 | 128 | const m = await mcp(tok["alice"]!, { repo: "alice/notes", env: "production", file: "app.db", sql: "SELECT * FROM users" }); |
| 125 | 129 | expect(JSON.stringify(m.body)).not.toContain("tok-secret-1"); |
| 130 | expect(JSON.stringify(m.body)).toMatch(/rowsRead.*elapsedMs|elapsedMs.*rowsRead/); | |
| 126 | 131 | }); |
| 127 | 132 | |
| 128 | 133 | test("read-only: writes are refused and change nothing; a read token may query", async () => { |
test/data-door.test.ts +3 −1
| @@ -166,7 +166,9 @@ describe("the data door", () => { | ||
| 166 | 166 | test("the SQL box runs a read-only query from the address and shows rows and ms", async () => { |
| 167 | 167 | const ok = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("SELECT name FROM users ORDER BY id")}`, "alice")).text(); |
| 168 | 168 | expect(ok).toContain("ann"); |
| 169 | expect(ok).toMatch(/\d+ ms/); | |
| 169 | expect(ok).toMatch(/2 rows read · \d+ ms/); | |
This checks the text but not its place. Add an assertion that the line comes after the table, so a test fails when it is above it. |
||
| 170 | expect(ok.indexOf("</table>")).toBeGreaterThan(-1); | |
| 171 | expect(ok.indexOf("rows read")).toBeGreaterThan(ok.lastIndexOf("</table>")); | |
| 170 | 172 | const sealed = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("SELECT api_token FROM users")}`, "alice")).text(); |
| 171 | 173 | expect(sealed).not.toContain("tok-ann-1"); |
| 172 | 174 | const bad = await (await get(`/alice/notes/data?source=${src()}&sql=${encodeURIComponent("DELETE FROM users")}`, "alice")).text(); |
test/data-query-limits.test.ts +9 −1
| @@ -120,8 +120,16 @@ heapTest("after a query that hits the heap limit, a connection of the forge's ow | ||
| 120 | 120 | "SELECT length(hex(zeroblob(100000000))) || length(hex(zeroblob(100000000)))", |
| 121 | 121 | ); |
| 122 | 122 | expect(r.ok).toBe(false); |
| 123 | // New pages need new allocations: with a process-wide limit left behind, this throws "out of memory". | |
| 123 | // New pages need new allocations: with a process-wide limit left behind, this throws "out of | |
| 124 | // memory". | |
| 124 | 125 | expect(main.query<{ s: number }, []>("SELECT sum(length(b)) AS s FROM m WHERE a >= 140000").get()?.s).toBeGreaterThan(0); |
| 125 | 126 | expect(() => main.query("INSERT INTO m (b) VALUES ('x')").run()).not.toThrow(); |
| 126 | 127 | main.close(); |
| 127 | 128 | }, 60_000); |
| 129 | ||
| 130 | test("a query says how many rows it read and how long it took", async () => { | |
| 131 | const r = await query(src, "SELECT id FROM t"); | |
| 132 | if (!r.ok) throw new Error(r.message); | |
| 133 | expect(r.rowsRead).toBe(1); | |
| 134 | expect(r.elapsedMs).toBeGreaterThanOrEqual(1); | |
| 135 | }); |
test/tui-data.test.ts +1 −1
| @@ -266,7 +266,7 @@ test(": caps rows as the page does and says so; a bad query is one plain sentenc | ||
| 266 | 266 | const { tui } = await fresh("alice"); |
| 267 | 267 | await open_(tui, 4); |
| 268 | 268 | await askSql(tui, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 1500) SELECT i FROM n"); |
| 269 | expect(tui.screen.toText()).toContain("1000 rows (cut)"); | |
| 269 | expect(tui.screen.toText()).toContain("1001 rows read (cut)"); | |
| 270 | 270 | press(tui, "esc"); |
| 271 | 271 | await askSql(tui, "SELECT nope FROM users"); |
| 272 | 272 | const text = tui.screen.toText(); |
new above
offline
The issue asks for the line under the table, and this renders it above the grid. Move it after the grid/empty block. Also,
rowsReadequalsrows.lengthhere, so "rows read" is a misleading label (docs say rows returned, not scanned). Use honest wording or count what was actually read.