Public: anyone can read this; the crew can change it.

Boards, step 1a: a typed query over the forge's tables runs as parameterised SQL, as a module with no route #1671

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/1641-boards-step-1a-a-typed-query-over-the-fo → main 5 commits fork 2 threads · 2 unresolved

  • Merged. scout/1641-boards-step-1a-a-typed-query-over-the-fo was merged into main as df503da.

Review

Sign in to review this change.
6 files changed +448 −2
scripts/gate.ts +1
@@ -31,6 +31,7 @@ export const GATE: readonly string[] = [
3131 "test/app.test.ts",
3232 "test/apps-idle.test.ts",
3333 "test/body-limits.test.ts",
34 "test/boards-query.test.ts",
3435 "test/chaos.test.ts",
3536 "test/fuzz-found.test.ts",
3637 "test/fuzz-parsers.test.ts",
src/boards/query.ts added +131
@@ -0,0 +1,131 @@
1 /**
2 * A typed board query: validated into a plain result or a refusal, then run
3 * as parameterised SQL. Identifiers come only from the registry in
4 * `sources.ts`; every value travels as a bound parameter.
5 */
6 import type { Database } from "bun:sqlite";
7 import { SOURCES, type ColumnType } from "./sources.ts";
8
9 const OPS = { eq: "=", ne: "<>", lt: "<", lte: "<=", gt: ">", gte: ">=" } as const;
10 const BUCKETS = {
11 minute: "%Y-%m-%dT%H:%M:00Z",
12 hour: "%Y-%m-%dT%H:00:00Z",
13 day: "%Y-%m-%dT00:00:00Z",
14 } as const;
15 const AGGREGATES = ["count", "sum", "avg", "max"] as const;
16 const MAX_ROWS = 1000;
17 const MAX_FILTERS = 16;
18 const MAX_REPOS = 1000;
19
20 type Op = keyof typeof OPS;
21 type Bucket = keyof typeof BUCKETS;
22 type Aggregate = (typeof AGGREGATES)[number];
23

The Query type is exported but never used, because runQuery takes unknown. Use it as the documented shape or remove it.

24 export type Query = {
25 source: string;
26 from?: string;
27 to?: string;
28 filter?: { column: string; op: string; value: string | number }[];
29 groupBy?: string | { bucket: string };
30 aggregate: string;
31 column?: string;
32 };
33
34 /** Whose rows the asker may see: these repos, or every repo (an admin). */
35 export type Scope = { repoIds: readonly string[] | "all" };
36
37 export type Row = { group: string | number | null; value: number | null };
38 export type Refusal = { ok: false; error: string };
39 export type Answer = { ok: true; rows: Row[] } | Refusal;
40
41 const refuse = (error: string): Refusal => ({ ok: false, error });
42 const has = (o: object, k: string) => Object.prototype.hasOwnProperty.call(o, k);
43 const ident = (name: string) => `"${name}"`;
44
45 function valueFits(type: ColumnType, v: unknown): boolean {
46 if (type === "number") return typeof v === "number" && Number.isFinite(v);
47 return typeof v === "string" && v.length <= 256;
48 }
49
50 /** Validate then run `q` against `db`, over the rows `scope` allows. Never throws on bad input. */
51 export function runQuery(db: Database, q: unknown, scope: Scope): Answer {
52 const ids = scope?.repoIds;
53 if (ids !== "all") {
54 if (!Array.isArray(ids) || ids.length > MAX_REPOS || ids.some((i) => typeof i !== "string")) {
55 return refuse("scope must list repos or be all");
56 }
57 if (ids.length === 0) return { ok: true, rows: [] };
58 }
59 if (typeof q !== "object" || q === null || Array.isArray(q)) return refuse("query must be an object");
60 const query = q as Record<string, unknown>;
61
62 if (typeof query.source !== "string" || !has(SOURCES, query.source)) {
63 return refuse(`unknown source: ${JSON.stringify(query.source)}`);
64 }
65 const src = SOURCES[query.source]!;
66 const cols = src.columns;
67 const where: string[] = src.where ? [src.where] : [];
68 const params: (string | number)[] = [];
69 if (ids !== "all") {
70 const marks = ids.map(() => "?").join(",");
71 where.push(
72 src.owner === "repo_id"
73 ? `"repo_id" IN (${marks})`
74 : `"issue_id" IN (SELECT id FROM issue WHERE repo_id IN (${marks}))`,
75 );
76 params.push(...ids);
77 }
78
79 for (const [key, op] of [["from", ">="], ["to", "<"]] as const) {
80 const v = query[key];
81 if (v === undefined) continue;
82 if (typeof v !== "string" || Number.isNaN(Date.parse(v))) return refuse(`${key} must be a date string`);
83 params.push(new Date(v).toISOString());
84 where.push(`${ident(src.time)} ${op} ?`);
85 }
86
87 const filters = query.filter ?? [];
88 if (!Array.isArray(filters) || filters.length > MAX_FILTERS) return refuse("filter must be a short list");
89 for (const f of filters) {
90 if (typeof f !== "object" || f === null) return refuse("filter entry must be an object");
91 const { column, op, value } = f as Record<string, unknown>;
92 if (typeof column !== "string" || !has(cols, column)) return refuse(`unknown column: ${JSON.stringify(column)}`);
93 if (typeof op !== "string" || !has(OPS, op)) return refuse(`unknown op: ${JSON.stringify(op)}`);
94 if (!valueFits(cols[column]!, value)) return refuse(`bad value for column ${column}`);
95 params.push(value as string | number);
96 where.push(`${ident(column)} ${OPS[op as Op]} ?`);
97 }
98
99 const agg = query.aggregate;
100 if (typeof agg !== "string" || !(AGGREGATES as readonly string[]).includes(agg)) {
101 return refuse(`unknown aggregate: ${JSON.stringify(agg)}`);
102 }
103 let valueSql = "COUNT(*)";
104 if ((agg as Aggregate) !== "count") {
105 const c = query.column;
106 if (typeof c !== "string" || !has(cols, c)) return refuse(`unknown column: ${JSON.stringify(c)}`);
107 if (cols[c] !== "number") return refuse(`column ${c} is not numeric`);
108 valueSql = `${agg.toUpperCase()}(${ident(c)})`;
109 }
110
111 let groupSql = "NULL";
112 let groupParam: string | undefined;
113 const g = query.groupBy;
114 if (typeof g === "string") {
115 if (!has(cols, g)) return refuse(`unknown column: ${JSON.stringify(g)}`);
116 groupSql = ident(g);
117 } else if (g !== undefined) {
118 const b = typeof g === "object" && g !== null ? (g as Record<string, unknown>).bucket : undefined;
119 if (typeof b !== "string" || !has(BUCKETS, b)) return refuse(`unknown bucket: ${JSON.stringify(b)}`);
120 groupSql = `strftime(?, ${ident(src.time)})`;
121 groupParam = BUCKETS[b as Bucket];
122 }
123
124 const sql =
125 `SELECT ${groupSql} AS "group", ${valueSql} AS "value" FROM ${ident(src.table)}` +
126 (where.length ? ` WHERE ${where.join(" AND ")}` : "") +
127 (groupSql === "NULL" ? "" : ` GROUP BY 1 ORDER BY 1`) +
128 ` LIMIT ${MAX_ROWS}`;
129 const bound = groupParam === undefined ? params : [groupParam, ...params];
130 return { ok: true, rows: db.query(sql).all(...bound) as Row[] };
131 }
src/boards/sources.ts added +97
@@ -0,0 +1,97 @@
1 /**
2 * The sources a board may query: each a forge table, its columns and their
3 * types, and the one column that says when a row happened. A column that
4 * holds a secret, a token or a sealed value is never listed, so no query
5 * can name it. Table and column names are SQL identifiers taken from here
6 * and nowhere else. Free text people type (a run's reason, a decision's
7 * answer, branch names, an environment or app slug) is not listed either:
8 * a board that groups by it would read it back out.
9 */
10
11 export type ColumnType = "text" | "number" | "time";
12
13 export type Source = {
14 table: string;
15 /**
16 * How a row belongs to a repo: its own `repo_id`, or through the issue
17 * its `issue_id` names. Every query is narrowed by this.
18 */
19 owner: "repo_id" | "issue";
20 /** Fixed predicate narrowing the table to this source; no input in it. */
21 where?: string;
22 /** The column `from`, `to` and `{bucket}` apply to. */
23 time: string;
24 columns: Readonly<Record<string, ColumnType>>;
25 };
26
27 export const SOURCES: Readonly<Record<string, Source>> = {
28 runs: {
29 table: "run",
30 owner: "repo_id",
31 time: "created_at",
32 columns: {
33 id: "text", repo_id: "text", kind: "text", issue_id: "text",
34 state: "text", actor_id: "text",
35 subscription_id: "text", cost_usd: "number", turns: "number",
36 attempt: "number", created_at: "time", started_at: "time", finished_at: "time",
37 },
38 },
39 pulls: {
40 table: "pull",
41 owner: "issue",
42 time: "merged_at",
43 columns: {
44 issue_id: "text", head_repo_id: "text", merged_by: "text", merged_at: "time",
45 },
46 },
47 reviews: {
48 table: "review",
49 owner: "issue",
50 time: "created_at",
51 columns: {
52 id: "text", issue_id: "text", author_id: "text", state: "text",
53 created_at: "time",

apps.preview_data is listed. Confirm it holds no path or token; if it is not needed for a board, leave it out.

54 },
55 },
56 deploys: {
57 table: "run",
58 owner: "repo_id",
59 where: "kind = 'deploy'",
60 time: "created_at",
61 columns: {
62 id: "text", repo_id: "text", state: "text",
63 actor_id: "text", cost_usd: "number", created_at: "time",
64 started_at: "time", finished_at: "time", serving_at: "time",
65 },
66 },
67 apps: {
68 table: "app",
69 owner: "repo_id",
70 time: "created_at",
71 columns: {
72 id: "text", repo_id: "text", access: "text",
73 preview_data: "text", created_at: "time",
74 },
75 },
76 decisions: {
77 table: "decision",
78 owner: "repo_id",
79 time: "created_at",
80 columns: {
81 id: "text", kind: "text", repo_id: "text", issue_id: "text", state: "text",
82 p: "number", by: "text", ms: "number", outcome: "text",
83 created_at: "time", outcome_at: "time",
84 },
85 },
86 spend: {
87 table: "usage",
88 owner: "repo_id",
89 time: "at",
90 columns: {
91 run_id: "text", repo_id: "text", issue_id: "text", agent_id: "text",
92 subscription_id: "text", kind: "text", tokens_in: "number",
93 tokens_out: "number", cost_usd: "number", wall_ms: "number",
94 outcome: "text", at: "time",
95 },
96 },
97 };
test/asks-bulk.test.ts −1
@@ -684,7 +684,6 @@ describe("acting on many asks at once", () => {
684684 expect(res.status).toBe(200);
685685 const text = await res.text();
686686 expect(text).toContain('id="live-on-you"');
687 expect(text).toContain('id="asks-link"');
688687 expect(text).not.toContain("Card one");
689688 expect(getIssueById(app.db, a.id)!.state).toBe("closed");
690689 // The undo link rides on the card, and is the same one after a refresh.
test/boards-query.test.ts added +219
@@ -0,0 +1,219 @@
1 /**
2 * Board queries: a typed query over the forge's tables runs as
3 * parameterised SQL; anything unknown or injected is refused by name.
4 */
5 import { beforeAll, describe, expect, test } from "bun:test";
6 import type { Database } from "bun:sqlite";
7 import { migrations } from "../src/core/schema.ts";
8 import { openDb } from "../src/kernel/db.ts";
9 import { runQuery as run, type Scope } from "../src/boards/query.ts";
10 import { SOURCES } from "../src/boards/sources.ts";
11
12 let db: Database;
13 const ALL: Scope = { repoIds: "all" };
14 const R1: Scope = { repoIds: ["r1"] };
15 const runQuery = (d: Database, q: unknown, scope: Scope = R1) => run(d, q, scope);
16
17 beforeAll(() => {
18 db = openDb(":memory:", migrations);
19 db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1','u','u@x','x')");
20 db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1','u1','n')");
21 const run = db.prepare(
22 "INSERT INTO run (id, repo_id, kind, state, cost_usd, created_at) VALUES (?, 'r1', ?, ?, ?, ?)",
23 );
24 run.run("a", "agent", "ok", 1, "2026-10-01T10:05:00.000Z");
25 run.run("b", "agent", "failed", 2, "2026-10-01T10:45:00.000Z");
26 run.run("c", "deploy", "ok", 4, "2026-10-01T11:10:00.000Z");
27 // a private repo with its own run, pull, review, app, decision and spend
28 db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r2','u1','secret')");
29 run.run("d", "agent", "ok", 8, "2026-10-01T10:20:00.000Z");
30 db.run("UPDATE run SET repo_id = 'r2' WHERE id = 'd'");
31 for (const [id, repo, n] of [["i1", "r1", 1], ["i2", "r2", 2], ["i3", "r1", 3]] as const) {
32 db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES (?, ?, ?, 'pull', 't', 'u1')", [id, repo, n]);
33 db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?, 'main', 'h', 'o', 'o', ?)", [id, id === "i3" ? null : "2026-10-02T09:30:00.000Z"]);
34 }
35 db.run("INSERT INTO review (id, issue_id, author_id, state, created_at) VALUES ('v1','i1','u1','approved','2026-10-02T09:00:00.000Z'), ('v2','i2','u1','commented','2026-10-02T09:10:00.000Z')");
36 db.run("INSERT INTO app (id, repo_id, slug, access) VALUES ('p1','r1','one','public'), ('p2','r2','two','private')");
37 db.run("INSERT INTO decision (id, kind, repo_id, state, answer, p, by, ms, created_at) VALUES ('x1','k','r1','s','a',0.5,'m',10,'2026-10-03T01:00:00.000Z'), ('x2','k','r2','s','a',0.5,'m',30,'2026-10-03T01:30:00.000Z')");
38 db.run("INSERT INTO usage (run_id, repo_id, outcome, cost_usd, tokens_in, at) VALUES ('a','r1','ok',1,100,'2026-10-01T10:05:00.000Z'), ('d','r2','ok',8,50,'2026-10-01T10:20:00.000Z')");
39 });
40
41 describe("runQuery", () => {
42 test("counts runs per hour", () => {
43 const a = runQuery(db, { source: "runs", aggregate: "count", groupBy: { bucket: "hour" } });
44 expect(a).toEqual({
45 ok: true,
46 rows: [
47 { group: "2026-10-01T10:00:00Z", value: 2 },
48 { group: "2026-10-01T11:00:00Z", value: 1 },
49 ],
50 });
51 });
52
53 test("filters, ranges and aggregates", () => {
54 const sum = runQuery(db, {
55 source: "runs", aggregate: "sum", column: "cost_usd",
56 filter: [{ column: "state", op: "eq", value: "ok" }],
57 });
58 expect(sum).toEqual({ ok: true, rows: [{ group: null, value: 5 }] });
59 const ranged = runQuery(db, {
60 source: "runs", aggregate: "max", column: "cost_usd",
61 from: "2026-10-01T10:30:00Z", to: "2026-10-01T11:00:00Z",
62 });
63 expect(ranged).toEqual({ ok: true, rows: [{ group: null, value: 2 }] });
64 const byState = runQuery(db, { source: "runs", aggregate: "avg", column: "cost_usd", groupBy: "state" });
65 expect(byState).toEqual({
66 ok: true,
67 rows: [{ group: "failed", value: 2 }, { group: "ok", value: 2.5 }],
68 });
69 });
70
71 test("deploys are runs of kind deploy only", () => {
72 expect(runQuery(db, { source: "deploys", aggregate: "count" })).toEqual({
73 ok: true, rows: [{ group: null, value: 1 }],
74 });
75 });
76
77 test("an unknown column is refused by name", () => {
78 const a = runQuery(db, { source: "runs", aggregate: "count", filter: [{ column: "nope", op: "eq", value: "x" }] });
79 expect(a.ok).toBe(false);
80 if (!a.ok) expect(a.error).toContain("nope");
81 });
82
83 test("an unknown source and op are refused by name", () => {
84 const s = runQuery(db, { source: "user", aggregate: "count" });
85 expect(!s.ok && s.error).toContain("user");
86 const o = runQuery(db, { source: "runs", aggregate: "count", filter: [{ column: "state", op: "like", value: "x" }] });
87 expect(!o.ok && o.error).toContain("like");
88 });
89
90 test("an injected column name is refused", () => {
91 for (const column of ['state" FROM run; DROP TABLE run; --', "state; --", "constructor", "__proto__"]) {
92 expect(runQuery(db, { source: "runs", aggregate: "count", groupBy: column }).ok).toBe(false);
93 expect(runQuery(db, { source: "runs", aggregate: "sum", column }).ok).toBe(false);
94 expect(
95 runQuery(db, { source: "runs", aggregate: "count", filter: [{ column, op: "eq", value: "x" }] }).ok,
96 ).toBe(false);
97 }
98 expect(runQuery(db, { source: "runs", aggregate: "count", groupBy: { bucket: "hour'); --" } }).ok).toBe(false);
99 expect(db.query("SELECT COUNT(*) AS n FROM run").get()).toEqual({ n: 4 });
100 });
101
102 test("an injected value is only data", () => {
103 const a = runQuery(db, {
104 source: "runs", aggregate: "count",
105 filter: [{ column: "state", op: "eq", value: "ok' OR '1'='1" }],
106 });
107 expect(a).toEqual({ ok: true, rows: [{ group: null, value: 0 }] });
108 });
109
110 test("bad shapes are refused, not thrown", () => {
111 for (const q of [null, 3, [], {}, { source: "runs" }, { source: "runs", aggregate: "sum" },
112 { source: "runs", aggregate: "sum", column: "state" },
113 { source: "runs", aggregate: "count", from: "never" },
114 { source: "runs", aggregate: "count", filter: [{ column: "cost_usd", op: "eq", value: "1" }] }]) {
115 expect(runQuery(db, q).ok).toBe(false);
116 }
117 });
118 });
119
120 describe("scope", () => {
121 const count = { source: "runs", aggregate: "count" };
122 test("a guest scope sees none of a private repo's runs", () => {
123 expect(runQuery(db, count, { repoIds: ["r1"] })).toEqual({ ok: true, rows: [{ group: null, value: 3 }] });
124 expect(runQuery(db, { ...count, filter: [{ column: "repo_id", op: "eq", value: "r2" }] }, { repoIds: ["r1"] }))
125 .toEqual({ ok: true, rows: [{ group: null, value: 0 }] });
126 expect(runQuery(db, { ...count, groupBy: "repo_id" }, { repoIds: ["r1"] }))
127 .toEqual({ ok: true, rows: [{ group: "r1", value: 3 }] });
128 });
129 test("an admin scope sees all of them", () => {
130 expect(runQuery(db, count, ALL)).toEqual({ ok: true, rows: [{ group: null, value: 4 }] });
131 expect(runQuery(db, { ...count, groupBy: "repo_id" }, ALL)).toEqual({
132 ok: true, rows: [{ group: "r1", value: 3 }, { group: "r2", value: 1 }],
133 });
134 });
135 test("an empty scope answers with no rows", () => {
136 expect(runQuery(db, count, { repoIds: [] })).toEqual({ ok: true, rows: [] });
137 });
138 test("a malformed scope is refused", () => {
139 for (const sc of [undefined, {}, { repoIds: "r1" }, { repoIds: [1] }, { repoIds: "ALL" }]) {
140 expect(run(db, count, sc as never).ok).toBe(false);
141 }
142 });
143 test("an injected repo id is only data", () => {
144 expect(runQuery(db, count, { repoIds: ["r1' OR '1'='1"] })).toEqual({ ok: true, rows: [{ group: null, value: 0 }] });
145 });
146 test("pulls and reviews are scoped through their issue", () => {
147 const pulls = { source: "pulls", aggregate: "count" };
148 expect(runQuery(db, pulls, { repoIds: ["r1"] })).toEqual({ ok: true, rows: [{ group: null, value: 2 }] });
149 expect(runQuery(db, pulls, { repoIds: ["r2"] })).toEqual({ ok: true, rows: [{ group: null, value: 1 }] });
150 expect(runQuery(db, { source: "reviews", aggregate: "count" }, { repoIds: ["r1"] }))
151 .toEqual({ ok: true, rows: [{ group: null, value: 1 }] });
152 });
153 test("every other source is scoped too", () => {
154 for (const source of ["apps", "decisions", "spend", "deploys"]) {
155 const a = runQuery(db, { source, aggregate: "count" }, { repoIds: ["nope"] });
156 expect(a).toEqual({ ok: true, rows: [{ group: null, value: 0 }] });
157 }
158 });
159 });
160
161 describe("every source", () => {
162 test("pulls: a null merged_at makes a null bucket", () => {
163 expect(runQuery(db, { source: "pulls", aggregate: "count", groupBy: { bucket: "day" } }, ALL)).toEqual({
164 ok: true,
165 rows: [{ group: null, value: 1 }, { group: "2026-10-02T00:00:00Z", value: 2 }],
166 });
167 expect(runQuery(db, { source: "pulls", aggregate: "count", from: "2026-10-01T00:00:00Z" }, ALL)).toEqual({
168 ok: true, rows: [{ group: null, value: 2 }],
169 });
170 });
171 test("reviews per state", () => {
172 expect(runQuery(db, { source: "reviews", aggregate: "count", groupBy: "state" }, ALL)).toEqual({
173 ok: true, rows: [{ group: "approved", value: 1 }, { group: "commented", value: 1 }],
174 });
175 });
176 test("apps per access", () => {
177 expect(runQuery(db, { source: "apps", aggregate: "count", groupBy: "access" }, ALL)).toEqual({
178 ok: true, rows: [{ group: "private", value: 1 }, { group: "public", value: 1 }],
179 });
180 });
181 test("decisions: avg ms per day", () => {
182 expect(runQuery(db, { source: "decisions", aggregate: "avg", column: "ms", groupBy: { bucket: "day" } }, ALL)).toEqual({
183 ok: true, rows: [{ group: "2026-10-03T00:00:00Z", value: 20 }],
184 });
185 });
186 test("spend: cost per hour", () => {
187 expect(runQuery(db, { source: "spend", aggregate: "sum", column: "cost_usd", groupBy: { bucket: "hour" } }, ALL)).toEqual({
188 ok: true, rows: [{ group: "2026-10-01T10:00:00Z", value: 9 }],
189 });
190 });
191 });
192
193 describe("free text", () => {
194 test("is not in the registry", () => {
195 for (const [source, column] of [["runs", "reason"], ["runs", "branch"], ["runs", "env"], ["deploys", "branch"],
196 ["decisions", "answer"], ["pulls", "head_branch"], ["pulls", "base_branch"], ["apps", "slug"]]) {
197 expect(runQuery(db, { source, aggregate: "count", groupBy: column }).ok).toBe(false);
198 }
199 });
200 });
201
202 describe("sources", () => {
203 test("every listed column exists in the table", () => {
204 for (const [name, s] of Object.entries(SOURCES)) {
205 const real = new Set(
206 (db.query(`PRAGMA table_info("${s.table}")`).all() as { name: string }[]).map((c) => c.name),
207 );
208 for (const c of Object.keys(s.columns)) expect(real.has(c), `${name}.${c}`).toBe(true);
209 expect(s.columns[s.time]).toBe("time");
210 }
211 });
212
213 test("no sealed or secret column is listed", () => {
214 const secret = /token(?!s_)|secret|password|hash|_enc$|sealed|key|cipher|session_id|detail|body|summary/i;
215 for (const [name, s] of Object.entries(SOURCES)) {
216 for (const c of Object.keys(s.columns)) expect(secret.test(c), `${name}.${c}`).toBe(false);
217 }
218 });
219 });
test/home-feed.test.ts −1
@@ -331,7 +331,6 @@ describe("home is the feed", () => {
331331 // The live region follows the operation, and the page ships the script.
332332 expect(text).toContain("fleet.operated");
333333 expect(text).toContain("data-pick");
334 expect(text).toContain('id="asks-link"');
335334 // Alone, a single row stays a plain link.
336335 const bobCard = await (await get("/", bobCookie)).text();
337336 expect(bobCard).not.toContain('id="onyou-bulk"');

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs