Public: anyone can read this; the crew can change it.

Data browser core: read any SQLite file a viewer may see — the forge's own (admins) and each app's per environment (its writers) — tables, rows, read-only SQL, secrets masked #8923

Parked opened this
Closed without merging

scout/forge:scout/7579-data-browser-core-read-any-sqlite-file-a → main 8 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
19 files changed +1036 −0
.gitignore +1
@@ -1,5 +1,6 @@
11 node_modules
22 data/
3 !src/data/
34 dist/
45 *.local
56 .DS_Store
docs/design/audit.md +1
@@ -57,6 +57,7 @@ Machine routes are not pages and have no entry: `/audit.csv`,
5757 - `/feed` — `studio/home.ts` · the composer and one timeline, what home was before it became the living pipeline (#8015) · `/` · none seen.
5858 - `/boards/data` — `pages/boards.ts` · the boards' query/data surface · `/board` · none seen.
5959 - `/asks` — `studio/asks_page.ts` · what the crew is waiting on me for · `/` "needs me" list · colours 2.
60 - `/data` — `pages/data.ts` · the SQLite files I may see, their tables and rows, secrets masked · `/system` · none seen.
6061 - `/asked` — `studio/asked_page.ts` · what I answered, as a record · `/asks` · none seen.
6162 - `/review` — `studio/review_page.ts` · every ask waiting on me, one row each, newest first · `/asks` (same asks, with buttons) · none seen.
6263 - `/flow` — `pages/flow.ts` · the pipeline from ready issue to live deploy, and the bottleneck · `/system`, `/board`, `/` · none seen. A topbar button points here.
docs/features/data-browser.md added +13
@@ -0,0 +1,13 @@
1 # Data browser (core)
2
3 `src/data/` reads SQLite files a viewer may see. Reach it at `/data` (linked from Apps for anyone who can see a source) or through the MCP tool `data`; a guest or a reader of an app sees a 404 / nothing.
4
5 - `listSources(db, dataDir, viewer)` — `forge.db` for admins; for each app the viewer can write, every `.db` under each environment's data dir. `findSource` resolves an id only through that list, so a reader gets `undefined` (a 404), and no input becomes a path.
6 - `tables(source)`, `rows(source, table, {filter, order, page})` — 50 rows a page; table and column names are checked against the schema.
7 - `query(source, sql, timeoutMs?)` — one `SELECT`/`WITH`, read-only connection (`query_only`), 1,000 rows, 5 s. It runs in a worker that is terminated at the cap, so even a query that never yields stops.
8 - Masking (`mask.ts`): columns named like token, secret, password, hash, sealed, key, cookie, … show `••••••`. Queries read through temporary views that shadow each table, so aliases and subqueries are masked too; `main.` qualifiers are refused. Masked columns cannot be searched or ordered. `ALLOW` lists `table.column` exceptions per source id.
9 - Connections open per call and close after; they never take a writer lock.
10
11 Views are masked whole when their text mentions a masked column (an alias cannot launder a secret). `FORCE` masks generic columns by source (`forge_setting.value`, …); `test/data-browser.test.ts` audits every `forge.db` column against the name rule and lists the ones reviewed as clear.
12
13 Not done: a SQL editor on the page, a terminal screen.
src/core/reserved.ts +1
@@ -86,6 +86,7 @@ export const SYSTEM_NAMES: readonly string[] = [
8686 "mcp",
8787 "metrics",
8888 "contract",
89 "data",
8990 "migrate",
9091 "mode",
9192 "new",
src/data/browse.ts added +142
@@ -0,0 +1,142 @@
1 /**
2 * Tables and rows of one source, read on a connection opened for the call
3 * and closed after it. Read-only: it takes no writer lock and a busy WAL
4 * answers in milliseconds or not at all.
5 */
6
7 import { Database } from "bun:sqlite";
8 import { cell, maskPlan, MASK } from "./mask.ts";
9 import type { Source } from "./sources.ts";
10
11 export const PAGE_SIZE = 50;
12 /** Deepest page served: a huge OFFSET is a slow scan. */
13 export const MAX_PAGE = 1000;
14
15 export interface TableInfo {
16 readonly name: string;
17 readonly rows: number;
18 }
19
20 export interface RowFilter {
21 /** Rows where any column (or just `column`) contains `text`. */
22 readonly text: string;
23 readonly column?: string;
24 }
25
26 export interface RowsOptions {
27 readonly filter?: RowFilter;
28 readonly order?: { readonly column: string; readonly dir?: "asc" | "desc" };
29 /** Zero-based. */
30 readonly page?: number;
31 }
32
33 export interface RowsResult {
34 readonly columns: readonly string[];
35 readonly rows: readonly (readonly (string | number | null)[])[];
36 readonly page: number;
37 readonly total: number;
38 }
39
40 export function openReadOnly(path: string): Database {
41 const db = new Database(path, { readonly: true });
42 db.exec("PRAGMA busy_timeout = 250");
43 db.exec("PRAGMA query_only = 1");
44 return db;
45 }
46
47 const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
48
49 function withDb<T>(source: Source, fn: (db: Database) => T): T {
50 const db = openReadOnly(source.path);
51 try {
52 return fn(db);
53 } finally {
54 db.close();
55 }
56 }
57
58 const tableNames = (db: Database): string[] =>
59 db
60 .query<{ name: string }, []>(
61 "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
62 )
63 .all()
64 .map((r) => r.name);
65
66 export const tables = (source: Source): TableInfo[] =>
67 withDb(source, (db) =>
68 tableNames(db).map((name) => ({
69 name,
70 rows:
71 db
72 .query<{ n: number }, []>(`SELECT count(*) AS n FROM ${quote(name)}`)
73 .get()?.n ?? 0,
74 })),
75 );
76
77 /** Rows of `table`, or `undefined` when the table does not exist. */
78 export function rows(
79 source: Source,
80 table: string,
81 opts: RowsOptions = {},
82 ): RowsResult | undefined {
83 return withDb(source, (db) => {
84 if (!tableNames(db).includes(table)) return undefined;
85 const columns = db
86 .query<{ name: string }, []>(`PRAGMA table_info(${quote(table)})`)
87 .all()
88 .map((c) => c.name);
89 const masked = maskPlan(db, source.id).get(table) ?? new Set<string>();
90 const where: string[] = [];
91 const args: string[] = [];
92 if (opts.filter && opts.filter.text !== "") {
93 const col = opts.filter.column;
94 if (col !== undefined && !columns.includes(col)) return undefined;
95 // Searching a masked column would leak it a character at a time.
96 const searchable = (col === undefined ? columns : [col]).filter(
97 (c) => !masked.has(c),
98 );
99 const like = opts.filter.text.replace(/[\\%_]/g, (m) => `\\${m}`);
100 where.push(
101 searchable.length === 0
102 ? "0"
103 : searchable
104 .map((c) => `CAST(${quote(c)} AS TEXT) LIKE ? ESCAPE '\\'`)
105 .join(" OR "),
106 );
107 for (let i = 0; i < searchable.length; i++) args.push(`%${like}%`);
108 }
109 let order = "";
110 if (opts.order) {
111 if (!columns.includes(opts.order.column)) return undefined;
112 if (masked.has(opts.order.column)) return undefined;
113 order = ` ORDER BY ${quote(opts.order.column)} ${opts.order.dir === "desc" ? "DESC" : "ASC"}`;
114 }
115 const clause = where.length ? ` WHERE ${where.join(" AND ")}` : "";
116 const wanted = Number(opts.page ?? 0);
117 const page = Number.isFinite(wanted)
118 ? Math.min(MAX_PAGE, Math.max(0, Math.floor(wanted)))
119 : 0;
120 const total =
121 db
122 .query<{ n: number }, string[]>(
123 `SELECT count(*) AS n FROM ${quote(table)}${clause}`,
124 )
125 .get(...args)?.n ?? 0;
126 const raw = db
127 .query<unknown[], string[]>(
128 `SELECT ${columns.map(quote).join(", ")} FROM ${quote(table)}${clause}${order} LIMIT ${PAGE_SIZE} OFFSET ${page * PAGE_SIZE}`,
129 )
130 .values(...args);
131 return {
132 columns,
133 page,
134 total,
135 rows: raw.map((r) =>
136 r.map((v, i) =>
137 masked.has(columns[i]!) && v !== null ? MASK : cell(v),
138 ),
139 ),
140 };
141 });
142 }
src/data/mask.ts added +100
@@ -0,0 +1,100 @@
1 import type { Database } from "bun:sqlite";
2
3 /**
4 * Which columns hold secrets, by rule. A masked cell shows that a value is
5 * there and nothing of it. A source may allow a column by name
6 * (`table.column`) when its content is known to be harmless.
7 */
8
9 export const MASK = "••••••";
10
11 const SECRET =
12 /_enc$|verifier|nonce|challenge|(^|_)code(_hash)?$|token|secret|passw(or)?d|passhash|(^|_)hash$|^hash$|sealed|api_?key|(^|_)key$|^key$|private|credential|cookie|(^|_)salt$|(^|_)sig(nature)?$|^pw/i;
13
14 /** Per-source allowlist: source id → `table.column` entries shown in clear. */
15 export const ALLOW: Record<string, readonly string[]> = {};
16
17 /**
18 * Per-source columns masked whatever their name: generic ones that hold
19 * secrets in some rows (a setting's `value`, a stored response `body`).
20 */
21 export const FORCE: Readonly<Record<string, readonly string[]>> = {
22 forge: [
23 "forge_setting.value",
24 "forge_setting_change.from_value",
25 "forge_setting_change.to_value",
26 "front_setting.value",
27 "user_setting.value",
28 "origin_cursor.value",
29 "sentinel_kept.value",
30 "idempotency.body",
31 "event.payload",
32 "sentinel_finding.evidence",
33 ],
34 };
35
36 export const isSecretColumn = (column: string): boolean => SECRET.test(column);
37
38 /** Whether `table.column` of a source shows only the mask. */
39 export function masker(sourceId: string): (table: string, column: string) => boolean {
40 const allow = new Set(ALLOW[sourceId] ?? []);
41 const force = new Set(FORCE[sourceId] ?? []);
42 return (table, column) => {
43 const key = `${table}.${column}`;
44 if (allow.has(key)) return false;
45 return force.has(key) || SECRET.test(column);
46 };
47 }
48
49 const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
50
51 /**
52 * The masked columns of every table and view in `db`. A view is judged by
53 * its text: if it mentions any masked column of the database, all of its
54 * columns are masked, so an alias cannot launder a secret.
55 */
56 export function maskPlan(db: Database, sourceId: string): Map<string, Set<string>> {
57 const masked = masker(sourceId);
58 const plan = new Map<string, Set<string>>();
59 const objects = db
60 .query<{ name: string; type: string; sql: string | null }, []>(
61 "SELECT name, type, sql FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'",
62 )
63 .all();
64 const columnsOf = (name: string): string[] =>
65 db
66 .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
67 .all()
68 .map((c) => c.name);
69 const secretNames = new Set<string>();
70 const cols = new Map<string, string[]>();
71 for (const o of objects) {
72 const c = columnsOf(o.name);
73 cols.set(o.name, c);
74 if (o.type === "table")
75 plan.set(o.name, new Set(c.filter((x) => masked(o.name, x))));
76 for (const x of plan.get(o.name) ?? []) secretNames.add(x.toLowerCase());
77 }
78 for (const o of objects) {
79 if (o.type !== "view") continue;
80 const text = (o.sql ?? "").toLowerCase();
81 const leaks = [...secretNames].some((n) =>
82 new RegExp(`(^|[^a-z0-9_])${n.replace(/[^a-z0-9_]/g, "")}($|[^a-z0-9_])`).test(text),
83 );
84 const c = cols.get(o.name)!;
85 plan.set(
86 o.name,
87 new Set(leaks ? c : c.filter((x) => masked(o.name, x))),
88 );
89 }
90 return plan;
91 }
92
93 /** A cell as plain JSON-safe data. */
94 export function cell(value: unknown): string | number | null {
95 if (value === null || value === undefined) return null;
96 if (typeof value === "bigint") return value.toString();
97 if (typeof value === "number" || typeof value === "string") return value;
98 if (value instanceof Uint8Array) return `<blob ${value.byteLength} bytes>`;
99 return String(value);
100 }
src/data/query.ts added +74
@@ -0,0 +1,74 @@
1 /**
2 * Read-only SQL against one source: `SELECT` or `WITH`, a read-only
3 * connection, a time cap (the worker is ended, which stops even a query
4 * that never yields) and a row cap.
5 */
6
7 import { checkQuery } from "./sql.ts";
8 import type { Source } from "./sources.ts";
9
10 export const QUERY_TIMEOUT_MS = 5000;
11 export const QUERY_ROW_CAP = 1000;
12
13 export type QueryResult =
14 | {
15 readonly ok: true;
16 readonly columns: readonly string[];
17 readonly rows: readonly (readonly (string | number | null)[])[];
18 readonly truncated: boolean;
19 }
20 | {
21 readonly ok: false;
22 readonly error: "empty" | "not_select" | "multiple" | "schema" | "timeout" | "sql";
23 readonly message: string;
24 };
25
26 const REFUSALS = {
27 empty: "Write a query.",
28 not_select: "Only SELECT and WITH statements are allowed.",
29 multiple: "One statement at a time.",
30 schema: "Name tables without a schema prefix.",
31 } as const;
32
33 export function query(
34 source: Source,
35 sql: string,
36 timeoutMs = QUERY_TIMEOUT_MS,
37 ): Promise<QueryResult> {
38 const verdict = checkQuery(sql);
39 if (verdict !== "ok")
40 return Promise.resolve({ ok: false, error: verdict, message: REFUSALS[verdict] });
41 return new Promise((resolve) => {
42 const worker = new Worker(new URL("./query_worker.ts", import.meta.url).href);
43 const done = (r: QueryResult) => {
44 clearTimeout(timer);
45 worker.terminate();
46 resolve(r);
47 };
48 const timer = setTimeout(
49 () =>
50 done({
51 ok: false,
52 error: "timeout",
53 message: `Stopped after ${timeoutMs / 1000} s.`,
54 }),
55 timeoutMs,
56 );
57 worker.onmessage = (e: MessageEvent) => {
58 const m = e.data;
59 done(
60 m.ok
61 ? { ok: true, columns: m.columns, rows: m.rows, truncated: m.truncated }
62 : { ok: false, error: "sql", message: String(m.message) },
63 );
64 };
65 worker.onerror = (e) =>
66 done({ ok: false, error: "sql", message: e.message || "query failed" });
67 worker.postMessage({
68 path: source.path,
69 sourceId: source.id,
70 sql,
71 limit: QUERY_ROW_CAP,
72 });
73 });
74 }
src/data/query_worker.ts added +62
@@ -0,0 +1,62 @@
1 /**
2 * Runs one query off the main thread so that a slow one can be stopped by
3 * ending the thread. Secret columns are shadowed by temporary views that
4 * show the mask, so an alias or a subquery cannot read them.
5 */
6
7 import { Database } from "bun:sqlite";
8 import { cell, maskPlan, MASK } from "./mask.ts";
9
10 declare const self: Worker;
11
12 interface Job {
13 readonly path: string;
14 readonly sourceId: string;
15 readonly sql: string;
16 readonly limit: number;
17 }
18
19 const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
20
21 self.onmessage = (event: MessageEvent<Job>) => {
22 const job = event.data;
23 let db: Database | undefined;
24 try {
25 db = new Database(job.path, { readonly: true });
26 db.exec("PRAGMA busy_timeout = 250");
27 for (const [name, hide] of maskPlan(db, job.sourceId)) {
28 if (hide.size === 0) continue;
29 const cols = db
30 .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
31 .all();
32 const select = cols
33 .map((c) =>
34 hide.has(c.name)
35 ? `CASE WHEN main.${quote(name)}.${quote(c.name)} IS NULL THEN NULL ELSE '${MASK}' END AS ${quote(c.name)}`
36 : `main.${quote(name)}.${quote(c.name)} AS ${quote(c.name)}`,
37 )
38 .join(", ");
39 db.exec(`CREATE TEMP VIEW ${quote(name)} AS SELECT ${select} FROM main.${quote(name)}`);
40 }
41 db.exec("PRAGMA query_only = 1");
42 const stmt = db.query(job.sql);
43 const columns = stmt.columnNames;
44 const out: (string | number | null)[][] = [];
45 let truncated = false;
46 for (const row of stmt.values() as unknown[][]) {
47 if (out.length >= job.limit) {
48 truncated = true;
49 break;
50 }
51 out.push(row.map(cell));
52 }
53 self.postMessage({ ok: true, columns, rows: out, truncated });
54 } catch (error) {
55 self.postMessage({
56 ok: false,
57 message: error instanceof Error ? error.message : String(error),
58 });
59 } finally {
60 db?.close();
61 }
62 };
src/data/sources.ts added +81
@@ -0,0 +1,81 @@
1 /**
2 * The SQLite files a viewer may see: the forge's own (admins), and each
3 * app's per environment (people who can write the app's repository). An id
4 * is only ever resolved by looking it up in this list, so no input becomes
5 * a path.
6 */
7
8 import type { Database } from "bun:sqlite";
9 import { existsSync, readdirSync } from "node:fs";
10 import { join } from "node:path";
11 import { listApps } from "../core/apps.ts";
12 import { getRepoById, writableRepoIds } from "../core/repos.ts";
13 import type { User } from "../core/users.ts";
14 import { envDataDir, isEnvName } from "../apps/data.ts";
15
16 export interface Source {
17 /** `forge`, or `app:<appId>:<env>:<file>`. */
18 readonly id: string;
19 readonly label: string;
20 readonly kind: "forge" | "app";
21 readonly path: string;
22 }
23
24 const DB_FILE = /^[\w.-]+\.db$/;
25
26 export function listSources(
27 db: Database,
28 dataDir: string,
29 viewer: User | undefined,
30 ): Source[] {
31 const out: Source[] = [];
32 if (viewer === undefined) return out;
33 if (viewer.isAdmin) {
34 const path = join(dataDir, "forge.db");
35 if (existsSync(path))
36 out.push({ id: "forge", label: "forge", kind: "forge", path });
37 }
38 const apps = listApps(db);
39 const writable = writableRepoIds(
40 db,
41 viewer,
42 apps.map((a) => a.repoId),
43 );
44 for (const app of apps) {
45 if (!writable.has(app.repoId) || getRepoById(db, app.repoId) === undefined)
46 continue;
47 let envs: string[];
48 try {
49 envs = readdirSync(join(dataDir, "apps", app.id)).filter(isEnvName);
50 } catch {
51 continue;
52 }
53 for (const env of envs.sort()) {
54 if (!isEnvName(env)) continue;
55 const dir = envDataDir(dataDir, app.id, env);
56 let files: string[];
57 try {
58 files = readdirSync(dir).filter((f) => DB_FILE.test(f));
59 } catch {
60 continue;
61 }
62 for (const file of files.sort())
63 out.push({
64 id: `app:${app.id}:${env}:${file}`,
65 label: `${app.slug} · ${env} · ${file}`,
66 kind: "app",
67 path: join(dir, file),
68 });
69 }
70 }
71 return out;
72 }
73
74 /** The source `id` names, if this viewer may see it; else `undefined` (a 404). */
75 export const findSource = (
76 db: Database,
77 dataDir: string,
78 viewer: User | undefined,
79 id: string,
80 ): Source | undefined =>
81 listSources(db, dataDir, viewer).find((s) => s.id === id);
src/data/sql.ts added +32
@@ -0,0 +1,32 @@
1 /** What a read-only query may be: one SELECT or WITH statement, nothing else. */
2
3 /** The statement with comments and string literals blanked; identifiers stay. */
4 function skeleton(sql: string): string {
5 let out = "";
6 for (let i = 0; i < sql.length; i++) {
7 const c = sql[i]!;
8 if (c === "-" && sql[i + 1] === "-") {
9 while (i < sql.length && sql[i] !== "\n") i++;
10 out += " ";
11 } else if (c === "/" && sql[i + 1] === "*") {
12 const end = sql.indexOf("*/", i + 2);
13 i = end < 0 ? sql.length : end + 1;
14 out += " ";
15 } else if (c === "'") {
16 i++;
17 while (i < sql.length && sql[i] !== "'") i++;
18 out += "''";
19 } else out += c;
20 }
21 return out;
22 }
23
24 export function checkQuery(sql: string): "ok" | "empty" | "not_select" | "multiple" | "schema" {
25 const s = skeleton(sql).trim().replace(/;+\s*$/, "").trim();
26 if (s === "") return "empty";
27 if (s.includes(";")) return "multiple";
28 if (!/^(select|with)\b/i.test(s)) return "not_select";
29 // `main.t` would step round the masking views that shadow `t`.
30 if (/(^|[^\w])("main"|\[main\]|`main`|main)\s*\./i.test(s)) return "schema";
31 return "ok";
32 }
src/http/mcp/tools/data.ts added +39
@@ -0,0 +1,39 @@
1 import { rows, tables } from "../../../data/browse.ts";
2 import { query } from "../../../data/query.ts";
3 import { findSource, listSources } from "../../../data/sources.ts";
4 import { bad, good, type Tool } from "../shared.ts";
5
6 export const tool: Tool = {
7 name: "data",
8 doc: "the SQLite files you may read (the forge's own for an admin; each app's per environment for its writers), with `source`: its tables and row counts, with `table`: a page of rows, with `sql`: a read-only SELECT (5 s, 1,000 rows); secret columns are masked",
9 description: "Read SQLite files you may see: tables, rows, read-only SQL; secrets masked.",
10 inputSchema: {
11 type: "object",
12 properties: {
13 source: { type: "string" },
14 table: { type: "string" },
15 page: { type: "integer" },
16 sql: { type: "string" },
17 },
18 },
19 writes: false,
20 async run(ctx, user, args) {
21 const { db, config } = ctx.app;
22 const { source: id, table, page, sql } = args;
23 for (const [k, v] of Object.entries({ source: id, table, sql }))
24 if (v !== undefined && typeof v !== "string") return bad("invalid", `\`${k}\` is text`);
25 if (page !== undefined && typeof page !== "number") return bad("invalid", "`page` is a number");
26 if (id === undefined) {
27 return good(listSources(db, config.dataDir, user).map((s) => ({ source: s.id, label: s.label })));
28 }
29 const source = findSource(db, config.dataDir, user, id as string);
30 if (!source) return bad("not_found", "no such source");
31 if (sql !== undefined) {
32 const result = await query(source, sql as string);
33 return result.ok ? good(result) : bad("invalid", result.message);
34 }
35 if (table === undefined) return good(tables(source));
36 const found = rows(source, table as string, page === undefined ? {} : { page: page as number });
37 return found ? good(found) : bad("not_found", "no such table");
38 },
39 };
src/http/mcp/tools/index.ts +1
@@ -44,6 +44,7 @@ export { tool as confirm_wait } from "./confirm_wait.ts";
4444 export { tool as contribute } from "./contribute.ts";
4545 export { tool as converge } from "./converge.ts";
4646 export { tool as crew } from "./crew.ts";
47 export { tool as data } from "./data.ts";
4748 export { tool as decisions_by } from "./decisions_by.ts";
4849 export { tool as detach } from "./detach.ts";
4950 export { tool as drills } from "./drills.ts";
src/http/server.ts +2
@@ -20,6 +20,7 @@ import { systemPage, systemStream } from "../web/pages/system.ts";
2020 import { contributePage } from "../web/pages/contribute.ts";
2121 import { pipelinePage } from "../web/pages/pipeline.ts";
2222 import { resiliencePage } from "../web/pages/resilience.ts";
23 import { dataPage } from "../web/pages/data.ts";
2324 import { limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts";
2425 import { measureSubmit } from "./measure.ts";
2526 import { settingsRoutes } from "./settings_routes.ts";
@@ -813,6 +814,7 @@ export function buildRouter(): Router<Handler> {
813814 pinnedRoutes(router);
814815 // Each THREATS.md protection against the last gate; literal, above /:owner.
815816 router.add("GET", "/resilience", resiliencePage);
817 router.add("GET", "/data", dataPage);
816818 // A selection's spend and runs over time; literal, above /:owner.
817819 router.add("GET", "/fleet/graph", fleetGraphPage);
818820 router.add("GET", "/fleet/graph/live", fleetGraphLive);
src/web/pages/apps.ts +5
@@ -12,6 +12,7 @@
1212 * 404 for everyone.
1313 */
1414
15 import { listSources } from "../../data/sources.ts";
1516 import { storageLimitMessage } from "../../core/storage.ts";
1617 import { repositoryLimitMessage } from "../../actions/repos.ts";
1718 import { appLimitWords } from "../../actions/app_limit.ts";
@@ -235,8 +236,12 @@ export async function appsList(ctx: Ctx): Promise<Response> {
235236 costLine(ctx.app.db, writableAppsOf(ctx, ctx.auth.user).repos),
236237 );
237238 const rendering = performance.now();
239 const hasData =
240 ctx.auth.user !== undefined &&
241 listSources(ctx.app.db, ctx.app.config.dataDir, ctx.auth.user).length > 0;
238242 const body = html`${when(ctx.auth.user, (user) => homeTabs("apps", unshippedAsks(ctx.app.db, user)))}
239243 ${pageHead("Apps", { actions: create })}
244 ${hasData ? html`<p class="muted small mb"><a href="/data">Data</a></p>` : null}
240245 ${cost}
241246 ${filters}
242247 ${archivedView
src/web/pages/data.ts added +87
@@ -0,0 +1,87 @@
1 /**
2 * /data: the SQLite files the viewer may see (`data/sources.ts`), the
3 * tables in one, and a page of a table's rows, secrets masked. Anyone who
4 * may see no source gets the not-found page, as a guest does.
5 */
6
7 import { PAGE_SIZE, rows, tables } from "../../data/browse.ts";
8 import { findSource, listSources } from "../../data/sources.ts";
9 import type { Ctx } from "../../http/context.ts";
10 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
11 import { notFoundPage, signInFirst } from "../chrome.ts";
12 import { html } from "../html.ts";
13 import { page } from "../layout.ts";
14 import { card } from "../ui.ts";
15
16 const href = (params: Record<string, string | number>): string =>
17 `/data?${new URLSearchParams(Object.entries(params).map(([k, v]): [string, string] => [k, String(v)]))}`;
18
19 // GET /data[?source=<id>[&table=<name>[&page=<n>]]]
20 export function dataPage(ctx: Ctx): Response {
21 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
22 const viewer = ctx.auth.user;
23 if (!viewer) return signInFirst(ctx);
24 const { db, config } = ctx.app;
25 const q = ctx.url.searchParams;
26 const all = listSources(db, config.dataDir, viewer);
27 if (all.length === 0) return notFoundPage(ctx);
28 const chosen = q.get("source");
29 const source = chosen === null ? undefined : findSource(db, config.dataDir, viewer, chosen);
30 if (chosen !== null && !source) return notFoundPage(ctx);
31
32 let body = html`<div class="page-head"><h1>Data</h1></div>
33 ${card(
34 html`<ul class="list">
35 ${all.map(
36 (s) =>
37 html`<li class="row">
38 <span class="row-main"><a class="row-title" href="${href({ source: s.id })}">${s.label}</a></span>
39 </li>`,
40 )}
41 </ul>`,
42 { pad: false },
43 )}`;
44
45 if (source) {
46 const list = tables(source);
47 const name = q.get("table");
48 const shown = name === null ? undefined : rows(source, name, { page: Number(q.get("page") ?? 0) });
49 if (name !== null && !shown) return notFoundPage(ctx);
50 body = html`${body}
51 <h2>${source.label}</h2>
52 ${card(
53 html`<ul class="list">
54 ${list.map(
55 (t) =>
56 html`<li class="row">
57 <span class="row-main"><a class="row-title" href="${href({ source: source.id, table: t.name })}">${t.name}</a></span>
58 <span class="row-trail">${String(t.rows)} rows</span>
59 </li>`,
60 )}
61 </ul>`,
62 { pad: false },
63 )}`;
64 if (shown && name !== null) {
65 const more = (shown.page + 1) * PAGE_SIZE < shown.total;
66 body = html`${body}
67 <h2>${name}</h2>
68 ${card(
69 html`<table>
70 <thead><tr>${shown.columns.map((c) => html`<th>${c}</th>`)}</tr></thead>
71 <tbody>
72 ${shown.rows.map(
73 (r) => html`<tr>${r.map((v) => html`<td>${v === null ? "" : String(v)}</td>`)}</tr>`,
74 )}
75 </tbody>
76 </table>`,
77 )}
78 <p class="muted">
79 ${String(shown.total)} rows${shown.page > 0 ? html` · <a href="${href({ source: source.id, table: name, page: shown.page - 1 })}">Newer page</a>` : ""}${more ? html` · <a href="${href({ source: source.id, table: name, page: shown.page + 1 })}">Next page</a>` : ""}
80 </p>`;
81 }
82 }
83 const rendering = performance.now();
84 const response = page({ ctx, title: "Data", body });
85 notePhase(ctx.request, "render", performance.now() - rendering);
86 return response;
87 }
test/data-browser.test.ts added +293
@@ -0,0 +1,293 @@
1 /**
2 * The data browser's core: who sees which SQLite file, what a query may be,
3 * and what is masked.
4 */
5 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
6 import { Database } from "bun:sqlite";
7 import { mkdirSync } from "node:fs";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp as createAppRecord } from "../src/core/apps.ts";
12 import { createRepo } from "../src/core/repos.ts";
13 import { createApp, type App } from "../src/app.ts";
14 import { envDataDir } from "../src/apps/data.ts";
15 import { createUser, type User } from "../src/core/users.ts";
16 import { rows, tables } from "../src/data/browse.ts";
17 import { ALLOW, maskPlan } from "../src/data/mask.ts";
18 import { query } from "../src/data/query.ts";
19 import { findSource, listSources } from "../src/data/sources.ts";
20 import { quietLogs } from "./helpers/globals.ts";
21
22 quietLogs();
23
24 const AUDITED_CLEAR: string[] = [
25 "app_domain.said",
26 "comment.body",
27 "degraded_span.sentence",
28 "feed_chip.value",
29 "fleet_operation.value",
30 "issue.body",
31 "oauth_flow.detail",
32 "oidc_key.public_jwk",
33 "probe.said",
34 "prompt_source.last_said",
35 "repo_upstream_check.detail",
36 "report.body",
37 "review.body",
38 "run.check_detail",
39 "subscription.resting_said",
40 "subscription_limit.said",
41 "user_key.blob",
42 "webhook_ping.payload",
43 "week_projection.body",
44 "week_retro.body",
45 ];
46
47 describe("data browser core", () => {
48 let dataDir: string;
49 let app: App;
50 let admin: User;
51 let alice: User;
52 let bob: User;
53 let appId = "";
54
55 const user = async (username: string, adminIfFirst = false) => {
56 const made = await createUser(app.db, {
57 origin: "seed",
58 trusted: true,
59 username,
60 email: `${username}@forge.local`,
61 password: "a-good-passphrase",
62 adminIfFirst,
63 });
64 if (!made.ok) throw new Error(made.error);
65 return made.value;
66 };
67 const ids = (viewer: User | undefined) =>
68 listSources(app.db, dataDir, viewer).map((s) => s.id);
69
70 beforeAll(async () => {
71 dataDir = await mkdtemp(join(tmpdir(), "forge-data-browser-"));
72 app = createApp({ dataDir, port: 0, host: "forge.test" });
73 admin = await user("owner1", true);
74 alice = await user("alice");
75 bob = await user("bob");
76 const repo = await createRepo(app.db, app.reposRoot, {
77 ownerId: alice.id,
78 name: "notes",
79 visibility: "public",
80 });
81 if (!repo.ok) throw new Error(repo.error);
82 const made = createAppRecord(app.db, {
83 repoId: repo.value.id,
84 slug: "notes",
85 access: "public",
86 });
87 if (!made.ok) throw new Error(made.error);
88 appId = made.value.id;
89 for (const env of ["production", "pull-3"] as const) {
90 mkdirSync(envDataDir(dataDir, appId, env), { recursive: true });
91 const d = new Database(join(envDataDir(dataDir, appId, env), "app.db"));
92 d.exec("PRAGMA journal_mode = WAL");
93 d.exec(
94 "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT, password_hash TEXT)",
95 );
96 d.exec("CREATE VIEW people AS SELECT name, api_token AS harmless FROM users");
97 d.exec(
98 `INSERT INTO users VALUES (1, 'ann', 'tok-ann-1', 'hash-1'), (2, 'ben', 'tok-ben-2', NULL)`,
99 );
100 d.close();
101 }
102 });
103
104 afterAll(async () => {
105 app.close();
106 await rm(dataDir, { recursive: true, force: true });
107 });
108
109 test("a writer sees their app's production and preview databases", () => {
110 expect(ids(alice)).toEqual([
111 `app:${appId}:production:app.db`,
112 `app:${appId}:pull-3:app.db`,
113 ]);
114 });
115
116 test("a reader of the app can't find its data", () => {
117 expect(ids(bob)).toEqual([]);
118 expect(ids(undefined)).toEqual([]);
119 expect(
120 findSource(app.db, dataDir, bob, `app:${appId}:production:app.db`),
121 ).toBeUndefined();
122 });
123
124 test("only an admin sees forge.db", () => {
125 expect(ids(admin)).toContain("forge");
126 expect(ids(alice)).not.toContain("forge");
127 expect(findSource(app.db, dataDir, alice, "forge")).toBeUndefined();
128 });
129
130 test("tables with counts, rows with filter, order and page", () => {
131 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
132 expect(tables(src)).toEqual([{ name: "users", rows: 2 }]);
133 const r = rows(src, "users", { filter: { text: "b" }, order: { column: "id", dir: "desc" } })!;
134 expect(r.total).toBe(1);
135 expect(r.rows[0]![1]).toBe("ben");
136 expect(rows(src, "nope")).toBeUndefined();
137 expect(rows(src, "users", { order: { column: "id; DROP TABLE users" } })).toBeUndefined();
138 });
139
140 test("a token column is masked, and cannot be searched or ordered", () => {
141 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
142 const r = rows(src, "users")!;
143 expect(JSON.stringify(r)).not.toContain("tok-ann");
144 expect(JSON.stringify(r)).not.toContain("hash-1");
145 expect(r.rows[1]![3]).toBeNull();
146 expect(rows(src, "users", { filter: { text: "tok-ann", column: "api_token" } })!.total).toBe(0);
147 expect(rows(src, "users", { order: { column: "api_token" } })).toBeUndefined();
148 });
149
150 test("a query cannot read a secret by alias or subquery either", async () => {
151 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
152 for (const sql of [
153 "SELECT api_token AS t FROM users",
154 "SELECT (SELECT api_token FROM users LIMIT 1) AS t",
155 "SELECT * FROM users",
156 ]) {
157 const r = await query(src, sql);
158 expect(r.ok).toBe(true);
159 expect(JSON.stringify(r)).not.toContain("tok-ann");
160 }
161 const bypass = await query(src, "SELECT api_token FROM main.users");
162 expect(bypass.ok).toBe(false);
163 const plain = await query(src, "SELECT name FROM users ORDER BY id");
164 expect(plain.ok && plain.rows).toEqual([["ann"], ["ben"]]);
165 });
166
167 test("INSERT and PRAGMA writable_schema are refused", async () => {
168 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
169 for (const sql of [
170 "INSERT INTO users (name) VALUES ('x')",
171 "PRAGMA writable_schema = 1",
172 "SELECT 1; DELETE FROM users",
173 "WITH x AS (SELECT 1) DELETE FROM users",
174 "-- hi\nUPDATE users SET name = 'x'",
175 ]) {
176 const r = await query(src, sql);
177 expect(r.ok).toBe(false);
178 }
179 expect(rows(src, "users")!.total).toBe(2);
180 });
181
182 test("a result stops at 1,000 rows", async () => {
183 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
184 const r = await query(
185 src,
186 "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 5000) SELECT i FROM n",
187 );
188 expect(r.ok && r.rows.length).toBe(1000);
189 expect(r.ok && r.truncated).toBe(true);
190 });
191
192 test("a slow query stops at the cap", async () => {
193 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
194 const started = performance.now();
195 const r = await query(
196 src,
197 "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n) SELECT count(*) FROM n",
198 400,
199 );
200 expect(r.ok).toBe(false);
201 expect(!r.ok && r.error).toBe("timeout");
202 expect(performance.now() - started).toBeLessThan(3000);
203 });
204
205 test("reading never blocks the app's own writes", () => {
206 const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
207 const w = new Database(src.path);
208 tables(src);
209 w.exec("INSERT INTO users (name) VALUES ('cy')");
210 w.close();
211 expect(rows(src, "users")!.total).toBe(3);
212 });
213
214 const prod = () => findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
215
216 test("a schema qualifier in any quoting is refused", async () => {
217 for (const from of [
218 '"main"."users"',
219 "[main].[users]",
220 "`main`.`users`",
221 'main /**/ .users',
222 '"MAIN" . users',
223 "main\n.users",
224 ]) {
225 const r = await query(prod(), `SELECT api_token FROM ${from}`);
226 expect(r.ok).toBe(false);
227 expect(JSON.stringify(r)).not.toContain("tok-ann");
228 }
229 });
230
231 test("a view that aliases a secret is masked, in rows and in queries", async () => {
232 const src = prod();
233 expect(JSON.stringify(rows(src, "people") ?? "")).not.toContain("tok-ann");
234 const r = await query(src, "SELECT harmless FROM people");
235 expect(r.ok).toBe(true);
236 expect(JSON.stringify(r)).not.toContain("tok-ann");
237 expect(JSON.stringify(r)).toContain("••••");
238 });
239
240 test("an allowlisted column is shown in clear, and only for its source", () => {
241 const src = prod();
242 ALLOW[src.id] = ["users.api_token"];
243 try {
244 expect(JSON.stringify(rows(src, "users"))).toContain("tok-ann-1");
245 expect(JSON.stringify(rows(src, "users"))).not.toContain("hash-1");
246 const other = findSource(app.db, dataDir, alice, `app:${appId}:pull-3:app.db`)!;
247 expect(JSON.stringify(rows(other, "users"))).not.toContain("tok-ann");
248 } finally {
249 delete ALLOW[src.id];
250 }
251 });
252
253 test("a filter on a column that does not exist finds nothing", () => {
254 expect(rows(prod(), "users", { filter: { text: "a", column: "nope" } })).toBeUndefined();
255 });
256
257 test("a guest or a non-writer cannot find forge", () => {
258 for (const who of [undefined, alice, bob]) expect(findSource(app.db, dataDir, who, "forge")).toBeUndefined();
259 });
260
261 test("a page that is not a finite number is the first page; a huge one is capped", () => {
262 const src = prod();
263 for (const page of [Number.NaN, Number.POSITIVE_INFINITY, -4])
264 expect(rows(src, "users", { page })!.page).toBe(0);
265 expect(rows(src, "users", { page: 1e12 })!.page).toBe(1000);
266 });
267
268 test("forge.db is masked for an admin, setting values included", async () => {
269 app.db.exec("INSERT OR REPLACE INTO forge_setting (key, value, updated_by, updated_at) VALUES ('smtp', 'SETTING-SECRET', 'x', 'now')");
270 const src = findSource(app.db, dataDir, admin, "forge")!;
271 expect(JSON.stringify(rows(src, "forge_setting"))).not.toContain("SETTING-SECRET");
272 const r = await query(src, "SELECT value FROM forge_setting");
273 expect(JSON.stringify(r)).not.toContain("SETTING-SECRET");
274 const users = JSON.stringify(rows(src, "user"));
275 expect(users).not.toContain("$argon2");
276 });
277
278 test("every forge.db column that could hold a secret is masked or audited as clear", () => {
279 const src = findSource(app.db, dataDir, admin, "forge")!;
280 const plan = maskPlan(new Database(src.path, { readonly: true }), "forge");
281 const loose = /value|body|payload|code|key|blob|nonce|verifier|secret|token|hash|salt|enc|jwk|evidence|said|detail/i;
282 const open: string[] = [];
283 for (const [table, masked] of plan) {
284 const db = new Database(src.path, { readonly: true });
285 for (const c of db.query<{ name: string }, []>(`PRAGMA table_info("${table}")`).all())
286 if (loose.test(c.name) && !masked.has(c.name)) open.push(`${table}.${c.name}`);
287 db.close();
288 }
289 // Reviewed: these hold public or operational text, never a credential.
290 // A new column that matches and is not here fails until someone looks.
291 expect(open.sort()).toEqual(AUDITED_CLEAR);
292 });
293 });
test/data-door.test.ts added +100
@@ -0,0 +1,100 @@
1 /** /data and the `data` MCP tool: gated by listSources, so a guest or a reader sees nothing. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { Database } from "bun:sqlite";
4 import { mkdirSync } from "node:fs";
5 import { mkdtemp, rm } from "node:fs/promises";
6 import { tmpdir } from "node:os";
7 import { join } from "node:path";
8 import { createApp, type App } from "../src/app.ts";
9 import { envDataDir } from "../src/apps/data.ts";
10 import { createApp as createAppRecord } from "../src/core/apps.ts";
11 import { createRepo } from "../src/core/repos.ts";
12 import { createSession } from "../src/core/sessions.ts";
13 import { createToken } from "../src/core/tokens.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { serve, type Servers } from "../src/http/server.ts";
16 import { quietLogs } from "./helpers/globals.ts";
17 import { mcpTool } from "./helpers/mcp.ts";
18
19 quietLogs();
20
21 describe("the data door", () => {
22 let dataDir: string;
23 let app: App;
24 let servers: Servers;
25 const cookie: Record<string, string> = {};
26 const token: Record<string, string> = {};
27 let appId = "";
28
29 beforeAll(async () => {
30 dataDir = await mkdtemp(join(tmpdir(), "forge-data-door-"));
31 app = createApp({ dataDir, port: 0, host: "forge.test" });
32 servers = await serve(app);
33 const users: Record<string, string> = {};
34 for (const name of ["owner1", "alice", "bob"]) {
35 const made = await createUser(app.db, {
36 origin: "seed", trusted: true, username: name,
37 email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "owner1",
38 });
39 if (!made.ok) throw new Error(made.error);
40 users[name] = made.value.id;
41 cookie[name] = `forge_session=${createSession(app.db, made.value.id).token}`;
42 token[name] = createToken(app.db, made.value.id, "t", "admin").token;
43 }
44 const repo = await createRepo(app.db, app.reposRoot, { ownerId: users["alice"]!, name: "notes", visibility: "public" });
45 if (!repo.ok) throw new Error(repo.error);
46 const rec = createAppRecord(app.db, { repoId: repo.value.id, slug: "notes", access: "public" });
47 if (!rec.ok) throw new Error(rec.error);
48 appId = rec.value.id;
49 mkdirSync(envDataDir(dataDir, appId, "production"), { recursive: true });
50 const d = new Database(join(envDataDir(dataDir, appId, "production"), "app.db"));
51 d.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT)");
52 d.exec("INSERT INTO users VALUES (1, 'ann', 'tok-ann-1')");
53 d.close();
54 }, 60_000);
55
56 afterAll(async () => {
57 servers.stop();
58 app.close();
59 await rm(dataDir, { recursive: true, force: true });
60 });
61
62 const get = (path: string, who?: string) =>
63 fetch(`${servers.url}${path}`, { headers: who ? { cookie: cookie[who]! } : {}, redirect: "manual" });
64 const src = () => encodeURIComponent(`app:${appId}:production:app.db`);
65
66 test("a writer reaches sources, tables and masked rows on the page", async () => {
67 const list = await (await get("/data", "alice")).text();
68 expect(list).toContain("notes · production · app.db");
69 const tables = await (await get(`/data?source=${src()}`, "alice")).text();
70 expect(tables).toContain("users");
71 const res = await get(`/data?source=${src()}&table=users`, "alice");
72 const rows = await res.text();
73 expect(rows).toContain("ann");
74 expect(rows).not.toContain("tok-ann-1");
75 expect(await (await get("/apps", "alice")).text()).toContain('href="/data"');
76 });
77
78 test("a reader, a guest and a wrong source see nothing", async () => {
79 expect((await get("/data", "bob")).status).toBe(404);
80 expect((await get(`/data?source=${src()}`, "bob")).status).toBe(404);
81 expect((await get("/data")).status).not.toBe(200);
82 expect((await get("/data?source=forge", "alice")).status).toBe(404);
83 expect((await get("/data?source=forge", "owner1")).status).toBe(200);
84 expect(await (await get("/apps", "bob")).text()).not.toContain('href="/data"');
85 });
86
87 test("the MCP tool lists, reads, queries and refuses the same way", async () => {
88 const id = `app:${appId}:production:app.db`;
89 expect((await mcpTool(servers.url, token["alice"]!, "data")).value).toEqual([
90 { source: id, label: "notes · production · app.db" },
91 ]);
92 expect((await mcpTool(servers.url, token["alice"]!, "data", { source: id })).value).toEqual([{ name: "users", rows: 1 }]);
93 const q = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "SELECT api_token FROM users" });
94 expect(JSON.stringify(q)).not.toContain("tok-ann-1");
95 const w = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "DELETE FROM users" });
96 expect(w.error).toBeDefined();
97 expect((await mcpTool(servers.url, token["bob"]!, "data", { source: id })).error).toBeDefined();
98 expect((await mcpTool(servers.url, token["bob"]!, "data")).value).toEqual([]);
99 });
100 });
test/forge-looks.test.ts +1
@@ -21,6 +21,7 @@ quietLogs();
2121 /** Top-level routes that are not pages, or are not worth a picture, and why. */
2222 const NOT_A_PAGE: Record<string, string> = {
2323 "/deploy-to": "a form for a forge admin that fills from its link; covered by test/pinned-page.test.ts",
24 "/data": "no picture: the cap of 24 looks is spent; a guest sees the not-found page and the page is covered by its own tests",
2425 "/asked": "no picture: the cap of 24 looks went to /notes, the page that now leads with a person's thoughts; /asked is covered by test/asked.test.ts",
2526 "/forges": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests",
2627 "/family": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests",
test/mcp.test.ts +1
@@ -210,6 +210,7 @@ describe("the forge over MCP", () => {
210210 const names = listed.body.result.tools.map((t: { name: string }) => t.name);
211211 // Listed by name, one file each (#5924), and `report` last.
212212 expect(names).toEqual(sortedThenReport([
213 "data",
213214 "drills",
214215 "sizes",
215216 "releases",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs