Public: anyone can read this; the crew can change it.
Data browser core: read any SQLite file a viewer may see — the forge's own (admins) and each app's per environment (its writers) — tables, rows, read-only SQL, secrets masked #8923
scout/forge:scout/7579-data-browser-core-read-any-sqlite-file-a → main 8 commits fork
19 files
changed +1036
−0
.gitignore +1
| @@ -1,5 +1,6 @@ | ||
| 1 | 1 | node_modules |
| 2 | 2 | data/ |
| 3 | !src/data/ | |
| 3 | 4 | dist/ |
| 4 | 5 | *.local |
| 5 | 6 | .DS_Store |
docs/design/audit.md +1
| @@ -57,6 +57,7 @@ Machine routes are not pages and have no entry: `/audit.csv`, | ||
| 57 | 57 | - `/feed` — `studio/home.ts` · the composer and one timeline, what home was before it became the living pipeline (#8015) · `/` · none seen. |
| 58 | 58 | - `/boards/data` — `pages/boards.ts` · the boards' query/data surface · `/board` · none seen. |
| 59 | 59 | - `/asks` — `studio/asks_page.ts` · what the crew is waiting on me for · `/` "needs me" list · colours 2. |
| 60 | - `/data` — `pages/data.ts` · the SQLite files I may see, their tables and rows, secrets masked · `/system` · none seen. | |
| 60 | 61 | - `/asked` — `studio/asked_page.ts` · what I answered, as a record · `/asks` · none seen. |
| 61 | 62 | - `/review` — `studio/review_page.ts` · every ask waiting on me, one row each, newest first · `/asks` (same asks, with buttons) · none seen. |
| 62 | 63 | - `/flow` — `pages/flow.ts` · the pipeline from ready issue to live deploy, and the bottleneck · `/system`, `/board`, `/` · none seen. A topbar button points here. |
docs/features/data-browser.md added +13
| @@ -0,0 +1,13 @@ | ||
| 1 | # Data browser (core) | |
| 2 | ||
| 3 | `src/data/` reads SQLite files a viewer may see. Reach it at `/data` (linked from Apps for anyone who can see a source) or through the MCP tool `data`; a guest or a reader of an app sees a 404 / nothing. | |
| 4 | ||
| 5 | - `listSources(db, dataDir, viewer)` — `forge.db` for admins; for each app the viewer can write, every `.db` under each environment's data dir. `findSource` resolves an id only through that list, so a reader gets `undefined` (a 404), and no input becomes a path. | |
| 6 | - `tables(source)`, `rows(source, table, {filter, order, page})` — 50 rows a page; table and column names are checked against the schema. | |
| 7 | - `query(source, sql, timeoutMs?)` — one `SELECT`/`WITH`, read-only connection (`query_only`), 1,000 rows, 5 s. It runs in a worker that is terminated at the cap, so even a query that never yields stops. | |
| 8 | - Masking (`mask.ts`): columns named like token, secret, password, hash, sealed, key, cookie, … show `••••••`. Queries read through temporary views that shadow each table, so aliases and subqueries are masked too; `main.` qualifiers are refused. Masked columns cannot be searched or ordered. `ALLOW` lists `table.column` exceptions per source id. | |
| 9 | - Connections open per call and close after; they never take a writer lock. | |
| 10 | ||
| 11 | Views are masked whole when their text mentions a masked column (an alias cannot launder a secret). `FORCE` masks generic columns by source (`forge_setting.value`, …); `test/data-browser.test.ts` audits every `forge.db` column against the name rule and lists the ones reviewed as clear. | |
| 12 | ||
| 13 | Not done: a SQL editor on the page, a terminal screen. |
src/core/reserved.ts +1
| @@ -86,6 +86,7 @@ export const SYSTEM_NAMES: readonly string[] = [ | ||
| 86 | 86 | "mcp", |
| 87 | 87 | "metrics", |
| 88 | 88 | "contract", |
| 89 | "data", | |
| 89 | 90 | "migrate", |
| 90 | 91 | "mode", |
| 91 | 92 | "new", |
src/data/browse.ts added +142
| @@ -0,0 +1,142 @@ | ||
| 1 | /** | |
| 2 | * Tables and rows of one source, read on a connection opened for the call | |
| 3 | * and closed after it. Read-only: it takes no writer lock and a busy WAL | |
| 4 | * answers in milliseconds or not at all. | |
| 5 | */ | |
| 6 | ||
| 7 | import { Database } from "bun:sqlite"; | |
| 8 | import { cell, maskPlan, MASK } from "./mask.ts"; | |
| 9 | import type { Source } from "./sources.ts"; | |
| 10 | ||
| 11 | export const PAGE_SIZE = 50; | |
| 12 | /** Deepest page served: a huge OFFSET is a slow scan. */ | |
| 13 | export const MAX_PAGE = 1000; | |
| 14 | ||
| 15 | export interface TableInfo { | |
| 16 | readonly name: string; | |
| 17 | readonly rows: number; | |
| 18 | } | |
| 19 | ||
| 20 | export interface RowFilter { | |
| 21 | /** Rows where any column (or just `column`) contains `text`. */ | |
| 22 | readonly text: string; | |
| 23 | readonly column?: string; | |
| 24 | } | |
| 25 | ||
| 26 | export interface RowsOptions { | |
| 27 | readonly filter?: RowFilter; | |
| 28 | readonly order?: { readonly column: string; readonly dir?: "asc" | "desc" }; | |
| 29 | /** Zero-based. */ | |
| 30 | readonly page?: number; | |
| 31 | } | |
| 32 | ||
| 33 | export interface RowsResult { | |
| 34 | readonly columns: readonly string[]; | |
| 35 | readonly rows: readonly (readonly (string | number | null)[])[]; | |
| 36 | readonly page: number; | |
| 37 | readonly total: number; | |
| 38 | } | |
| 39 | ||
| 40 | export function openReadOnly(path: string): Database { | |
| 41 | const db = new Database(path, { readonly: true }); | |
| 42 | db.exec("PRAGMA busy_timeout = 250"); | |
| 43 | db.exec("PRAGMA query_only = 1"); | |
| 44 | return db; | |
| 45 | } | |
| 46 | ||
| 47 | const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`; | |
| 48 | ||
| 49 | function withDb<T>(source: Source, fn: (db: Database) => T): T { | |
| 50 | const db = openReadOnly(source.path); | |
| 51 | try { | |
| 52 | return fn(db); | |
| 53 | } finally { | |
| 54 | db.close(); | |
| 55 | } | |
| 56 | } | |
| 57 | ||
| 58 | const tableNames = (db: Database): string[] => | |
| 59 | db | |
| 60 | .query<{ name: string }, []>( | |
| 61 | "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", | |
| 62 | ) | |
| 63 | .all() | |
| 64 | .map((r) => r.name); | |
| 65 | ||
| 66 | export const tables = (source: Source): TableInfo[] => | |
| 67 | withDb(source, (db) => | |
| 68 | tableNames(db).map((name) => ({ | |
| 69 | name, | |
| 70 | rows: | |
| 71 | db | |
| 72 | .query<{ n: number }, []>(`SELECT count(*) AS n FROM ${quote(name)}`) | |
| 73 | .get()?.n ?? 0, | |
| 74 | })), | |
| 75 | ); | |
| 76 | ||
| 77 | /** Rows of `table`, or `undefined` when the table does not exist. */ | |
| 78 | export function rows( | |
| 79 | source: Source, | |
| 80 | table: string, | |
| 81 | opts: RowsOptions = {}, | |
| 82 | ): RowsResult | undefined { | |
| 83 | return withDb(source, (db) => { | |
| 84 | if (!tableNames(db).includes(table)) return undefined; | |
| 85 | const columns = db | |
| 86 | .query<{ name: string }, []>(`PRAGMA table_info(${quote(table)})`) | |
| 87 | .all() | |
| 88 | .map((c) => c.name); | |
| 89 | const masked = maskPlan(db, source.id).get(table) ?? new Set<string>(); | |
| 90 | const where: string[] = []; | |
| 91 | const args: string[] = []; | |
| 92 | if (opts.filter && opts.filter.text !== "") { | |
| 93 | const col = opts.filter.column; | |
| 94 | if (col !== undefined && !columns.includes(col)) return undefined; | |
| 95 | // Searching a masked column would leak it a character at a time. | |
| 96 | const searchable = (col === undefined ? columns : [col]).filter( | |
| 97 | (c) => !masked.has(c), | |
| 98 | ); | |
| 99 | const like = opts.filter.text.replace(/[\\%_]/g, (m) => `\\${m}`); | |
| 100 | where.push( | |
| 101 | searchable.length === 0 | |
| 102 | ? "0" | |
| 103 | : searchable | |
| 104 | .map((c) => `CAST(${quote(c)} AS TEXT) LIKE ? ESCAPE '\\'`) | |
| 105 | .join(" OR "), | |
| 106 | ); | |
| 107 | for (let i = 0; i < searchable.length; i++) args.push(`%${like}%`); | |
| 108 | } | |
| 109 | let order = ""; | |
| 110 | if (opts.order) { | |
| 111 | if (!columns.includes(opts.order.column)) return undefined; | |
| 112 | if (masked.has(opts.order.column)) return undefined; | |
| 113 | order = ` ORDER BY ${quote(opts.order.column)} ${opts.order.dir === "desc" ? "DESC" : "ASC"}`; | |
| 114 | } | |
| 115 | const clause = where.length ? ` WHERE ${where.join(" AND ")}` : ""; | |
| 116 | const wanted = Number(opts.page ?? 0); | |
| 117 | const page = Number.isFinite(wanted) | |
| 118 | ? Math.min(MAX_PAGE, Math.max(0, Math.floor(wanted))) | |
| 119 | : 0; | |
| 120 | const total = | |
| 121 | db | |
| 122 | .query<{ n: number }, string[]>( | |
| 123 | `SELECT count(*) AS n FROM ${quote(table)}${clause}`, | |
| 124 | ) | |
| 125 | .get(...args)?.n ?? 0; | |
| 126 | const raw = db | |
| 127 | .query<unknown[], string[]>( | |
| 128 | `SELECT ${columns.map(quote).join(", ")} FROM ${quote(table)}${clause}${order} LIMIT ${PAGE_SIZE} OFFSET ${page * PAGE_SIZE}`, | |
| 129 | ) | |
| 130 | .values(...args); | |
| 131 | return { | |
| 132 | columns, | |
| 133 | page, | |
| 134 | total, | |
| 135 | rows: raw.map((r) => | |
| 136 | r.map((v, i) => | |
| 137 | masked.has(columns[i]!) && v !== null ? MASK : cell(v), | |
| 138 | ), | |
| 139 | ), | |
| 140 | }; | |
| 141 | }); | |
| 142 | } |
src/data/mask.ts added +100
| @@ -0,0 +1,100 @@ | ||
| 1 | import type { Database } from "bun:sqlite"; | |
| 2 | ||
| 3 | /** | |
| 4 | * Which columns hold secrets, by rule. A masked cell shows that a value is | |
| 5 | * there and nothing of it. A source may allow a column by name | |
| 6 | * (`table.column`) when its content is known to be harmless. | |
| 7 | */ | |
| 8 | ||
| 9 | export const MASK = "••••••"; | |
| 10 | ||
| 11 | const SECRET = | |
| 12 | /_enc$|verifier|nonce|challenge|(^|_)code(_hash)?$|token|secret|passw(or)?d|passhash|(^|_)hash$|^hash$|sealed|api_?key|(^|_)key$|^key$|private|credential|cookie|(^|_)salt$|(^|_)sig(nature)?$|^pw/i; | |
| 13 | ||
| 14 | /** Per-source allowlist: source id → `table.column` entries shown in clear. */ | |
| 15 | export const ALLOW: Record<string, readonly string[]> = {}; | |
| 16 | ||
| 17 | /** | |
| 18 | * Per-source columns masked whatever their name: generic ones that hold | |
| 19 | * secrets in some rows (a setting's `value`, a stored response `body`). | |
| 20 | */ | |
| 21 | export const FORCE: Readonly<Record<string, readonly string[]>> = { | |
| 22 | forge: [ | |
| 23 | "forge_setting.value", | |
| 24 | "forge_setting_change.from_value", | |
| 25 | "forge_setting_change.to_value", | |
| 26 | "front_setting.value", | |
| 27 | "user_setting.value", | |
| 28 | "origin_cursor.value", | |
| 29 | "sentinel_kept.value", | |
| 30 | "idempotency.body", | |
| 31 | "event.payload", | |
| 32 | "sentinel_finding.evidence", | |
| 33 | ], | |
| 34 | }; | |
| 35 | ||
| 36 | export const isSecretColumn = (column: string): boolean => SECRET.test(column); | |
| 37 | ||
| 38 | /** Whether `table.column` of a source shows only the mask. */ | |
| 39 | export function masker(sourceId: string): (table: string, column: string) => boolean { | |
| 40 | const allow = new Set(ALLOW[sourceId] ?? []); | |
| 41 | const force = new Set(FORCE[sourceId] ?? []); | |
| 42 | return (table, column) => { | |
| 43 | const key = `${table}.${column}`; | |
| 44 | if (allow.has(key)) return false; | |
| 45 | return force.has(key) || SECRET.test(column); | |
| 46 | }; | |
| 47 | } | |
| 48 | ||
| 49 | const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`; | |
| 50 | ||
| 51 | /** | |
| 52 | * The masked columns of every table and view in `db`. A view is judged by | |
| 53 | * its text: if it mentions any masked column of the database, all of its | |
| 54 | * columns are masked, so an alias cannot launder a secret. | |
| 55 | */ | |
| 56 | export function maskPlan(db: Database, sourceId: string): Map<string, Set<string>> { | |
| 57 | const masked = masker(sourceId); | |
| 58 | const plan = new Map<string, Set<string>>(); | |
| 59 | const objects = db | |
| 60 | .query<{ name: string; type: string; sql: string | null }, []>( | |
| 61 | "SELECT name, type, sql FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'", | |
| 62 | ) | |
| 63 | .all(); | |
| 64 | const columnsOf = (name: string): string[] => | |
| 65 | db | |
| 66 | .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`) | |
| 67 | .all() | |
| 68 | .map((c) => c.name); | |
| 69 | const secretNames = new Set<string>(); | |
| 70 | const cols = new Map<string, string[]>(); | |
| 71 | for (const o of objects) { | |
| 72 | const c = columnsOf(o.name); | |
| 73 | cols.set(o.name, c); | |
| 74 | if (o.type === "table") | |
| 75 | plan.set(o.name, new Set(c.filter((x) => masked(o.name, x)))); | |
| 76 | for (const x of plan.get(o.name) ?? []) secretNames.add(x.toLowerCase()); | |
| 77 | } | |
| 78 | for (const o of objects) { | |
| 79 | if (o.type !== "view") continue; | |
| 80 | const text = (o.sql ?? "").toLowerCase(); | |
| 81 | const leaks = [...secretNames].some((n) => | |
| 82 | new RegExp(`(^|[^a-z0-9_])${n.replace(/[^a-z0-9_]/g, "")}($|[^a-z0-9_])`).test(text), | |
| 83 | ); | |
| 84 | const c = cols.get(o.name)!; | |
| 85 | plan.set( | |
| 86 | o.name, | |
| 87 | new Set(leaks ? c : c.filter((x) => masked(o.name, x))), | |
| 88 | ); | |
| 89 | } | |
| 90 | return plan; | |
| 91 | } | |
| 92 | ||
| 93 | /** A cell as plain JSON-safe data. */ | |
| 94 | export function cell(value: unknown): string | number | null { | |
| 95 | if (value === null || value === undefined) return null; | |
| 96 | if (typeof value === "bigint") return value.toString(); | |
| 97 | if (typeof value === "number" || typeof value === "string") return value; | |
| 98 | if (value instanceof Uint8Array) return `<blob ${value.byteLength} bytes>`; | |
| 99 | return String(value); | |
| 100 | } |
src/data/query.ts added +74
| @@ -0,0 +1,74 @@ | ||
| 1 | /** | |
| 2 | * Read-only SQL against one source: `SELECT` or `WITH`, a read-only | |
| 3 | * connection, a time cap (the worker is ended, which stops even a query | |
| 4 | * that never yields) and a row cap. | |
| 5 | */ | |
| 6 | ||
| 7 | import { checkQuery } from "./sql.ts"; | |
| 8 | import type { Source } from "./sources.ts"; | |
| 9 | ||
| 10 | export const QUERY_TIMEOUT_MS = 5000; | |
| 11 | export const QUERY_ROW_CAP = 1000; | |
| 12 | ||
| 13 | export type QueryResult = | |
| 14 | | { | |
| 15 | readonly ok: true; | |
| 16 | readonly columns: readonly string[]; | |
| 17 | readonly rows: readonly (readonly (string | number | null)[])[]; | |
| 18 | readonly truncated: boolean; | |
| 19 | } | |
| 20 | | { | |
| 21 | readonly ok: false; | |
| 22 | readonly error: "empty" | "not_select" | "multiple" | "schema" | "timeout" | "sql"; | |
| 23 | readonly message: string; | |
| 24 | }; | |
| 25 | ||
| 26 | const REFUSALS = { | |
| 27 | empty: "Write a query.", | |
| 28 | not_select: "Only SELECT and WITH statements are allowed.", | |
| 29 | multiple: "One statement at a time.", | |
| 30 | schema: "Name tables without a schema prefix.", | |
| 31 | } as const; | |
| 32 | ||
| 33 | export function query( | |
| 34 | source: Source, | |
| 35 | sql: string, | |
| 36 | timeoutMs = QUERY_TIMEOUT_MS, | |
| 37 | ): Promise<QueryResult> { | |
| 38 | const verdict = checkQuery(sql); | |
| 39 | if (verdict !== "ok") | |
| 40 | return Promise.resolve({ ok: false, error: verdict, message: REFUSALS[verdict] }); | |
| 41 | return new Promise((resolve) => { | |
| 42 | const worker = new Worker(new URL("./query_worker.ts", import.meta.url).href); | |
| 43 | const done = (r: QueryResult) => { | |
| 44 | clearTimeout(timer); | |
| 45 | worker.terminate(); | |
| 46 | resolve(r); | |
| 47 | }; | |
| 48 | const timer = setTimeout( | |
| 49 | () => | |
| 50 | done({ | |
| 51 | ok: false, | |
| 52 | error: "timeout", | |
| 53 | message: `Stopped after ${timeoutMs / 1000} s.`, | |
| 54 | }), | |
| 55 | timeoutMs, | |
| 56 | ); | |
| 57 | worker.onmessage = (e: MessageEvent) => { | |
| 58 | const m = e.data; | |
| 59 | done( | |
| 60 | m.ok | |
| 61 | ? { ok: true, columns: m.columns, rows: m.rows, truncated: m.truncated } | |
| 62 | : { ok: false, error: "sql", message: String(m.message) }, | |
| 63 | ); | |
| 64 | }; | |
| 65 | worker.onerror = (e) => | |
| 66 | done({ ok: false, error: "sql", message: e.message || "query failed" }); | |
| 67 | worker.postMessage({ | |
| 68 | path: source.path, | |
| 69 | sourceId: source.id, | |
| 70 | sql, | |
| 71 | limit: QUERY_ROW_CAP, | |
| 72 | }); | |
| 73 | }); | |
| 74 | } |
src/data/query_worker.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | /** | |
| 2 | * Runs one query off the main thread so that a slow one can be stopped by | |
| 3 | * ending the thread. Secret columns are shadowed by temporary views that | |
| 4 | * show the mask, so an alias or a subquery cannot read them. | |
| 5 | */ | |
| 6 | ||
| 7 | import { Database } from "bun:sqlite"; | |
| 8 | import { cell, maskPlan, MASK } from "./mask.ts"; | |
| 9 | ||
| 10 | declare const self: Worker; | |
| 11 | ||
| 12 | interface Job { | |
| 13 | readonly path: string; | |
| 14 | readonly sourceId: string; | |
| 15 | readonly sql: string; | |
| 16 | readonly limit: number; | |
| 17 | } | |
| 18 | ||
| 19 | const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`; | |
| 20 | ||
| 21 | self.onmessage = (event: MessageEvent<Job>) => { | |
| 22 | const job = event.data; | |
| 23 | let db: Database | undefined; | |
| 24 | try { | |
| 25 | db = new Database(job.path, { readonly: true }); | |
| 26 | db.exec("PRAGMA busy_timeout = 250"); | |
| 27 | for (const [name, hide] of maskPlan(db, job.sourceId)) { | |
| 28 | if (hide.size === 0) continue; | |
| 29 | const cols = db | |
| 30 | .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`) | |
| 31 | .all(); | |
| 32 | const select = cols | |
| 33 | .map((c) => | |
| 34 | hide.has(c.name) | |
| 35 | ? `CASE WHEN main.${quote(name)}.${quote(c.name)} IS NULL THEN NULL ELSE '${MASK}' END AS ${quote(c.name)}` | |
| 36 | : `main.${quote(name)}.${quote(c.name)} AS ${quote(c.name)}`, | |
| 37 | ) | |
| 38 | .join(", "); | |
| 39 | db.exec(`CREATE TEMP VIEW ${quote(name)} AS SELECT ${select} FROM main.${quote(name)}`); | |
| 40 | } | |
| 41 | db.exec("PRAGMA query_only = 1"); | |
| 42 | const stmt = db.query(job.sql); | |
| 43 | const columns = stmt.columnNames; | |
| 44 | const out: (string | number | null)[][] = []; | |
| 45 | let truncated = false; | |
| 46 | for (const row of stmt.values() as unknown[][]) { | |
| 47 | if (out.length >= job.limit) { | |
| 48 | truncated = true; | |
| 49 | break; | |
| 50 | } | |
| 51 | out.push(row.map(cell)); | |
| 52 | } | |
| 53 | self.postMessage({ ok: true, columns, rows: out, truncated }); | |
| 54 | } catch (error) { | |
| 55 | self.postMessage({ | |
| 56 | ok: false, | |
| 57 | message: error instanceof Error ? error.message : String(error), | |
| 58 | }); | |
| 59 | } finally { | |
| 60 | db?.close(); | |
| 61 | } | |
| 62 | }; |
src/data/sources.ts added +81
| @@ -0,0 +1,81 @@ | ||
| 1 | /** | |
| 2 | * The SQLite files a viewer may see: the forge's own (admins), and each | |
| 3 | * app's per environment (people who can write the app's repository). An id | |
| 4 | * is only ever resolved by looking it up in this list, so no input becomes | |
| 5 | * a path. | |
| 6 | */ | |
| 7 | ||
| 8 | import type { Database } from "bun:sqlite"; | |
| 9 | import { existsSync, readdirSync } from "node:fs"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { listApps } from "../core/apps.ts"; | |
| 12 | import { getRepoById, writableRepoIds } from "../core/repos.ts"; | |
| 13 | import type { User } from "../core/users.ts"; | |
| 14 | import { envDataDir, isEnvName } from "../apps/data.ts"; | |
| 15 | ||
| 16 | export interface Source { | |
| 17 | /** `forge`, or `app:<appId>:<env>:<file>`. */ | |
| 18 | readonly id: string; | |
| 19 | readonly label: string; | |
| 20 | readonly kind: "forge" | "app"; | |
| 21 | readonly path: string; | |
| 22 | } | |
| 23 | ||
| 24 | const DB_FILE = /^[\w.-]+\.db$/; | |
| 25 | ||
| 26 | export function listSources( | |
| 27 | db: Database, | |
| 28 | dataDir: string, | |
| 29 | viewer: User | undefined, | |
| 30 | ): Source[] { | |
| 31 | const out: Source[] = []; | |
| 32 | if (viewer === undefined) return out; | |
| 33 | if (viewer.isAdmin) { | |
| 34 | const path = join(dataDir, "forge.db"); | |
| 35 | if (existsSync(path)) | |
| 36 | out.push({ id: "forge", label: "forge", kind: "forge", path }); | |
| 37 | } | |
| 38 | const apps = listApps(db); | |
| 39 | const writable = writableRepoIds( | |
| 40 | db, | |
| 41 | viewer, | |
| 42 | apps.map((a) => a.repoId), | |
| 43 | ); | |
| 44 | for (const app of apps) { | |
| 45 | if (!writable.has(app.repoId) || getRepoById(db, app.repoId) === undefined) | |
| 46 | continue; | |
| 47 | let envs: string[]; | |
| 48 | try { | |
| 49 | envs = readdirSync(join(dataDir, "apps", app.id)).filter(isEnvName); | |
| 50 | } catch { | |
| 51 | continue; | |
| 52 | } | |
| 53 | for (const env of envs.sort()) { | |
| 54 | if (!isEnvName(env)) continue; | |
| 55 | const dir = envDataDir(dataDir, app.id, env); | |
| 56 | let files: string[]; | |
| 57 | try { | |
| 58 | files = readdirSync(dir).filter((f) => DB_FILE.test(f)); | |
| 59 | } catch { | |
| 60 | continue; | |
| 61 | } | |
| 62 | for (const file of files.sort()) | |
| 63 | out.push({ | |
| 64 | id: `app:${app.id}:${env}:${file}`, | |
| 65 | label: `${app.slug} · ${env} · ${file}`, | |
| 66 | kind: "app", | |
| 67 | path: join(dir, file), | |
| 68 | }); | |
| 69 | } | |
| 70 | } | |
| 71 | return out; | |
| 72 | } | |
| 73 | ||
| 74 | /** The source `id` names, if this viewer may see it; else `undefined` (a 404). */ | |
| 75 | export const findSource = ( | |
| 76 | db: Database, | |
| 77 | dataDir: string, | |
| 78 | viewer: User | undefined, | |
| 79 | id: string, | |
| 80 | ): Source | undefined => | |
| 81 | listSources(db, dataDir, viewer).find((s) => s.id === id); |
src/data/sql.ts added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | /** What a read-only query may be: one SELECT or WITH statement, nothing else. */ | |
| 2 | ||
| 3 | /** The statement with comments and string literals blanked; identifiers stay. */ | |
| 4 | function skeleton(sql: string): string { | |
| 5 | let out = ""; | |
| 6 | for (let i = 0; i < sql.length; i++) { | |
| 7 | const c = sql[i]!; | |
| 8 | if (c === "-" && sql[i + 1] === "-") { | |
| 9 | while (i < sql.length && sql[i] !== "\n") i++; | |
| 10 | out += " "; | |
| 11 | } else if (c === "/" && sql[i + 1] === "*") { | |
| 12 | const end = sql.indexOf("*/", i + 2); | |
| 13 | i = end < 0 ? sql.length : end + 1; | |
| 14 | out += " "; | |
| 15 | } else if (c === "'") { | |
| 16 | i++; | |
| 17 | while (i < sql.length && sql[i] !== "'") i++; | |
| 18 | out += "''"; | |
| 19 | } else out += c; | |
| 20 | } | |
| 21 | return out; | |
| 22 | } | |
| 23 | ||
| 24 | export function checkQuery(sql: string): "ok" | "empty" | "not_select" | "multiple" | "schema" { | |
| 25 | const s = skeleton(sql).trim().replace(/;+\s*$/, "").trim(); | |
| 26 | if (s === "") return "empty"; | |
| 27 | if (s.includes(";")) return "multiple"; | |
| 28 | if (!/^(select|with)\b/i.test(s)) return "not_select"; | |
| 29 | // `main.t` would step round the masking views that shadow `t`. | |
| 30 | if (/(^|[^\w])("main"|\[main\]|`main`|main)\s*\./i.test(s)) return "schema"; | |
| 31 | return "ok"; | |
| 32 | } |
src/http/mcp/tools/data.ts added +39
| @@ -0,0 +1,39 @@ | ||
| 1 | import { rows, tables } from "../../../data/browse.ts"; | |
| 2 | import { query } from "../../../data/query.ts"; | |
| 3 | import { findSource, listSources } from "../../../data/sources.ts"; | |
| 4 | import { bad, good, type Tool } from "../shared.ts"; | |
| 5 | ||
| 6 | export const tool: Tool = { | |
| 7 | name: "data", | |
| 8 | doc: "the SQLite files you may read (the forge's own for an admin; each app's per environment for its writers), with `source`: its tables and row counts, with `table`: a page of rows, with `sql`: a read-only SELECT (5 s, 1,000 rows); secret columns are masked", | |
| 9 | description: "Read SQLite files you may see: tables, rows, read-only SQL; secrets masked.", | |
| 10 | inputSchema: { | |
| 11 | type: "object", | |
| 12 | properties: { | |
| 13 | source: { type: "string" }, | |
| 14 | table: { type: "string" }, | |
| 15 | page: { type: "integer" }, | |
| 16 | sql: { type: "string" }, | |
| 17 | }, | |
| 18 | }, | |
| 19 | writes: false, | |
| 20 | async run(ctx, user, args) { | |
| 21 | const { db, config } = ctx.app; | |
| 22 | const { source: id, table, page, sql } = args; | |
| 23 | for (const [k, v] of Object.entries({ source: id, table, sql })) | |
| 24 | if (v !== undefined && typeof v !== "string") return bad("invalid", `\`${k}\` is text`); | |
| 25 | if (page !== undefined && typeof page !== "number") return bad("invalid", "`page` is a number"); | |
| 26 | if (id === undefined) { | |
| 27 | return good(listSources(db, config.dataDir, user).map((s) => ({ source: s.id, label: s.label }))); | |
| 28 | } | |
| 29 | const source = findSource(db, config.dataDir, user, id as string); | |
| 30 | if (!source) return bad("not_found", "no such source"); | |
| 31 | if (sql !== undefined) { | |
| 32 | const result = await query(source, sql as string); | |
| 33 | return result.ok ? good(result) : bad("invalid", result.message); | |
| 34 | } | |
| 35 | if (table === undefined) return good(tables(source)); | |
| 36 | const found = rows(source, table as string, page === undefined ? {} : { page: page as number }); | |
| 37 | return found ? good(found) : bad("not_found", "no such table"); | |
| 38 | }, | |
| 39 | }; |
src/http/mcp/tools/index.ts +1
| @@ -44,6 +44,7 @@ export { tool as confirm_wait } from "./confirm_wait.ts"; | ||
| 44 | 44 | export { tool as contribute } from "./contribute.ts"; |
| 45 | 45 | export { tool as converge } from "./converge.ts"; |
| 46 | 46 | export { tool as crew } from "./crew.ts"; |
| 47 | export { tool as data } from "./data.ts"; | |
| 47 | 48 | export { tool as decisions_by } from "./decisions_by.ts"; |
| 48 | 49 | export { tool as detach } from "./detach.ts"; |
| 49 | 50 | export { tool as drills } from "./drills.ts"; |
src/http/server.ts +2
| @@ -20,6 +20,7 @@ import { systemPage, systemStream } from "../web/pages/system.ts"; | ||
| 20 | 20 | import { contributePage } from "../web/pages/contribute.ts"; |
| 21 | 21 | import { pipelinePage } from "../web/pages/pipeline.ts"; |
| 22 | 22 | import { resiliencePage } from "../web/pages/resilience.ts"; |
| 23 | import { dataPage } from "../web/pages/data.ts"; | |
| 23 | 24 | import { limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts"; |
| 24 | 25 | import { measureSubmit } from "./measure.ts"; |
| 25 | 26 | import { settingsRoutes } from "./settings_routes.ts"; |
| @@ -813,6 +814,7 @@ export function buildRouter(): Router<Handler> { | ||
| 813 | 814 | pinnedRoutes(router); |
| 814 | 815 | // Each THREATS.md protection against the last gate; literal, above /:owner. |
| 815 | 816 | router.add("GET", "/resilience", resiliencePage); |
| 817 | router.add("GET", "/data", dataPage); | |
| 816 | 818 | // A selection's spend and runs over time; literal, above /:owner. |
| 817 | 819 | router.add("GET", "/fleet/graph", fleetGraphPage); |
| 818 | 820 | router.add("GET", "/fleet/graph/live", fleetGraphLive); |
src/web/pages/apps.ts +5
| @@ -12,6 +12,7 @@ | ||
| 12 | 12 | * 404 for everyone. |
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | import { listSources } from "../../data/sources.ts"; | |
| 15 | 16 | import { storageLimitMessage } from "../../core/storage.ts"; |
| 16 | 17 | import { repositoryLimitMessage } from "../../actions/repos.ts"; |
| 17 | 18 | import { appLimitWords } from "../../actions/app_limit.ts"; |
| @@ -235,8 +236,12 @@ export async function appsList(ctx: Ctx): Promise<Response> { | ||
| 235 | 236 | costLine(ctx.app.db, writableAppsOf(ctx, ctx.auth.user).repos), |
| 236 | 237 | ); |
| 237 | 238 | const rendering = performance.now(); |
| 239 | const hasData = | |
| 240 | ctx.auth.user !== undefined && | |
| 241 | listSources(ctx.app.db, ctx.app.config.dataDir, ctx.auth.user).length > 0; | |
| 238 | 242 | const body = html`${when(ctx.auth.user, (user) => homeTabs("apps", unshippedAsks(ctx.app.db, user)))} |
| 239 | 243 | ${pageHead("Apps", { actions: create })} |
| 244 | ${hasData ? html`<p class="muted small mb"><a href="/data">Data</a></p>` : null} | |
| 240 | 245 | ${cost} |
| 241 | 246 | ${filters} |
| 242 | 247 | ${archivedView |
src/web/pages/data.ts added +87
| @@ -0,0 +1,87 @@ | ||
| 1 | /** | |
| 2 | * /data: the SQLite files the viewer may see (`data/sources.ts`), the | |
| 3 | * tables in one, and a page of a table's rows, secrets masked. Anyone who | |
| 4 | * may see no source gets the not-found page, as a guest does. | |
| 5 | */ | |
| 6 | ||
| 7 | import { PAGE_SIZE, rows, tables } from "../../data/browse.ts"; | |
| 8 | import { findSource, listSources } from "../../data/sources.ts"; | |
| 9 | import type { Ctx } from "../../http/context.ts"; | |
| 10 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; | |
| 11 | import { notFoundPage, signInFirst } from "../chrome.ts"; | |
| 12 | import { html } from "../html.ts"; | |
| 13 | import { page } from "../layout.ts"; | |
| 14 | import { card } from "../ui.ts"; | |
| 15 | ||
| 16 | const href = (params: Record<string, string | number>): string => | |
| 17 | `/data?${new URLSearchParams(Object.entries(params).map(([k, v]): [string, string] => [k, String(v)]))}`; | |
| 18 | ||
| 19 | // GET /data[?source=<id>[&table=<name>[&page=<n>]]] | |
| 20 | export function dataPage(ctx: Ctx): Response { | |
| 21 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); | |
| 22 | const viewer = ctx.auth.user; | |
| 23 | if (!viewer) return signInFirst(ctx); | |
| 24 | const { db, config } = ctx.app; | |
| 25 | const q = ctx.url.searchParams; | |
| 26 | const all = listSources(db, config.dataDir, viewer); | |
| 27 | if (all.length === 0) return notFoundPage(ctx); | |
| 28 | const chosen = q.get("source"); | |
| 29 | const source = chosen === null ? undefined : findSource(db, config.dataDir, viewer, chosen); | |
| 30 | if (chosen !== null && !source) return notFoundPage(ctx); | |
| 31 | ||
| 32 | let body = html`<div class="page-head"><h1>Data</h1></div> | |
| 33 | ${card( | |
| 34 | html`<ul class="list"> | |
| 35 | ${all.map( | |
| 36 | (s) => | |
| 37 | html`<li class="row"> | |
| 38 | <span class="row-main"><a class="row-title" href="${href({ source: s.id })}">${s.label}</a></span> | |
| 39 | </li>`, | |
| 40 | )} | |
| 41 | </ul>`, | |
| 42 | { pad: false }, | |
| 43 | )}`; | |
| 44 | ||
| 45 | if (source) { | |
| 46 | const list = tables(source); | |
| 47 | const name = q.get("table"); | |
| 48 | const shown = name === null ? undefined : rows(source, name, { page: Number(q.get("page") ?? 0) }); | |
| 49 | if (name !== null && !shown) return notFoundPage(ctx); | |
| 50 | body = html`${body} | |
| 51 | <h2>${source.label}</h2> | |
| 52 | ${card( | |
| 53 | html`<ul class="list"> | |
| 54 | ${list.map( | |
| 55 | (t) => | |
| 56 | html`<li class="row"> | |
| 57 | <span class="row-main"><a class="row-title" href="${href({ source: source.id, table: t.name })}">${t.name}</a></span> | |
| 58 | <span class="row-trail">${String(t.rows)} rows</span> | |
| 59 | </li>`, | |
| 60 | )} | |
| 61 | </ul>`, | |
| 62 | { pad: false }, | |
| 63 | )}`; | |
| 64 | if (shown && name !== null) { | |
| 65 | const more = (shown.page + 1) * PAGE_SIZE < shown.total; | |
| 66 | body = html`${body} | |
| 67 | <h2>${name}</h2> | |
| 68 | ${card( | |
| 69 | html`<table> | |
| 70 | <thead><tr>${shown.columns.map((c) => html`<th>${c}</th>`)}</tr></thead> | |
| 71 | <tbody> | |
| 72 | ${shown.rows.map( | |
| 73 | (r) => html`<tr>${r.map((v) => html`<td>${v === null ? "" : String(v)}</td>`)}</tr>`, | |
| 74 | )} | |
| 75 | </tbody> | |
| 76 | </table>`, | |
| 77 | )} | |
| 78 | <p class="muted"> | |
| 79 | ${String(shown.total)} rows${shown.page > 0 ? html` · <a href="${href({ source: source.id, table: name, page: shown.page - 1 })}">Newer page</a>` : ""}${more ? html` · <a href="${href({ source: source.id, table: name, page: shown.page + 1 })}">Next page</a>` : ""} | |
| 80 | </p>`; | |
| 81 | } | |
| 82 | } | |
| 83 | const rendering = performance.now(); | |
| 84 | const response = page({ ctx, title: "Data", body }); | |
| 85 | notePhase(ctx.request, "render", performance.now() - rendering); | |
| 86 | return response; | |
| 87 | } |
test/data-browser.test.ts added +293
| @@ -0,0 +1,293 @@ | ||
| 1 | /** | |
| 2 | * The data browser's core: who sees which SQLite file, what a query may be, | |
| 3 | * and what is masked. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import { Database } from "bun:sqlite"; | |
| 7 | import { mkdirSync } from "node:fs"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp as createAppRecord } from "../src/core/apps.ts"; | |
| 12 | import { createRepo } from "../src/core/repos.ts"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { envDataDir } from "../src/apps/data.ts"; | |
| 15 | import { createUser, type User } from "../src/core/users.ts"; | |
| 16 | import { rows, tables } from "../src/data/browse.ts"; | |
| 17 | import { ALLOW, maskPlan } from "../src/data/mask.ts"; | |
| 18 | import { query } from "../src/data/query.ts"; | |
| 19 | import { findSource, listSources } from "../src/data/sources.ts"; | |
| 20 | import { quietLogs } from "./helpers/globals.ts"; | |
| 21 | ||
| 22 | quietLogs(); | |
| 23 | ||
| 24 | const AUDITED_CLEAR: string[] = [ | |
| 25 | "app_domain.said", | |
| 26 | "comment.body", | |
| 27 | "degraded_span.sentence", | |
| 28 | "feed_chip.value", | |
| 29 | "fleet_operation.value", | |
| 30 | "issue.body", | |
| 31 | "oauth_flow.detail", | |
| 32 | "oidc_key.public_jwk", | |
| 33 | "probe.said", | |
| 34 | "prompt_source.last_said", | |
| 35 | "repo_upstream_check.detail", | |
| 36 | "report.body", | |
| 37 | "review.body", | |
| 38 | "run.check_detail", | |
| 39 | "subscription.resting_said", | |
| 40 | "subscription_limit.said", | |
| 41 | "user_key.blob", | |
| 42 | "webhook_ping.payload", | |
| 43 | "week_projection.body", | |
| 44 | "week_retro.body", | |
| 45 | ]; | |
| 46 | ||
| 47 | describe("data browser core", () => { | |
| 48 | let dataDir: string; | |
| 49 | let app: App; | |
| 50 | let admin: User; | |
| 51 | let alice: User; | |
| 52 | let bob: User; | |
| 53 | let appId = ""; | |
| 54 | ||
| 55 | const user = async (username: string, adminIfFirst = false) => { | |
| 56 | const made = await createUser(app.db, { | |
| 57 | origin: "seed", | |
| 58 | trusted: true, | |
| 59 | username, | |
| 60 | email: `${username}@forge.local`, | |
| 61 | password: "a-good-passphrase", | |
| 62 | adminIfFirst, | |
| 63 | }); | |
| 64 | if (!made.ok) throw new Error(made.error); | |
| 65 | return made.value; | |
| 66 | }; | |
| 67 | const ids = (viewer: User | undefined) => | |
| 68 | listSources(app.db, dataDir, viewer).map((s) => s.id); | |
| 69 | ||
| 70 | beforeAll(async () => { | |
| 71 | dataDir = await mkdtemp(join(tmpdir(), "forge-data-browser-")); | |
| 72 | app = createApp({ dataDir, port: 0, host: "forge.test" }); | |
| 73 | admin = await user("owner1", true); | |
| 74 | alice = await user("alice"); | |
| 75 | bob = await user("bob"); | |
| 76 | const repo = await createRepo(app.db, app.reposRoot, { | |
| 77 | ownerId: alice.id, | |
| 78 | name: "notes", | |
| 79 | visibility: "public", | |
| 80 | }); | |
| 81 | if (!repo.ok) throw new Error(repo.error); | |
| 82 | const made = createAppRecord(app.db, { | |
| 83 | repoId: repo.value.id, | |
| 84 | slug: "notes", | |
| 85 | access: "public", | |
| 86 | }); | |
| 87 | if (!made.ok) throw new Error(made.error); | |
| 88 | appId = made.value.id; | |
| 89 | for (const env of ["production", "pull-3"] as const) { | |
| 90 | mkdirSync(envDataDir(dataDir, appId, env), { recursive: true }); | |
| 91 | const d = new Database(join(envDataDir(dataDir, appId, env), "app.db")); | |
| 92 | d.exec("PRAGMA journal_mode = WAL"); | |
| 93 | d.exec( | |
| 94 | "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT, password_hash TEXT)", | |
| 95 | ); | |
| 96 | d.exec("CREATE VIEW people AS SELECT name, api_token AS harmless FROM users"); | |
| 97 | d.exec( | |
| 98 | `INSERT INTO users VALUES (1, 'ann', 'tok-ann-1', 'hash-1'), (2, 'ben', 'tok-ben-2', NULL)`, | |
| 99 | ); | |
| 100 | d.close(); | |
| 101 | } | |
| 102 | }); | |
| 103 | ||
| 104 | afterAll(async () => { | |
| 105 | app.close(); | |
| 106 | await rm(dataDir, { recursive: true, force: true }); | |
| 107 | }); | |
| 108 | ||
| 109 | test("a writer sees their app's production and preview databases", () => { | |
| 110 | expect(ids(alice)).toEqual([ | |
| 111 | `app:${appId}:production:app.db`, | |
| 112 | `app:${appId}:pull-3:app.db`, | |
| 113 | ]); | |
| 114 | }); | |
| 115 | ||
| 116 | test("a reader of the app can't find its data", () => { | |
| 117 | expect(ids(bob)).toEqual([]); | |
| 118 | expect(ids(undefined)).toEqual([]); | |
| 119 | expect( | |
| 120 | findSource(app.db, dataDir, bob, `app:${appId}:production:app.db`), | |
| 121 | ).toBeUndefined(); | |
| 122 | }); | |
| 123 | ||
| 124 | test("only an admin sees forge.db", () => { | |
| 125 | expect(ids(admin)).toContain("forge"); | |
| 126 | expect(ids(alice)).not.toContain("forge"); | |
| 127 | expect(findSource(app.db, dataDir, alice, "forge")).toBeUndefined(); | |
| 128 | }); | |
| 129 | ||
| 130 | test("tables with counts, rows with filter, order and page", () => { | |
| 131 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 132 | expect(tables(src)).toEqual([{ name: "users", rows: 2 }]); | |
| 133 | const r = rows(src, "users", { filter: { text: "b" }, order: { column: "id", dir: "desc" } })!; | |
| 134 | expect(r.total).toBe(1); | |
| 135 | expect(r.rows[0]![1]).toBe("ben"); | |
| 136 | expect(rows(src, "nope")).toBeUndefined(); | |
| 137 | expect(rows(src, "users", { order: { column: "id; DROP TABLE users" } })).toBeUndefined(); | |
| 138 | }); | |
| 139 | ||
| 140 | test("a token column is masked, and cannot be searched or ordered", () => { | |
| 141 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 142 | const r = rows(src, "users")!; | |
| 143 | expect(JSON.stringify(r)).not.toContain("tok-ann"); | |
| 144 | expect(JSON.stringify(r)).not.toContain("hash-1"); | |
| 145 | expect(r.rows[1]![3]).toBeNull(); | |
| 146 | expect(rows(src, "users", { filter: { text: "tok-ann", column: "api_token" } })!.total).toBe(0); | |
| 147 | expect(rows(src, "users", { order: { column: "api_token" } })).toBeUndefined(); | |
| 148 | }); | |
| 149 | ||
| 150 | test("a query cannot read a secret by alias or subquery either", async () => { | |
| 151 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 152 | for (const sql of [ | |
| 153 | "SELECT api_token AS t FROM users", | |
| 154 | "SELECT (SELECT api_token FROM users LIMIT 1) AS t", | |
| 155 | "SELECT * FROM users", | |
| 156 | ]) { | |
| 157 | const r = await query(src, sql); | |
| 158 | expect(r.ok).toBe(true); | |
| 159 | expect(JSON.stringify(r)).not.toContain("tok-ann"); | |
| 160 | } | |
| 161 | const bypass = await query(src, "SELECT api_token FROM main.users"); | |
| 162 | expect(bypass.ok).toBe(false); | |
| 163 | const plain = await query(src, "SELECT name FROM users ORDER BY id"); | |
| 164 | expect(plain.ok && plain.rows).toEqual([["ann"], ["ben"]]); | |
| 165 | }); | |
| 166 | ||
| 167 | test("INSERT and PRAGMA writable_schema are refused", async () => { | |
| 168 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 169 | for (const sql of [ | |
| 170 | "INSERT INTO users (name) VALUES ('x')", | |
| 171 | "PRAGMA writable_schema = 1", | |
| 172 | "SELECT 1; DELETE FROM users", | |
| 173 | "WITH x AS (SELECT 1) DELETE FROM users", | |
| 174 | "-- hi\nUPDATE users SET name = 'x'", | |
| 175 | ]) { | |
| 176 | const r = await query(src, sql); | |
| 177 | expect(r.ok).toBe(false); | |
| 178 | } | |
| 179 | expect(rows(src, "users")!.total).toBe(2); | |
| 180 | }); | |
| 181 | ||
| 182 | test("a result stops at 1,000 rows", async () => { | |
| 183 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 184 | const r = await query( | |
| 185 | src, | |
| 186 | "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 5000) SELECT i FROM n", | |
| 187 | ); | |
| 188 | expect(r.ok && r.rows.length).toBe(1000); | |
| 189 | expect(r.ok && r.truncated).toBe(true); | |
| 190 | }); | |
| 191 | ||
| 192 | test("a slow query stops at the cap", async () => { | |
| 193 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 194 | const started = performance.now(); | |
| 195 | const r = await query( | |
| 196 | src, | |
| 197 | "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n) SELECT count(*) FROM n", | |
| 198 | 400, | |
| 199 | ); | |
| 200 | expect(r.ok).toBe(false); | |
| 201 | expect(!r.ok && r.error).toBe("timeout"); | |
| 202 | expect(performance.now() - started).toBeLessThan(3000); | |
| 203 | }); | |
| 204 | ||
| 205 | test("reading never blocks the app's own writes", () => { | |
| 206 | const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 207 | const w = new Database(src.path); | |
| 208 | tables(src); | |
| 209 | w.exec("INSERT INTO users (name) VALUES ('cy')"); | |
| 210 | w.close(); | |
| 211 | expect(rows(src, "users")!.total).toBe(3); | |
| 212 | }); | |
| 213 | ||
| 214 | const prod = () => findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!; | |
| 215 | ||
| 216 | test("a schema qualifier in any quoting is refused", async () => { | |
| 217 | for (const from of [ | |
| 218 | '"main"."users"', | |
| 219 | "[main].[users]", | |
| 220 | "`main`.`users`", | |
| 221 | 'main /**/ .users', | |
| 222 | '"MAIN" . users', | |
| 223 | "main\n.users", | |
| 224 | ]) { | |
| 225 | const r = await query(prod(), `SELECT api_token FROM ${from}`); | |
| 226 | expect(r.ok).toBe(false); | |
| 227 | expect(JSON.stringify(r)).not.toContain("tok-ann"); | |
| 228 | } | |
| 229 | }); | |
| 230 | ||
| 231 | test("a view that aliases a secret is masked, in rows and in queries", async () => { | |
| 232 | const src = prod(); | |
| 233 | expect(JSON.stringify(rows(src, "people") ?? "")).not.toContain("tok-ann"); | |
| 234 | const r = await query(src, "SELECT harmless FROM people"); | |
| 235 | expect(r.ok).toBe(true); | |
| 236 | expect(JSON.stringify(r)).not.toContain("tok-ann"); | |
| 237 | expect(JSON.stringify(r)).toContain("••••"); | |
| 238 | }); | |
| 239 | ||
| 240 | test("an allowlisted column is shown in clear, and only for its source", () => { | |
| 241 | const src = prod(); | |
| 242 | ALLOW[src.id] = ["users.api_token"]; | |
| 243 | try { | |
| 244 | expect(JSON.stringify(rows(src, "users"))).toContain("tok-ann-1"); | |
| 245 | expect(JSON.stringify(rows(src, "users"))).not.toContain("hash-1"); | |
| 246 | const other = findSource(app.db, dataDir, alice, `app:${appId}:pull-3:app.db`)!; | |
| 247 | expect(JSON.stringify(rows(other, "users"))).not.toContain("tok-ann"); | |
| 248 | } finally { | |
| 249 | delete ALLOW[src.id]; | |
| 250 | } | |
| 251 | }); | |
| 252 | ||
| 253 | test("a filter on a column that does not exist finds nothing", () => { | |
| 254 | expect(rows(prod(), "users", { filter: { text: "a", column: "nope" } })).toBeUndefined(); | |
| 255 | }); | |
| 256 | ||
| 257 | test("a guest or a non-writer cannot find forge", () => { | |
| 258 | for (const who of [undefined, alice, bob]) expect(findSource(app.db, dataDir, who, "forge")).toBeUndefined(); | |
| 259 | }); | |
| 260 | ||
| 261 | test("a page that is not a finite number is the first page; a huge one is capped", () => { | |
| 262 | const src = prod(); | |
| 263 | for (const page of [Number.NaN, Number.POSITIVE_INFINITY, -4]) | |
| 264 | expect(rows(src, "users", { page })!.page).toBe(0); | |
| 265 | expect(rows(src, "users", { page: 1e12 })!.page).toBe(1000); | |
| 266 | }); | |
| 267 | ||
| 268 | test("forge.db is masked for an admin, setting values included", async () => { | |
| 269 | app.db.exec("INSERT OR REPLACE INTO forge_setting (key, value, updated_by, updated_at) VALUES ('smtp', 'SETTING-SECRET', 'x', 'now')"); | |
| 270 | const src = findSource(app.db, dataDir, admin, "forge")!; | |
| 271 | expect(JSON.stringify(rows(src, "forge_setting"))).not.toContain("SETTING-SECRET"); | |
| 272 | const r = await query(src, "SELECT value FROM forge_setting"); | |
| 273 | expect(JSON.stringify(r)).not.toContain("SETTING-SECRET"); | |
| 274 | const users = JSON.stringify(rows(src, "user")); | |
| 275 | expect(users).not.toContain("$argon2"); | |
| 276 | }); | |
| 277 | ||
| 278 | test("every forge.db column that could hold a secret is masked or audited as clear", () => { | |
| 279 | const src = findSource(app.db, dataDir, admin, "forge")!; | |
| 280 | const plan = maskPlan(new Database(src.path, { readonly: true }), "forge"); | |
| 281 | const loose = /value|body|payload|code|key|blob|nonce|verifier|secret|token|hash|salt|enc|jwk|evidence|said|detail/i; | |
| 282 | const open: string[] = []; | |
| 283 | for (const [table, masked] of plan) { | |
| 284 | const db = new Database(src.path, { readonly: true }); | |
| 285 | for (const c of db.query<{ name: string }, []>(`PRAGMA table_info("${table}")`).all()) | |
| 286 | if (loose.test(c.name) && !masked.has(c.name)) open.push(`${table}.${c.name}`); | |
| 287 | db.close(); | |
| 288 | } | |
| 289 | // Reviewed: these hold public or operational text, never a credential. | |
| 290 | // A new column that matches and is not here fails until someone looks. | |
| 291 | expect(open.sort()).toEqual(AUDITED_CLEAR); | |
| 292 | }); | |
| 293 | }); |
test/data-door.test.ts added +100
| @@ -0,0 +1,100 @@ | ||
| 1 | /** /data and the `data` MCP tool: gated by listSources, so a guest or a reader sees nothing. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { Database } from "bun:sqlite"; | |
| 4 | import { mkdirSync } from "node:fs"; | |
| 5 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 6 | import { tmpdir } from "node:os"; | |
| 7 | import { join } from "node:path"; | |
| 8 | import { createApp, type App } from "../src/app.ts"; | |
| 9 | import { envDataDir } from "../src/apps/data.ts"; | |
| 10 | import { createApp as createAppRecord } from "../src/core/apps.ts"; | |
| 11 | import { createRepo } from "../src/core/repos.ts"; | |
| 12 | import { createSession } from "../src/core/sessions.ts"; | |
| 13 | import { createToken } from "../src/core/tokens.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 16 | import { quietLogs } from "./helpers/globals.ts"; | |
| 17 | import { mcpTool } from "./helpers/mcp.ts"; | |
| 18 | ||
| 19 | quietLogs(); | |
| 20 | ||
| 21 | describe("the data door", () => { | |
| 22 | let dataDir: string; | |
| 23 | let app: App; | |
| 24 | let servers: Servers; | |
| 25 | const cookie: Record<string, string> = {}; | |
| 26 | const token: Record<string, string> = {}; | |
| 27 | let appId = ""; | |
| 28 | ||
| 29 | beforeAll(async () => { | |
| 30 | dataDir = await mkdtemp(join(tmpdir(), "forge-data-door-")); | |
| 31 | app = createApp({ dataDir, port: 0, host: "forge.test" }); | |
| 32 | servers = await serve(app); | |
| 33 | const users: Record<string, string> = {}; | |
| 34 | for (const name of ["owner1", "alice", "bob"]) { | |
| 35 | const made = await createUser(app.db, { | |
| 36 | origin: "seed", trusted: true, username: name, | |
| 37 | email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "owner1", | |
| 38 | }); | |
| 39 | if (!made.ok) throw new Error(made.error); | |
| 40 | users[name] = made.value.id; | |
| 41 | cookie[name] = `forge_session=${createSession(app.db, made.value.id).token}`; | |
| 42 | token[name] = createToken(app.db, made.value.id, "t", "admin").token; | |
| 43 | } | |
| 44 | const repo = await createRepo(app.db, app.reposRoot, { ownerId: users["alice"]!, name: "notes", visibility: "public" }); | |
| 45 | if (!repo.ok) throw new Error(repo.error); | |
| 46 | const rec = createAppRecord(app.db, { repoId: repo.value.id, slug: "notes", access: "public" }); | |
| 47 | if (!rec.ok) throw new Error(rec.error); | |
| 48 | appId = rec.value.id; | |
| 49 | mkdirSync(envDataDir(dataDir, appId, "production"), { recursive: true }); | |
| 50 | const d = new Database(join(envDataDir(dataDir, appId, "production"), "app.db")); | |
| 51 | d.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT)"); | |
| 52 | d.exec("INSERT INTO users VALUES (1, 'ann', 'tok-ann-1')"); | |
| 53 | d.close(); | |
| 54 | }, 60_000); | |
| 55 | ||
| 56 | afterAll(async () => { | |
| 57 | servers.stop(); | |
| 58 | app.close(); | |
| 59 | await rm(dataDir, { recursive: true, force: true }); | |
| 60 | }); | |
| 61 | ||
| 62 | const get = (path: string, who?: string) => | |
| 63 | fetch(`${servers.url}${path}`, { headers: who ? { cookie: cookie[who]! } : {}, redirect: "manual" }); | |
| 64 | const src = () => encodeURIComponent(`app:${appId}:production:app.db`); | |
| 65 | ||
| 66 | test("a writer reaches sources, tables and masked rows on the page", async () => { | |
| 67 | const list = await (await get("/data", "alice")).text(); | |
| 68 | expect(list).toContain("notes · production · app.db"); | |
| 69 | const tables = await (await get(`/data?source=${src()}`, "alice")).text(); | |
| 70 | expect(tables).toContain("users"); | |
| 71 | const res = await get(`/data?source=${src()}&table=users`, "alice"); | |
| 72 | const rows = await res.text(); | |
| 73 | expect(rows).toContain("ann"); | |
| 74 | expect(rows).not.toContain("tok-ann-1"); | |
| 75 | expect(await (await get("/apps", "alice")).text()).toContain('href="/data"'); | |
| 76 | }); | |
| 77 | ||
| 78 | test("a reader, a guest and a wrong source see nothing", async () => { | |
| 79 | expect((await get("/data", "bob")).status).toBe(404); | |
| 80 | expect((await get(`/data?source=${src()}`, "bob")).status).toBe(404); | |
| 81 | expect((await get("/data")).status).not.toBe(200); | |
| 82 | expect((await get("/data?source=forge", "alice")).status).toBe(404); | |
| 83 | expect((await get("/data?source=forge", "owner1")).status).toBe(200); | |
| 84 | expect(await (await get("/apps", "bob")).text()).not.toContain('href="/data"'); | |
| 85 | }); | |
| 86 | ||
| 87 | test("the MCP tool lists, reads, queries and refuses the same way", async () => { | |
| 88 | const id = `app:${appId}:production:app.db`; | |
| 89 | expect((await mcpTool(servers.url, token["alice"]!, "data")).value).toEqual([ | |
| 90 | { source: id, label: "notes · production · app.db" }, | |
| 91 | ]); | |
| 92 | expect((await mcpTool(servers.url, token["alice"]!, "data", { source: id })).value).toEqual([{ name: "users", rows: 1 }]); | |
| 93 | const q = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "SELECT api_token FROM users" }); | |
| 94 | expect(JSON.stringify(q)).not.toContain("tok-ann-1"); | |
| 95 | const w = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "DELETE FROM users" }); | |
| 96 | expect(w.error).toBeDefined(); | |
| 97 | expect((await mcpTool(servers.url, token["bob"]!, "data", { source: id })).error).toBeDefined(); | |
| 98 | expect((await mcpTool(servers.url, token["bob"]!, "data")).value).toEqual([]); | |
| 99 | }); | |
| 100 | }); |
test/forge-looks.test.ts +1
| @@ -21,6 +21,7 @@ quietLogs(); | ||
| 21 | 21 | /** Top-level routes that are not pages, or are not worth a picture, and why. */ |
| 22 | 22 | const NOT_A_PAGE: Record<string, string> = { |
| 23 | 23 | "/deploy-to": "a form for a forge admin that fills from its link; covered by test/pinned-page.test.ts", |
| 24 | "/data": "no picture: the cap of 24 looks is spent; a guest sees the not-found page and the page is covered by its own tests", | |
| 24 | 25 | "/asked": "no picture: the cap of 24 looks went to /notes, the page that now leads with a person's thoughts; /asked is covered by test/asked.test.ts", |
| 25 | 26 | "/forges": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests", |
| 26 | 27 | "/family": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests", |
test/mcp.test.ts +1
| @@ -210,6 +210,7 @@ describe("the forge over MCP", () => { | ||
| 210 | 210 | const names = listed.body.result.tools.map((t: { name: string }) => t.name); |
| 211 | 211 | // Listed by name, one file each (#5924), and `report` last. |
| 212 | 212 | expect(names).toEqual(sortedThenReport([ |
| 213 | "data", | |
| 213 | 214 | "drills", |
| 214 | 215 | "sizes", |
| 215 | 216 | "releases", |
new above
offline