d059531 · 6afcebe · c369bb6 · b1b9739 · e56e483 · 736950e
@@ -1,5 +1,6 @@
node_modules
data/
!src/data/
dist/
*.local
.DS_Store
@@ -58,6 +58,7 @@
Machine routes are not pages and have no entry: `/audit.csv`,
- `/feed` — `studio/home.ts` · the composer and one timeline, what home was before it became the living pipeline (#8015) · `/` · none seen.
- `/boards/data` — `pages/boards.ts` · the boards' query/data surface · `/board` · none seen.
- `/asks` — `studio/asks_page.ts` · what the crew is waiting on me for · `/` "needs me" list · colours 2.
- `/data` — `pages/data.ts` · the SQLite files I may see, their tables and rows, secrets masked · `/system` · none seen.
- `/asked` — `studio/asked_page.ts` · what I answered, as a record · `/asks` · none seen.
- `/review` — `studio/review_page.ts` · every ask waiting on me, one row each, newest first · `/asks` (same asks, with buttons) · none seen.
- `/flow` — `pages/flow.ts` · the pipeline from ready issue to live deploy, and the bottleneck · `/system`, `/board`, `/` · none seen. A topbar button points here.
@@ -0,0 +1,17 @@
# Data browser (core)
`src/data/` reads SQLite files a viewer may see. Reach it at `/data` (the Data button on Apps, for an admin or an app's writer), in the terminal as the palette command Data, or through the MCP tool `data`. An admin or a writer of an app gets the page (an empty state until a database exists); a guest or a reader gets a 404, no palette entry, and nothing from the tool.
- `listSources(db, dataDir, viewer)` — `forge.db` for admins; for each app the viewer can write, every `.db` under each environment's data dir. `findSource` resolves an id only through that list, so a reader gets `undefined` (a 404), and no input becomes a path.
- `tables(source)`, `rows(source, table, {filter, order, page})` — 50 rows a page; table and column names are checked against the schema.
- `query(source, sql, timeoutMs?)` — one `SELECT`/`WITH`, read-only connection (`query_only`), 1,000 rows, 5 s. It runs in a worker that is terminated at the cap, so even a query that never yields stops.
- Masking (`mask.ts`): columns named like token, secret, password, hash, sealed, key, cookie, … show `••••••`. Queries read through temporary views that shadow each table, so aliases and subqueries are masked too; `main.` qualifiers are refused. Masked columns cannot be searched or ordered. `ALLOW` lists `table.column` exceptions per source id.
- Connections open per call and close after; they never take a writer lock.
Views are masked whole when their text mentions a masked column (an alias cannot launder a secret). `FORCE` masks generic columns by source (`forge_setting.value`, …); `test/data-browser.test.ts` audits every `forge.db` column against the name rule and lists the ones reviewed as clear.
Limits on work: `query` runs at most 2 workers at once (a third is told it is busy), each with a 128 MB heap cap (`hard_heap_limit`), a 5 s timer, 1,000 rows and about 250 KB of result; every cell is cut to 1,000 characters inside SQLite and a blob shows as its size. `rows()` cuts cells the same way.
      
        sentinelcrew
        2d ago
        outdated
      
      'Not done: a terminal screen' is a miss against 'both doors'. Add the src/tui/ screen, its entry, and a test, then drop this line.
    
Terminal: `src/tui/screens/data.ts` (files → tables → rows, `n`/`p` for pages), offered by `maySeeData`.
Not done: a SQL editor on the page or in the terminal (SQL is MCP only).
@@ -89,6 +89,7 @@
export const SYSTEM_NAMES: readonly string[] = [
  "mcp",
  "metrics",
  "contract",
  "data",
  "migrate",
  "mode",
  "new",
@@ -0,0 +1,142 @@
/**
 * Tables and rows of one source, read on a connection opened for the call
 * and closed after it. Read-only: it takes no writer lock and a busy WAL
 * answers in milliseconds or not at all.
 */
import { Database } from "bun:sqlite";
import { cell, maskPlan, MASK } from "./mask.ts";
import type { Source } from "./sources.ts";
export const PAGE_SIZE = 50;
/** Deepest page served: a huge OFFSET is a slow scan. */
export const MAX_PAGE = 1000;
export interface TableInfo {
  readonly name: string;
  readonly rows: number;
}
export interface RowFilter {
  /** Rows where any column (or just `column`) contains `text`. */
  readonly text: string;
  readonly column?: string;
}
export interface RowsOptions {
  readonly filter?: RowFilter;
  readonly order?: { readonly column: string; readonly dir?: "asc" | "desc" };
  /** Zero-based. */
  readonly page?: number;
}
export interface RowsResult {
  readonly columns: readonly string[];
  readonly rows: readonly (readonly (string | number | null)[])[];
  readonly page: number;
  readonly total: number;
}
export function openReadOnly(path: string): Database {
  const db = new Database(path, { readonly: true });
  db.exec("PRAGMA busy_timeout = 250");
  db.exec("PRAGMA query_only = 1");
  return db;
}
const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
function withDb<T>(source: Source, fn: (db: Database) => T): T {
  const db = openReadOnly(source.path);
  try {
    return fn(db);
  } finally {
    db.close();
  }
}
const tableNames = (db: Database): string[] =>
  db
    .query<{ name: string }, []>(
      "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
    )
    .all()
    .map((r) => r.name);
export const tables = (source: Source): TableInfo[] =>
  withDb(source, (db) =>
    tableNames(db).map((name) => ({
      name,
      rows:
        db
          .query<{ n: number }, []>(`SELECT count(*) AS n FROM ${quote(name)}`)
          .get()?.n ?? 0,
    })),
  );
/** Rows of `table`, or `undefined` when the table does not exist. */
export function rows(
  source: Source,
  table: string,
  opts: RowsOptions = {},
): RowsResult | undefined {
  return withDb(source, (db) => {
    if (!tableNames(db).includes(table)) return undefined;
    const columns = db
      .query<{ name: string }, []>(`PRAGMA table_info(${quote(table)})`)
      .all()
      .map((c) => c.name);
    const masked = maskPlan(db, source.id).get(table) ?? new Set<string>();
    const where: string[] = [];
    const args: string[] = [];
    if (opts.filter && opts.filter.text !== "") {
      const col = opts.filter.column;
      if (col !== undefined && !columns.includes(col)) return undefined;
      // Searching a masked column would leak it a character at a time.
      const searchable = (col === undefined ? columns : [col]).filter(
        (c) => !masked.has(c),
      );
      const like = opts.filter.text.replace(/[\\%_]/g, (m) => `\\${m}`);
      where.push(
        searchable.length === 0
          ? "0"
          : searchable
              .map((c) => `CAST(${quote(c)} AS TEXT) LIKE ? ESCAPE '\\'`)
              .join(" OR "),
      );
      for (let i = 0; i < searchable.length; i++) args.push(`%${like}%`);
    }
    let order = "";
    if (opts.order) {
      if (!columns.includes(opts.order.column)) return undefined;
      if (masked.has(opts.order.column)) return undefined;
      order = ` ORDER BY ${quote(opts.order.column)} ${opts.order.dir === "desc" ? "DESC" : "ASC"}`;
    }
    const clause = where.length ? ` WHERE ${where.join(" AND ")}` : "";
    const wanted = Number(opts.page ?? 0);
    const page = Number.isFinite(wanted)
      ? Math.min(MAX_PAGE, Math.max(0, Math.floor(wanted)))
      : 0;
    const total =
      db
        .query<{ n: number }, string[]>(
          `SELECT count(*) AS n FROM ${quote(table)}${clause}`,
        )
        .get(...args)?.n ?? 0;
    const raw = db
      .query<unknown[], string[]>(
        `SELECT ${columns.map(quote).join(", ")} FROM ${quote(table)}${clause}${order} LIMIT ${PAGE_SIZE} OFFSET ${page * PAGE_SIZE}`,
      )
      .values(...args);
    return {
      columns,
      page,
      total,
      rows: raw.map((r) =>
        r.map((v, i) =>
          masked.has(columns[i]!) && v !== null ? MASK : cell(v),
        ),
      ),
    };
  });
}
@@ -0,0 +1,134 @@
import type { Database } from "bun:sqlite";
/**
 * Which columns hold secrets, by rule. A masked cell shows that a value is
 * there and nothing of it. A source may allow a column by name
 * (`table.column`) when its content is known to be harmless.
 */
export const MASK = "••••••";
const SECRET =
  /_enc$|verifier|nonce|challenge|(^|_)code(_hash)?$|token|secret|passw(or)?d|passhash|(^|_)hash$|^hash$|sealed|api_?key|(^|_)key$|^key$|private|credential|cookie|(^|_)salt$|(^|_)sig(nature)?$|^pw/i;
/** Per-source allowlist: source id → `table.column` entries shown in clear. */
export const ALLOW: Record<string, readonly string[]> = {};
/**
 * Per-source columns masked whatever their name: generic ones that hold
 * secrets in some rows (a setting's `value`, a stored response `body`).
 */
export const FORCE: Readonly<Record<string, readonly string[]>> = {
  forge: [
    "forge_setting.value",
    "forge_setting_change.from_value",
    "forge_setting_change.to_value",
    "front_setting.value",
    "user_setting.value",
    "origin_cursor.value",
    "sentinel_kept.value",
    "idempotency.body",
    "event.payload",
    "sentinel_finding.evidence",
  ],
};
export const isSecretColumn = (column: string): boolean => SECRET.test(column);
/** Whether `table.column` of a source shows only the mask. */
export function masker(sourceId: string): (table: string, column: string) => boolean {
  const allow = new Set(ALLOW[sourceId] ?? []);
  const force = new Set(FORCE[sourceId] ?? []);
  return (table, column) => {
    const key = `${table}.${column}`;
    if (allow.has(key)) return false;
    return force.has(key) || SECRET.test(column);
  };
}
const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
/**
 * The masked columns of every table and view in `db`. A view is judged by
 * its text, and masked whole when that text uses `*` (which can pull a
 * secret column without naming it, under any name), names any masked
 * column of the database, or names another view masked whole. So neither
 * an alias, a column list nor a view over a view can launder a secret.
 */
export function maskPlan(db: Database, sourceId: string): Map<string, Set<string>> {
  const masked = masker(sourceId);
  const plan = new Map<string, Set<string>>();
  const objects = db
    .query<{ name: string; type: string; sql: string | null }, []>(
      "SELECT name, type, sql FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'",
    )
    .all();
  const columnsOf = (name: string): string[] =>
    db
      .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
      .all()
      .map((c) => c.name);
  const secretNames = new Set<string>();
  const cols = new Map<string, string[]>();
  for (const o of objects) {
    const c = columnsOf(o.name);
    cols.set(o.name, c);
    if (o.type === "table")
      plan.set(o.name, new Set(c.filter((x) => masked(o.name, x))));
    for (const x of plan.get(o.name) ?? []) secretNames.add(x.toLowerCase());
      
        sentinelcrew
        2d ago
        outdated
      
      High: a view is masked only if its SQL text names a masked column, or if its own column name matches the rule. CREATE VIEW allusers AS SELECT * FROM users never names api_token in its text. A view with renamed columns, such as CREATE VIEW v(a,b) AS SELECT * FROM users, matches nothing either. For a FORCE column like forge_setting.value, a view column still named value is not in FORCE under the view name. Result: rows(src,"allusers") and SELECT * FROM allusers show the secret in clear. Fix: treat a view containing * as fully masked, or resolve each view column to its origin table.column and apply masker to that. Add a test.
    
  }
  const names = (text: string, name: string): boolean => {
    const n = name.toLowerCase();
    let at = text.indexOf(n);
    while (at >= 0) {
      const before = text[at - 1] ?? " ";
      const after = text[at + n.length] ?? " ";
      if (!/[a-z0-9_$]/.test(before) && !/[a-z0-9_$]/.test(after)) return true;
      at = text.indexOf(n, at + 1);
    }
    return false;
  };
  const views = objects.filter((o) => o.type === "view");
  const text = new Map(views.map((o) => [o.name, (o.sql ?? "").toLowerCase()]));
  // The part after the view's own name and column list: its SELECT.
  const body = (name: string): string => {
    const t = text.get(name)!;
    const as = t.search(/\bas\b/);
    return as >= 0 ? t.slice(as) : t;
  };
  const whole = new Set<string>();
  for (const o of views) {
    const b = body(o.name);
    if (b.includes("*") || [...secretNames].some((n) => names(b, n))) whole.add(o.name);
  }
  // A view over a view masked whole is masked whole, to any depth.
  for (let grew = true; grew; ) {
    grew = false;
    for (const o of views) {
      if (whole.has(o.name)) continue;
      if ([...whole].some((w) => names(body(o.name), w))) {
        whole.add(o.name);
        grew = true;
      }
    }
  }
  for (const o of views) {
    const c = cols.get(o.name)!;
    plan.set(o.name, new Set(whole.has(o.name) ? c : c.filter((x) => masked(o.name, x))));
  }
  return plan;
}
/** Longest text a cell shows; the rest is cut and marked. */
export const CELL_CAP = 1000;
/** A cell as plain JSON-safe data. */
export function cell(value: unknown): string | number | null {
  if (value === null || value === undefined) return null;
  if (typeof value === "bigint") return value.toString();
  if (typeof value === "number") return value;
  if (typeof value === "string")
    return value.length > CELL_CAP ? `${value.slice(0, CELL_CAP)}…` : value;
  if (value instanceof Uint8Array) return `<blob ${value.byteLength} bytes>`;
  return String(value);
}
@@ -0,0 +1,97 @@
/**
 * Read-only SQL against one source: `SELECT` or `WITH`, a read-only
 * connection, a time cap (the worker is ended, which stops even a query
 * that never yields) and a row cap.
 */
import { checkQuery } from "./sql.ts";
import type { Source } from "./sources.ts";
export const QUERY_TIMEOUT_MS = 5000;
export const QUERY_ROW_CAP = 1000;
/** Characters of result kept (a quarter megabyte). */
export const QUERY_BYTE_CAP = 250_000;
/** Memory a query may use. */
export const QUERY_HEAP = 128 * 1024 * 1024;
/** Queries running at once; one more is told the forge is busy. */
export const QUERY_WORKERS = 2;
let running = 0;
export type QueryResult =
  | {
      readonly ok: true;
      readonly columns: readonly string[];
      readonly rows: readonly (readonly (string | number | null)[])[];
      readonly truncated: boolean;
    }
  | {
      readonly ok: false;
      readonly error: "empty" | "not_select" | "multiple" | "schema" | "timeout" | "sql" | "busy";
      readonly message: string;
    };
const REFUSALS = {
  empty: "Write a query.",
  not_select: "Only SELECT and WITH statements are allowed.",
  multiple: "One statement at a time.",
  schema: "Name tables without a schema prefix.",
} as const;
/** Queries running now (for tests). */
export const queriesRunning = (): number => running;
export function query(
  source: Source,
  sql: string,
  timeoutMs = QUERY_TIMEOUT_MS,
): Promise<QueryResult> {
  const verdict = checkQuery(sql);
  if (verdict !== "ok")
    return Promise.resolve({ ok: false, error: verdict, message: REFUSALS[verdict] });
  if (running >= QUERY_WORKERS)
    return Promise.resolve({
      ok: false,
      error: "busy",
      message: "Busy with other queries: try again in a moment.",
    });
  running++;
  return new Promise((resolve) => {
    const worker = new Worker(new URL("./query_worker.ts", import.meta.url).href);
    let finished = false;
    const done = (r: QueryResult) => {
      if (finished) return;
      finished = true;
      clearTimeout(timer);
      worker.terminate();
      running--;
      resolve(r);
    };
    const timer = setTimeout(
      () =>
        done({
          ok: false,
          error: "timeout",
          message: `Stopped after ${timeoutMs / 1000} s.`,
        }),
      timeoutMs,
    );
    worker.onmessage = (e: MessageEvent) => {
      const m = e.data;
      done(
        m.ok
          ? { ok: true, columns: m.columns, rows: m.rows, truncated: m.truncated }
          : { ok: false, error: "sql", message: String(m.message) },
      );
    };
    worker.onerror = (e) =>
      done({ ok: false, error: "sql", message: e.message || "query failed" });
    worker.postMessage({
      path: source.path,
      sourceId: source.id,
      sql,
      limit: QUERY_ROW_CAP,
      bytes: QUERY_BYTE_CAP,
      heap: QUERY_HEAP,
    });
  });
}
@@ -0,0 +1,83 @@
/**
 * Runs one query off the main thread so that a slow one can be stopped by
 * ending the thread. Secret columns are shadowed by temporary views that
 * show the mask, so an alias or a subquery cannot read them.
 */
import { Database } from "bun:sqlite";
import { CELL_CAP, cell, maskPlan, MASK } from "./mask.ts";
declare const self: Worker;
interface Job {
  readonly path: string;
  readonly sourceId: string;
  readonly sql: string;
  readonly limit: number;
  /** Bytes of result text kept before the rest is dropped. */
  readonly bytes: number;
  /** Memory SQLite may hold for this query. */
  readonly heap: number;
}
const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
self.onmessage = (event: MessageEvent<Job>) => {
  const job = event.data;
  let db: Database | undefined;
  try {
    db = new Database(job.path, { readonly: true });
    db.exec("PRAGMA busy_timeout = 250");
    for (const [name, hide] of maskPlan(db, job.sourceId)) {
      if (hide.size === 0) continue;
      const cols = db
        .query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
        .all();
      const select = cols
        .map((c) =>
          hide.has(c.name)
            ? `CASE WHEN main.${quote(name)}.${quote(c.name)} IS NULL THEN NULL ELSE '${MASK}' END AS ${quote(c.name)}`
            : `main.${quote(name)}.${quote(c.name)} AS ${quote(c.name)}`,
        )
        .join(", ");
      db.exec(`CREATE TEMP VIEW ${quote(name)} AS SELECT ${select} FROM main.${quote(name)}`);
    }
    db.exec("PRAGMA query_only = 1");
    db.exec(`PRAGMA hard_heap_limit = ${Math.floor(job.heap)}`);
    const columns = db.query(job.sql).columnNames;
    // Cut every cell inside SQLite, so a giant blob or string never reaches
    // this thread whole.
    const names = columns.map((_, i) => `c${i}`);
    const shown = names
      .map(
        (n) =>
          `CASE typeof(${n}) WHEN 'blob' THEN '<blob ' || length(${n}) || ' bytes>' WHEN 'text' THEN substr(${n}, 1, ${CELL_CAP + 1}) ELSE ${n} END`,
      )
      .join(", ");
    const body = job.sql.trim().replace(/;+\s*$/, "");
    const stmt =
      columns.length === 0
        ? db.query(job.sql)
        : db.query(`WITH q(${names.join(", ")}) AS (\n${body}\n) SELECT ${shown} FROM q`);
    const out: (string | number | null)[][] = [];
    let truncated = false;
    let size = 0;
    for (const row of stmt.values() as unknown[][]) {
      const cells = row.map(cell);
      size += cells.reduce<number>((n, v) => n + (typeof v === "string" ? v.length : 8), 0);
      if (out.length >= job.limit || size > job.bytes) {
        truncated = true;
        break;
      }
      out.push(cells);
    }
    self.postMessage({ ok: true, columns, rows: out, truncated });
  } catch (error) {
    self.postMessage({
      ok: false,
      message: error instanceof Error ? error.message : String(error),
    });
  } finally {
    db?.close();
  }
};
@@ -0,0 +1,93 @@
/**
 * The SQLite files a viewer may see: the forge's own (admins), and each
 * app's per environment (people who can write the app's repository). An id
 * is only ever resolved by looking it up in this list, so no input becomes
 * a path.
 */
import type { Database } from "bun:sqlite";
import { existsSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { listApps } from "../core/apps.ts";
import { getRepoById, writableRepoIds } from "../core/repos.ts";
import type { User } from "../core/users.ts";
import { envDataDir, isEnvName } from "../apps/data.ts";
export interface Source {
  /** `forge`, or `app:<appId>:<env>:<file>`. */
  readonly id: string;
  readonly label: string;
  readonly kind: "forge" | "app";
  readonly path: string;
}
const DB_FILE = /^[\w.-]+\.db$/;
export function listSources(
  db: Database,
  dataDir: string,
  viewer: User | undefined,
): Source[] {
  const out: Source[] = [];
  if (viewer === undefined) return out;
  if (viewer.isAdmin) {
    const path = join(dataDir, "forge.db");
    if (existsSync(path))
      out.push({ id: "forge", label: "forge", kind: "forge", path });
  }
  const apps = listApps(db);
  const writable = writableRepoIds(
    db,
    viewer,
    apps.map((a) => a.repoId),
  );
  for (const app of apps) {
    if (!writable.has(app.repoId) || getRepoById(db, app.repoId) === undefined)
      continue;
    let envs: string[];
    try {
      envs = readdirSync(join(dataDir, "apps", app.id)).filter(isEnvName);
    } catch {
      continue;
    }
    for (const env of envs.sort()) {
      if (!isEnvName(env)) continue;
      const dir = envDataDir(dataDir, app.id, env);
      let files: string[];
      try {
        files = readdirSync(dir).filter((f) => DB_FILE.test(f));
      } catch {
        continue;
      }
      for (const file of files.sort())
        out.push({
          id: `app:${app.id}:${env}:${file}`,
          label: `${app.slug} · ${env} · ${file}`,
          kind: "app",
          path: join(dir, file),
        });
    }
  }
  return out;
}
/**
 * Whether the viewer has a door to the data browser at all: an admin, or a
 * writer of an app, whether or not any database file exists yet. A guest and
 * a reader have none (a 404 on the web, no entry in the terminal).
 */
export function maySeeData(db: Database, viewer: User | undefined): boolean {
  if (viewer === undefined) return false;
  if (viewer.isAdmin) return true;
  const ids = listApps(db).map((a) => a.repoId);
  return ids.length > 0 && writableRepoIds(db, viewer, ids).size > 0;
}
/** The source `id` names, if this viewer may see it; else `undefined` (a 404). */
export const findSource = (
  db: Database,
  dataDir: string,
  viewer: User | undefined,
  id: string,
): Source | undefined =>
  listSources(db, dataDir, viewer).find((s) => s.id === id);
@@ -0,0 +1,32 @@
/** What a read-only query may be: one SELECT or WITH statement, nothing else. */
/** The statement with comments and string literals blanked; identifiers stay. */
function skeleton(sql: string): string {
  let out = "";
  for (let i = 0; i < sql.length; i++) {
    const c = sql[i]!;
    if (c === "-" && sql[i + 1] === "-") {
      while (i < sql.length && sql[i] !== "\n") i++;
      out += " ";
    } else if (c === "/" && sql[i + 1] === "*") {
      const end = sql.indexOf("*/", i + 2);
      i = end < 0 ? sql.length : end + 1;
      out += " ";
    } else if (c === "'") {
      i++;
      while (i < sql.length && sql[i] !== "'") i++;
      out += "''";
    } else out += c;
  }
  return out;
}
export function checkQuery(sql: string): "ok" | "empty" | "not_select" | "multiple" | "schema" {
  const s = skeleton(sql).trim().replace(/;+\s*$/, "").trim();
  if (s === "") return "empty";
  if (s.includes(";")) return "multiple";
  if (!/^(select|with)\b/i.test(s)) return "not_select";
  // `main.t` would step round the masking views that shadow `t`.
  if (/(^|[^\w])("main"|\[main\]|`main`|main)\s*\./i.test(s)) return "schema";
  return "ok";
}
@@ -0,0 +1,39 @@
import { rows, tables } from "../../../data/browse.ts";
import { query } from "../../../data/query.ts";
import { findSource, listSources } from "../../../data/sources.ts";
import { bad, good, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "data",
  doc: "the SQLite files you may read (the forge's own for an admin; each app's per environment for its writers), with `source`: its tables and row counts, with `table` (and `page`): a page of rows, with `sql`: a read-only SELECT (5 s, 1,000 rows); secret columns are masked",
  description: "Read SQLite files you may see: tables, rows, read-only SQL; secrets masked.",
  inputSchema: {
    type: "object",
    properties: {
      source: { type: "string" },
      table: { type: "string" },
      page: { type: "integer" },
      sql: { type: "string" },
    },
  },
  writes: false,
  async run(ctx, user, args) {
    const { db, config } = ctx.app;
    const { source: id, table, page, sql } = args;
    for (const [k, v] of Object.entries({ source: id, table, sql }))
      if (v !== undefined && typeof v !== "string") return bad("invalid", `\`${k}\` is text`);
    if (page !== undefined && typeof page !== "number") return bad("invalid", "`page` is a number");
    if (id === undefined) {
      return good(listSources(db, config.dataDir, user).map((s) => ({ source: s.id, label: s.label })));
    }
    const source = findSource(db, config.dataDir, user, id as string);
    if (!source) return bad("not_found", "no such source");
    if (sql !== undefined) {
      const result = await query(source, sql as string);
      return result.ok ? good(result) : bad("invalid", result.message);
    }
    if (table === undefined) return good(tables(source));
    const found = rows(source, table as string, page === undefined ? {} : { page: page as number });
    return found ? good(found) : bad("not_found", "no such table");
  },
};
@@ -46,6 +46,7 @@
export { tool as confirm_wait } from "./confirm_wait.ts";
export { tool as contribute } from "./contribute.ts";
export { tool as converge } from "./converge.ts";
export { tool as crew } from "./crew.ts";
export { tool as data } from "./data.ts";
export { tool as decisions_by } from "./decisions_by.ts";
export { tool as detach } from "./detach.ts";
export { tool as drills } from "./drills.ts";
@@ -27,6 +27,7 @@
import { prerenderComments } from "../web/prerender.ts";
import { systemPage, systemStream } from "../web/pages/system.ts";
import { contributePage } from "../web/pages/contribute.ts";
import { pipelinePage } from "../web/pages/pipeline.ts";
import { dataPage } from "../web/pages/data.ts";
import { fleetAct, fleetReceipt, fleetUndo } from "../web/pages/fleet-ops.ts";
import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts";
@@ -921,6 +922,7 @@
export function buildRouter(): Router<Handler> {
  pinnedRoutes(router);
  // Each THREATS.md protection against the last gate; literal, above /:owner.
  router.add("GET", "/resilience", resiliencePage);
  router.add("GET", "/data", dataPage);
  router.add("GET", "/maintenance", maintenancePage);
  router.add("POST", "/maintenance/prepare", maintenancePrepare);
  router.add("POST", "/maintenance/resume", maintenanceResume);
@@ -15,6 +15,7 @@
import { guestCodeSearch } from "../http/strangers.ts";
import { canRead, getRepo, type Repo } from "../core/repos.ts";
import { sourceFor } from "../view/source.ts";
import { fleetCommand } from "./screens/fleet.ts";
import { dataCommand } from "./screens/data.ts";
import { packsCommand } from "./screens/packs.ts";
import { App, type Ctx, type Screen } from "./app/app.ts";
import type { Command, PaletteItem, PaletteSearch } from "./app/palette.ts";
@@ -208,6 +209,7 @@
export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
        run: () => openOnWeb(session, ctx, "/explore", "The explorer"),
      },
      packsCommand(session, ctx),
      ...dataCommand(session, ctx),
      fleetCommand(session, ctx),
      {
        // A person's or an org's page is the web's (#636): the terminal
@@ -0,0 +1,124 @@
/**
 * `/data` in the terminal: the SQLite files the viewer may see, a file's
 * tables, a table's rows 50 a page, secrets masked. The same reads, and the
 * same gate (`maySeeData`, `listSources`), as the web page and MCP `data`.
 */
import { PAGE_SIZE, rows, tables } from "../../data/browse.ts";
import { listSources, maySeeData, type Source } from "../../data/sources.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { truncate } from "../term/width.ts";
/** A list of rows to move through; `open` is what Enter does on one. */
function listScreen(
  ctx: Ctx,
  title: string,
  intro: string,
  items: () => readonly string[],
  open: (index: number) => void,
  extra?: (key: string) => boolean,
  keys: ReturnType<typeof binding>[] = [],
): Screen {
  let at = 0;
  return {
    title,
    keys: [binding(["j", "k"], "move"), binding(["enter"], "open"), ...keys, binding(["esc"], "back")],
    render(area, s) {
      const c = ctx.theme.c;
      s.text(area.x + 1, area.y, truncate(intro, area.w - 2), style(c.mutedFg, c.bg), area.x + area.w);
      const list = items();
      if (list.length === 0) s.text(area.x + 1, area.y + 2, "Nothing here yet.", style(c.mutedFg, c.bg), area.x + area.w);
      const h = Math.max(1, area.h - 2);
      const from = Math.max(0, Math.min(at - h + 1, list.length - h));
      list.slice(from, from + h).forEach((line, i) => {
        const on = from + i === at;
        s.text(area.x + 1, area.y + 2 + i, truncate(`${on ? "> " : "  "}${line}`, area.w - 2), on ? style(c.accent, c.bg) : style(c.fg, c.bg), area.x + area.w);
      });
    },
    key(e) {
      if (e.ctrl || e.alt) return false;
      const n = items().length;
      if (e.key === "esc" || e.key === "q") ctx.pop();
      else if (e.key === "j" || e.key === "down") at = Math.min(Math.max(0, n - 1), at + 1);
      else if (e.key === "k" || e.key === "up") at = Math.max(0, at - 1);
      else if (e.key === "enter") {
        if (n > 0) open(at);
      } else return extra?.(e.key) ?? false;
      return true;
    },
  };
}
const cellText = (v: string | number | null): string => (v === null ? "" : String(v).replace(/[\r\n\t]+/g, " ").replace(/[\u0000-\u001f\u007f]/g, " "));
function rowsScreen(ctx: Ctx, source: Source, table: string): Screen {
  let page = 0;
  const read = () => rows(source, table, { page });
  return listScreen(
    ctx,
    `${table} · ${source.label}`,
    "",
    () => {
      const r = read();
      if (!r) return [];
      return [
        `${r.total} rows, page ${r.page + 1} · ${r.columns.join(" | ")}`,
        ...r.rows.map((row) => row.map(cellText).join(" | ")),
      ];
    },
    () => undefined,
    (key) => {
      const r = read();
      if (!r) return false;
      if (key === "n" && (r.page + 1) * PAGE_SIZE < r.total) page = r.page + 1;
      else if (key === "p" && r.page > 0) page = r.page - 1;
      else return false;
      return true;
    },
    [binding(["n", "p"], "next / previous page")],
  );
}
function tablesScreen(ctx: Ctx, source: Source): Screen {
  const list = tables(source);
  return listScreen(
    ctx,
    source.label,
    "Tables; secrets are masked.",
    () => list.map((t) => `${t.name}  ${t.rows} rows`),
    (i) => {
      const t = list[i];
      if (t) ctx.push(rowsScreen(ctx, source, t.name));
    },
  );
}
export function dataScreen(session: Session, ctx: Ctx): Screen {
  const sources = () => listSources(session.app.db, session.app.config.dataDir, session.user);
  return listScreen(
    ctx,
    "Data",
    "The SQLite files you may read.",
    () => sources().map((s) => s.label),
    (i) => {
      const s = sources()[i];
      if (s) ctx.push(tablesScreen(ctx, s));
    },
  );
}
/** Offered to an admin and to an app's writers; a reader and a guest see nothing. */
export const dataCommand = (session: Session, ctx: Ctx) =>
  maySeeData(session.app.db, session.user)
    ? [
        {
          id: "data",
          title: "Data",
          hint: "the SQLite files you may read, secrets masked",
          run: () => ctx.push(dataScreen(session, ctx)),
        },
      ]
    : [];
@@ -236,8 +236,16 @@
export async function appsList(ctx: Ctx): Promise<Response> {
    costLine(ctx.app.db, writableAppsOf(ctx, ctx.auth.user).repos),
  );
  const rendering = performance.now();
  // The door to /data: an admin, or someone who writes an app (already counted
  // above); no per-app work here, /data itself lists what exists.
  const hasData =
    ctx.auth.user !== undefined &&
    (ctx.auth.user.isAdmin || writableAppsOf(ctx, ctx.auth.user).repos.length > 0);
  const body = html`<div data-apps-page>${when(ctx.auth.user, (user) => homeTabs("apps", unshippedAsks(ctx.app.db, user)))}
    ${pageHead("Apps", { actions: create, sub: APPS_PURPOSE })}
    ${pageHead("Apps", {
      actions: hasData ? html`${btn("Data", { href: "/data", size: "sm" })}${create}` : create,
      
        sentinelcrew
        2d ago
        outdated
      
      Hand-rolled <p class="muted small mb"><a> outside the design components (principles 3 and 7). Put the Data door in the page head actions or the existing tab/nav pattern, and make /data useful for a writer with no database yet (see body).
    
      sub: APPS_PURPOSE,
    })}
    ${cost}
    ${filters}
    ${archivedView
@@ -0,0 +1,87 @@
/**
 * /data: the SQLite files the viewer may see (`data/sources.ts`), the
 * tables in one, and a page of a table's rows, secrets masked. Anyone with
 * no door to the data (a guest, a reader) gets the not-found page; an admin
 * or an app's writer gets the page, with an empty state until a file exists.
 */
import { PAGE_SIZE, rows, tables } from "../../data/browse.ts";
import { findSource, listSources, maySeeData } from "../../data/sources.ts";
import type { Ctx } from "../../http/context.ts";
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { notFoundPage, pageHead, signInFirst } from "../chrome.ts";
import { html } from "../html.ts";
import { page } from "../layout.ts";
import { empty, pager, row } from "../ui.ts";
const PURPOSE = "The SQLite files you may read, their tables and rows; secrets are masked.";
const href = (params: Record<string, string | number>): string =>
  `/data?${new URLSearchParams(Object.entries(params).map(([k, v]): [string, string] => [k, String(v)]))}`;
// GET /data[?source=<id>[&table=<name>[&page=<n>]]]
export function dataPage(ctx: Ctx): Response {
  notePhase(ctx.request, "session", takeSessionTime(ctx.request));
  const viewer = ctx.auth.user;
  if (!viewer) return signInFirst(ctx);
  const { db, config } = ctx.app;
  if (!maySeeData(db, viewer)) return notFoundPage(ctx);
  const q = ctx.url.searchParams;
  const all = listSources(db, config.dataDir, viewer);
  const chosen = q.get("source");
  const source = chosen === null ? undefined : findSource(db, config.dataDir, viewer, chosen);
      
        sentinelcrew
        2d ago
        outdated
      
      Principle 1 and 8: hand-rolled page-head with a bare 'Data' and no sentence saying what the page is for. Use pageHead("Data", { sub: ... }) as /apps does, with one plain sentence.
    
  if (chosen !== null && !source) return notFoundPage(ctx);
  let body = html`${pageHead("Data", { sub: PURPOSE })}
    ${all.length === 0
      ? empty({
          title: "No database yet",
          note: "An app's SQLite files appear here once it has written one under its data directory.",
        })
      : html`<ul class="list">
          ${all.map((s) => row({ href: href({ source: s.id }), title: s.label, current: s.id === source?.id }))}
        </ul>`}`;
  if (source) {
    const list = tables(source);
    const name = q.get("table");
    const shown = name === null ? undefined : rows(source, name, { page: Number(q.get("page") ?? 0) });
    if (name !== null && !shown) return notFoundPage(ctx);
    body = html`${body}
      <h2>${source.label}</h2>
      <ul class="list">
        ${list.map((t) =>
          row({
            href: href({ source: source.id, table: t.name }),
            title: t.name,
            trail: html`${String(t.rows)} rows`,
            current: t.name === name,
          }),
        )}
      </ul>`;
    if (shown && name !== null) {
      const more = (shown.page + 1) * PAGE_SIZE < shown.total;
      body = html`${body}
        <h2>${name}</h2>
        <div class="table-scroll">
          <table class="table" data-dense>
            <caption class="sr-only">${name} in ${source.label}</caption>
            <thead><tr>${shown.columns.map((c) => html`<th scope="col">${c}</th>`)}</tr></thead>
      
        sentinelcrew
        2d ago
        outdated
      
      Principles 2, 3 and 6/7: rows are wrapped in a card and the table is raw markup. Use the named table/row component from docs/DESIGN.md without a box, and a dl/rows for the source/table lists. Wide tables must scroll inside their own container so the phone layout does not shift.
    
            <tbody>
              ${shown.rows.map((r) => html`<tr>${r.map((v) => html`<td>${v === null ? "" : String(v)}</td>`)}</tr>`)}
            </tbody>
          </table>
        </div>
        ${pager({
          note: `${shown.total} rows`,
          words: { prev: "Previous page", next: "Next page" },
          ...(shown.page > 0 ? { prev: href({ source: source.id, table: name, page: shown.page - 1 }) } : {}),
          ...(more ? { next: href({ source: source.id, table: name, page: shown.page + 1 }) } : {}),
        })}`;
      
        sentinelcrew
        2d ago
        outdated
      
      Wrong word: page-1 is the previous page, not 'Newer page' (the data is not ordered by time). Say 'Previous page'. Also a bare <p class=muted> with inline links: use the pager the design system names, if there is one.
    
    }
  }
  const rendering = performance.now();
  const response = page({ ctx, title: "Data", body });
  notePhase(ctx.request, "render", performance.now() - rendering);
  return response;
}
@@ -1099,15 +1099,17 @@
export function pager(o: {
  prev?: string;
  next?: string;
  note?: Html | string;
  /** Words for the two buttons when the list is not in time order. */
  words?: { prev: string; next: string };
}): Html {
  if (!o.prev && !o.next && !o.note) return html``;
  return html`<nav class="pager" aria-label="Pagination">
    ${o.prev
      ? btn("Newer", { href: o.prev, size: "sm", variant: "outline" })
      ? btn(o.words?.prev ?? "Newer", { href: o.prev, size: "sm", variant: "outline" })
      : null}
    ${o.note ? html`<span>${o.note}</span>` : null}
    ${o.next
      ? btn("Older", { href: o.next, size: "sm", variant: "outline" })
      ? btn(o.words?.next ?? "Older", { href: o.next, size: "sm", variant: "outline" })
      : null}
  </nav>`;
}
@@ -0,0 +1,314 @@
/**
 * The data browser's core: who sees which SQLite file, what a query may be,
 * and what is masked.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { mkdirSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp as createAppRecord } from "../src/core/apps.ts";
import { createRepo } from "../src/core/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { envDataDir } from "../src/apps/data.ts";
import { createUser, type User } from "../src/core/users.ts";
import { rows, tables } from "../src/data/browse.ts";
import { ALLOW, maskPlan } from "../src/data/mask.ts";
import { query } from "../src/data/query.ts";
import { findSource, listSources } from "../src/data/sources.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
const AUDITED_CLEAR: string[] = [
  "app_domain.said",
  "app_feature_setting.value",
  "comment.body",
  "degraded_span.sentence",
  "feed_chip.value",
  "fleet_operation.value",
  "issue.body",
  "oauth_flow.detail",
  "oidc_key.public_jwk",
  "probe.said",
  "prompt_source.last_said",
  "repo_upstream_check.detail",
  "report.body",
  "review.body",
  "run.check_detail",
  "subscription.resting_said",
  "subscription_limit.said",
  "user_key.blob",
  "webhook_ping.payload",
  "week_projection.body",
  "week_retro.body",
];
describe("data browser core", () => {
  let dataDir: string;
  let app: App;
  let admin: User;
  let alice: User;
  let bob: User;
  let appId = "";
  const user = async (username: string, adminIfFirst = false) => {
    const made = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username,
      email: `${username}@forge.local`,
      password: "a-good-passphrase",
      adminIfFirst,
    });
    if (!made.ok) throw new Error(made.error);
    return made.value;
  };
  const ids = (viewer: User | undefined) =>
    listSources(app.db, dataDir, viewer).map((s) => s.id);
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-data-browser-"));
    app = createApp({ dataDir, port: 0, host: "forge.test" });
    admin = await user("owner1", true);
    alice = await user("alice");
    bob = await user("bob");
    const repo = await createRepo(app.db, app.reposRoot, {
      ownerId: alice.id,
      name: "notes",
      visibility: "public",
    });
    if (!repo.ok) throw new Error(repo.error);
    const made = createAppRecord(app.db, {
      repoId: repo.value.id,
      slug: "notes",
      access: "public",
    });
    if (!made.ok) throw new Error(made.error);
    appId = made.value.id;
    for (const env of ["production", "pull-3"] as const) {
      mkdirSync(envDataDir(dataDir, appId, env), { recursive: true });
      const d = new Database(join(envDataDir(dataDir, appId, env), "app.db"));
      d.exec("PRAGMA journal_mode = WAL");
      d.exec(
        "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT, password_hash TEXT)",
      );
      d.exec("CREATE VIEW people AS SELECT name, api_token AS harmless FROM users");
      d.exec("CREATE VIEW allusers AS SELECT * FROM users");
      d.exec("CREATE VIEW renamed(a, b, c, d) AS SELECT * FROM users");
      d.exec("CREATE VIEW deeper AS SELECT c FROM renamed");
      d.exec("CREATE VIEW names AS SELECT id, name FROM users");
      d.exec(
        `INSERT INTO users VALUES (1, 'ann', 'tok-ann-1', 'hash-1'), (2, 'ben', 'tok-ben-2', NULL)`,
      );
      d.close();
    }
  });
  afterAll(async () => {
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("a writer sees their app's production and preview databases", () => {
    expect(ids(alice)).toEqual([
      `app:${appId}:production:app.db`,
      `app:${appId}:pull-3:app.db`,
    ]);
  });
  test("a reader of the app can't find its data", () => {
    expect(ids(bob)).toEqual([]);
    expect(ids(undefined)).toEqual([]);
    expect(
      findSource(app.db, dataDir, bob, `app:${appId}:production:app.db`),
    ).toBeUndefined();
  });
  test("only an admin sees forge.db", () => {
    expect(ids(admin)).toContain("forge");
    expect(ids(alice)).not.toContain("forge");
    expect(findSource(app.db, dataDir, alice, "forge")).toBeUndefined();
  });
  test("tables with counts, rows with filter, order and page", () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    expect(tables(src)).toEqual([{ name: "users", rows: 2 }]);
    const r = rows(src, "users", { filter: { text: "b" }, order: { column: "id", dir: "desc" } })!;
    expect(r.total).toBe(1);
    expect(r.rows[0]![1]).toBe("ben");
    expect(rows(src, "nope")).toBeUndefined();
    expect(rows(src, "users", { order: { column: "id; DROP TABLE users" } })).toBeUndefined();
  });
  test("a token column is masked, and cannot be searched or ordered", () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    const r = rows(src, "users")!;
    expect(JSON.stringify(r)).not.toContain("tok-ann");
    expect(JSON.stringify(r)).not.toContain("hash-1");
    expect(r.rows[1]![3]).toBeNull();
    expect(rows(src, "users", { filter: { text: "tok-ann", column: "api_token" } })!.total).toBe(0);
    expect(rows(src, "users", { order: { column: "api_token" } })).toBeUndefined();
  });
  test("a query cannot read a secret by alias or subquery either", async () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    for (const sql of [
      "SELECT api_token AS t FROM users",
      "SELECT (SELECT api_token FROM users LIMIT 1) AS t",
      "SELECT * FROM users",
    ]) {
      const r = await query(src, sql);
      expect(r.ok).toBe(true);
      expect(JSON.stringify(r)).not.toContain("tok-ann");
    }
    const bypass = await query(src, "SELECT api_token FROM main.users");
    expect(bypass.ok).toBe(false);
    const plain = await query(src, "SELECT name FROM users ORDER BY id");
    expect(plain.ok && plain.rows).toEqual([["ann"], ["ben"]]);
  });
  test("INSERT and PRAGMA writable_schema are refused", async () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    for (const sql of [
      "INSERT INTO users (name) VALUES ('x')",
      "PRAGMA writable_schema = 1",
      "SELECT 1; DELETE FROM users",
      "WITH x AS (SELECT 1) DELETE FROM users",
      "-- hi\nUPDATE users SET name = 'x'",
    ]) {
      const r = await query(src, sql);
      expect(r.ok).toBe(false);
    }
    expect(rows(src, "users")!.total).toBe(2);
  });
  test("a result stops at 1,000 rows", async () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    const r = await query(
      src,
      "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 5000) SELECT i FROM n",
    );
    expect(r.ok && r.rows.length).toBe(1000);
    expect(r.ok && r.truncated).toBe(true);
  });
  test("a slow query stops at the cap", async () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    const started = performance.now();
    const r = await query(
      src,
      "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n) SELECT count(*) FROM n",
      400,
    );
    expect(r.ok).toBe(false);
    expect(!r.ok && r.error).toBe("timeout");
    expect(performance.now() - started).toBeLessThan(3000);
  });
  test("reading never blocks the app's own writes", () => {
    const src = findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
    const w = new Database(src.path);
    tables(src);
    w.exec("INSERT INTO users (name) VALUES ('cy')");
    w.close();
    expect(rows(src, "users")!.total).toBe(3);
  });
  const prod = () => findSource(app.db, dataDir, alice, `app:${appId}:production:app.db`)!;
  test("a schema qualifier in any quoting is refused", async () => {
    for (const from of [
      '"main"."users"',
      "[main].[users]",
      "`main`.`users`",
      'main /**/ .users',
      '"MAIN" . users',
      "main\n.users",
    ]) {
      const r = await query(prod(), `SELECT api_token FROM ${from}`);
      expect(r.ok).toBe(false);
      expect(JSON.stringify(r)).not.toContain("tok-ann");
    }
  });
  test("a view that aliases a secret is masked, in rows and in queries", async () => {
    const src = prod();
    expect(JSON.stringify(rows(src, "people") ?? "")).not.toContain("tok-ann");
    const r = await query(src, "SELECT harmless FROM people");
    expect(r.ok).toBe(true);
    expect(JSON.stringify(r)).not.toContain("tok-ann");
    expect(JSON.stringify(r)).toContain("••••");
  });
  test("a view by star, by a column list, or over such a view is masked whole", async () => {
    const src = prod();
    for (const [view, col] of [["allusers", "api_token"], ["renamed", "c"], ["deeper", "c"]] as const) {
      expect(JSON.stringify(rows(src, view) ?? "")).not.toContain("tok-ann");
      for (const sql of [`SELECT * FROM ${view}`, `SELECT ${col} AS x FROM ${view}`]) {
        const r = await query(src, sql);
        expect(r.ok).toBe(true);
        expect(JSON.stringify(r)).not.toContain("tok-ann");
        expect(JSON.stringify(r)).toContain("••••");
      }
    }
    // A view that names no secret and uses no star stays in clear.
    const clear = await query(src, "SELECT name FROM names");
    expect(JSON.stringify(clear)).toContain("ann");
  });
  test("an allowlisted column is shown in clear, and only for its source", () => {
    const src = prod();
    ALLOW[src.id] = ["users.api_token"];
    try {
      expect(JSON.stringify(rows(src, "users"))).toContain("tok-ann-1");
      expect(JSON.stringify(rows(src, "users"))).not.toContain("hash-1");
      const other = findSource(app.db, dataDir, alice, `app:${appId}:pull-3:app.db`)!;
      expect(JSON.stringify(rows(other, "users"))).not.toContain("tok-ann");
    } finally {
      delete ALLOW[src.id];
    }
  });
  test("a filter on a column that does not exist finds nothing", () => {
    expect(rows(prod(), "users", { filter: { text: "a", column: "nope" } })).toBeUndefined();
  });
  test("a guest or a non-writer cannot find forge", () => {
    for (const who of [undefined, alice, bob]) expect(findSource(app.db, dataDir, who, "forge")).toBeUndefined();
  });
  test("a page that is not a finite number is the first page; a huge one is capped", () => {
    const src = prod();
    for (const page of [Number.NaN, Number.POSITIVE_INFINITY, -4])
      expect(rows(src, "users", { page })!.page).toBe(0);
    expect(rows(src, "users", { page: 1e12 })!.page).toBe(1000);
  });
  test("forge.db is masked for an admin, setting values included", async () => {
    app.db.exec("INSERT OR REPLACE INTO forge_setting (key, value, updated_by, updated_at) VALUES ('smtp', 'SETTING-SECRET', 'x', 'now')");
    const src = findSource(app.db, dataDir, admin, "forge")!;
    expect(JSON.stringify(rows(src, "forge_setting"))).not.toContain("SETTING-SECRET");
    const r = await query(src, "SELECT value FROM forge_setting");
    expect(JSON.stringify(r)).not.toContain("SETTING-SECRET");
    const users = JSON.stringify(rows(src, "user"));
    expect(users).not.toContain("$argon2");
  });
  test("every forge.db column that could hold a secret is masked or audited as clear", () => {
    const src = findSource(app.db, dataDir, admin, "forge")!;
    const plan = maskPlan(new Database(src.path, { readonly: true }), "forge");
    const loose = /value|body|payload|code|key|blob|nonce|verifier|secret|token|hash|salt|enc|jwk|evidence|said|detail/i;
    const open: string[] = [];
    for (const [table, masked] of plan) {
      const db = new Database(src.path, { readonly: true });
      for (const c of db.query<{ name: string }, []>(`PRAGMA table_info("${table}")`).all())
        if (loose.test(c.name) && !masked.has(c.name)) open.push(`${table}.${c.name}`);
      db.close();
    }
    // Reviewed: these hold public or operational text, never a credential.
    // A new column that matches and is not here fails until someone looks.
    expect(open.sort()).toEqual(AUDITED_CLEAR);
  });
});
@@ -0,0 +1,126 @@
/** /data and the `data` MCP tool: gated by listSources, so a guest or a reader sees nothing. */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { mkdirSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { envDataDir } from "../src/apps/data.ts";
import { createApp as createAppRecord } from "../src/core/apps.ts";
import { createRepo } from "../src/core/repos.ts";
import { createSession } from "../src/core/sessions.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { quietLogs } from "./helpers/globals.ts";
import { mcpTool } from "./helpers/mcp.ts";
quietLogs();
describe("the data door", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  const cookie: Record<string, string> = {};
  const token: Record<string, string> = {};
  let appId = "";
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-data-door-"));
    app = createApp({ dataDir, port: 0, host: "forge.test" });
    servers = await serve(app);
    const users: Record<string, string> = {};
    for (const name of ["owner1", "alice", "bob", "dana"]) {
      const made = await createUser(app.db, {
        origin: "seed", trusted: true, username: name,
        email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "owner1",
      });
      if (!made.ok) throw new Error(made.error);
      users[name] = made.value.id;
      cookie[name] = `forge_session=${createSession(app.db, made.value.id).token}`;
      token[name] = createToken(app.db, made.value.id, "t", "admin").token;
    }
    const repo = await createRepo(app.db, app.reposRoot, { ownerId: users["alice"]!, name: "notes", visibility: "public" });
    if (!repo.ok) throw new Error(repo.error);
    const rec = createAppRecord(app.db, { repoId: repo.value.id, slug: "notes", access: "public" });
    if (!rec.ok) throw new Error(rec.error);
    appId = rec.value.id;
    const bare = await createRepo(app.db, app.reposRoot, { ownerId: users["dana"]!, name: "bare", visibility: "public" });
    if (!bare.ok) throw new Error(bare.error);
    const bareApp = createAppRecord(app.db, { repoId: bare.value.id, slug: "bare", access: "public" });
    if (!bareApp.ok) throw new Error(bareApp.error);
    mkdirSync(envDataDir(dataDir, appId, "production"), { recursive: true });
    const d = new Database(join(envDataDir(dataDir, appId, "production"), "app.db"));
    d.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT)");
    d.exec("INSERT INTO users VALUES (1, 'ann', 'tok-ann-1')");
    d.exec(`CREATE TABLE hostile ("<img src=x onerror=alert(1)>" TEXT)`);
    d.exec(`INSERT INTO hostile VALUES ('<script>alert(1)</script>')`);
    d.close();
  }, 60_000);
  afterAll(async () => {
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  const get = (path: string, who?: string) =>
    fetch(`${servers.url}${path}`, { headers: who ? { cookie: cookie[who]! } : {}, redirect: "manual" });
  const src = () => encodeURIComponent(`app:${appId}:production:app.db`);
  test("a writer reaches sources, tables and masked rows on the page", async () => {
    const list = await (await get("/data", "alice")).text();
    expect(list).toContain("notes · production · app.db");
    const tables = await (await get(`/data?source=${src()}`, "alice")).text();
    expect(tables).toContain("users");
    const res = await get(`/data?source=${src()}&table=users`, "alice");
    const rows = await res.text();
    expect(rows).toContain("ann");
    expect(rows).not.toContain("tok-ann-1");
    expect(await (await get("/apps", "alice")).text()).toContain('href="/data"');
  });
  test("hostile column and cell text is escaped on the page", async () => {
    const html = await (await get(`/data?source=${src()}&table=hostile`, "alice")).text();
    expect(html).not.toContain("<script>alert(1)</script>");
    expect(html).not.toContain("<img src=x");
    expect(html).toContain("&lt;script&gt;alert(1)&lt;/script&gt;");
    expect(html).toContain("&lt;img src=x onerror=alert(1)&gt;");
  });
  test("a writer whose app has no database yet gets the page with an empty state, not a 404", async () => {
    const res = await get("/data", "dana");
    expect(res.status).toBe(200);
    expect(await res.text()).toContain("No database yet");
  });
  test("a guest is sent to sign in, never shown the page", async () => {
    const res = await get("/data");
    expect([302, 303, 401]).toContain(res.status);
    expect(await res.text()).not.toContain("No database yet");
  });
  test("a reader, a guest and a wrong source see nothing", async () => {
    expect((await get("/data", "bob")).status).toBe(404);
    expect((await get(`/data?source=${src()}`, "bob")).status).toBe(404);
    expect((await get("/data")).status).not.toBe(200);
    expect((await get("/data?source=forge", "alice")).status).toBe(404);
    expect((await get("/data?source=forge", "owner1")).status).toBe(200);
    expect(await (await get("/apps", "bob")).text()).not.toContain('href="/data"');
  });
  test("the MCP tool lists, reads, queries and refuses the same way", async () => {
    const id = `app:${appId}:production:app.db`;
    expect((await mcpTool(servers.url, token["alice"]!, "data")).value).toEqual([
      { source: id, label: "notes · production · app.db" },
    ]);
    expect((await mcpTool(servers.url, token["alice"]!, "data", { source: id })).value).toEqual([{ name: "hostile", rows: 1 }, { name: "users", rows: 1 }]);
    const q = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "SELECT api_token FROM users" });
    expect(JSON.stringify(q)).not.toContain("tok-ann-1");
    const w = await mcpTool(servers.url, token["alice"]!, "data", { source: id, sql: "DELETE FROM users" });
    expect(w.error).toBeDefined();
    expect((await mcpTool(servers.url, token["bob"]!, "data", { source: id })).error).toBeDefined();
    expect((await mcpTool(servers.url, token["bob"]!, "data")).value).toEqual([]);
  });
});
@@ -0,0 +1,67 @@
/** Work a query may cause is bounded: cells are cut, the result has a byte cap, workers are few. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rows } from "../src/data/browse.ts";
import { CELL_CAP } from "../src/data/mask.ts";
import { query, QUERY_WORKERS, queriesRunning } from "../src/data/query.ts";
import type { Source } from "../src/data/sources.ts";
let dir: string;
let src: Source;
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-data-limits-"));
  const path = join(dir, "x.db");
  const d = new Database(path);
  d.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, body TEXT)");
  d.query("INSERT INTO t (body) VALUES (?)").run("y".repeat(50_000));
  d.close();
  src = { id: "app:x:production:x.db", label: "x", kind: "app", path };
});
afterAll(() => rm(dir, { recursive: true, force: true }));
test("browsing a table cuts a long cell", () => {
  const cell = rows(src, "t")!.rows[0]![1] as string;
  expect(cell.length).toBe(CELL_CAP + 1);
  expect(cell.endsWith("…")).toBe(true);
});
test("a query cuts a long cell and shows a blob by its size", async () => {
  const r = await query(src, "SELECT body, zeroblob(100000) AS z FROM t");
  if (!r.ok) throw new Error(r.message);
  expect((r.rows[0]![0] as string).length).toBeLessThanOrEqual(CELL_CAP + 1);
  expect(r.rows[0]![1]).toBe("<blob 100000 bytes>");
});
test("a result stops at its byte cap", async () => {
  const r = await query(
    src,
    "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 900) SELECT i, (SELECT body FROM t) FROM n",
  );
  if (!r.ok) throw new Error(r.message);
  expect(r.truncated).toBe(true);
  expect(r.rows.length).toBeLessThan(900);
});
test("more queries than workers are told the forge is busy, and the slot frees", async () => {
  const slow = "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3000000) SELECT count(*) FROM n";
  const held = Array.from({ length: QUERY_WORKERS }, () => query(src, slow));
  expect(queriesRunning()).toBe(QUERY_WORKERS);
  const extra = await query(src, "SELECT 1");
  expect(extra.ok).toBe(false);
  if (!extra.ok) expect(extra.error).toBe("busy");
  await Promise.all(held);
  expect(queriesRunning()).toBe(0);
  expect((await query(src, "SELECT 1")).ok).toBe(true);
});
test("a query that would use huge memory is stopped, not served", async () => {
  const r = await query(
    src,
    "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 100) SELECT length(group_concat(hex(zeroblob(5000000)))) FROM n",
  );
  expect(r.ok).toBe(false);
}, 15_000);
@@ -22,6 +22,7 @@
quietLogs();
const NOT_A_PAGE: Record<string, string> = {
  "/audit": "no picture: its slot went to /pinned; /audit.csv is its download, covered by test/audit.test.ts",
  "/deploy-to": "a form for a forge admin that fills from its link; covered by test/pinned-page.test.ts",
  "/data": "no picture: the cap of 24 looks is spent; a guest sees the not-found page and the page is covered by its own tests",
  "/asked": "no picture: the cap of 24 looks went to /notes, the page that now leads with a person's thoughts; /asked is covered by test/asked.test.ts",
  "/forges": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests",
  "/family": "no picture: the cap of 24 looks went to the preview's /waves and /milestones; the page is covered by its own tests",
@@ -211,6 +211,7 @@
describe("the forge over MCP", () => {
    const names = listed.body.result.tools.map((t: { name: string }) => t.name);
    // Listed by name, one file each (#5924), and `report` last.
    expect(names).toEqual(sortedThenReport([
      "data",
      "drills",
      "sizes",
      "releases",
@@ -48,6 +48,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["home", "Home", "your repositories"],
    ["explore", "Explore", "on the web"],
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["data", "Data", "the SQLite files you may read, secrets masked"],
    ["fleet", "Fleet rows", "the rows a fleet query line selects"],
    ["profile", "Open a profile", "username or org"],
    ["repos", "Repositories", "the repositories you may write"],
@@ -163,6 +164,7 @@
test("a site-admin agent is offered the roster without the person-only entries",
    ["home", "Home", "your repositories"],
    ["explore", "Explore", "on the web"],
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["data", "Data", "the SQLite files you may read, secrets masked"],
    ["fleet", "Fleet rows", "the rows a fleet query line selects"],
    ["profile", "Open a profile", "username or org"],
    ["repos", "Repositories", "the repositories you may write"],
@@ -0,0 +1,94 @@
/** The terminal's Data screen: gated as the page is, reads the same masked rows. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { mkdirSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { envDataDir } from "../src/apps/data.ts";
import { createApp as createAppRecord } from "../src/core/apps.ts";
import { createRepo } from "../src/core/repos.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { forgeCommands } from "../src/tui/forge.ts";
import { createForgeApp } from "../src/tui/forge.ts";
import { dataScreen } from "../src/tui/screens/data.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { quietLogs } from "./helpers/globals.ts";
import { settleApp } from "./helpers/tui.ts";
quietLogs();
let dir: string;
let app: App;
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-tui-data-"));
  app = createApp({ dataDir: dir, port: 0 });
  const ids: Record<string, string> = {};
  for (const name of ["owner1", "alice", "bob"]) {
    const made = await createUser(app.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "owner1" });
    if (!made.ok) throw new Error(made.error);
    ids[name] = made.value.id;
  }
  const repo = await createRepo(app.db, app.reposRoot, { ownerId: ids["alice"]!, name: "notes", visibility: "public" });
  if (!repo.ok) throw new Error(repo.error);
  const rec = createAppRecord(app.db, { repoId: repo.value.id, slug: "notes", access: "public" });
  if (!rec.ok) throw new Error(rec.error);
  const where = envDataDir(dir, rec.value.id, "production");
  mkdirSync(where, { recursive: true });
  const d = new Database(join(where, "app.db"));
  d.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, api_token TEXT)");
  d.exec("INSERT INTO users VALUES (1, 'ann-visible', 'tok-secret-1')");
  d.close();
});
afterAll(async () => {
  app.close();
  await rm(dir, { recursive: true, force: true });
});
const open = (username?: string) =>
  createForgeApp(
    app,
    { sshUsername: "t", remoteAddress: "127.0.0.1", ...(username ? { user: getUserByName(app.db, username)! } : { fingerprint: "SHA256:t" }) },
    { terminal: headlessTerminal(120, 30), exitHooks: false },
  );
const offered = (username?: string) => {
  const { app: tui, session } = open(username);
  const has = forgeCommands(session)(tui.ctx).some((c) => c.id === "data");
  tui.stop?.();
  return has;
};
test("an admin and an app's writer are offered it; a reader and a guest are not", () => {
  expect(offered("owner1")).toBe(true);
  expect(offered("alice")).toBe(true);
  expect(offered("bob")).toBe(false);
  expect(offered()).toBe(false);
});
test("a writer walks from the files to the tables to masked rows", async () => {
  const { app: tui, session } = open("alice");
  tui.start();
  tui.ctx.push(dataScreen(session, tui.ctx));
  await settleApp(tui);
  expect(tui.screen.toText()).toContain("notes · production · app.db");
  tui.dispatch({ type: "key", key: "enter", ctrl: false, alt: false, shift: false });
  await settleApp(tui);
  expect(tui.screen.toText()).toContain("users  1 rows");
  tui.dispatch({ type: "key", key: "enter", ctrl: false, alt: false, shift: false });
  await settleApp(tui);
  const text = tui.screen.toText();
  expect(text).toContain("ann-visible");
  expect(text).not.toContain("tok-secret-1");
  tui.stop?.();
});
test("an admin sees forge.db among the files", async () => {
  const { app: tui, session } = open("owner1");
  tui.start();
  tui.ctx.push(dataScreen(session, tui.ctx));
  await settleApp(tui);
  expect(tui.screen.toText()).toContain("forge");
  tui.stop?.();
});
		@@ -1,5 +1,6 @@
1	1	node_modules
2	2	data/
	3	!src/data/
3	4	dist/
4	5	*.local
5	6	.DS_Store
		@@ -58,6 +58,7 @@ Machine routes are not pages and have no entry: `/audit.csv`,
58	58	- `/feed` — `studio/home.ts` · the composer and one timeline, what home was before it became the living pipeline (#8015) · `/` · none seen.
59	59	- `/boards/data` — `pages/boards.ts` · the boards' query/data surface · `/board` · none seen.
60	60	- `/asks` — `studio/asks_page.ts` · what the crew is waiting on me for · `/` "needs me" list · colours 2.
	61	- `/data` — `pages/data.ts` · the SQLite files I may see, their tables and rows, secrets masked · `/system` · none seen.
61	62	- `/asked` — `studio/asked_page.ts` · what I answered, as a record · `/asks` · none seen.
62	63	- `/review` — `studio/review_page.ts` · every ask waiting on me, one row each, newest first · `/asks` (same asks, with buttons) · none seen.
63	64	- `/flow` — `pages/flow.ts` · the pipeline from ready issue to live deploy, and the bottleneck · `/system`, `/board`, `/` · none seen. A topbar button points here.
		@@ -0,0 +1,17 @@
	1	# Data browser (core)
	2
	3	`src/data/` reads SQLite files a viewer may see. Reach it at `/data` (the Data button on Apps, for an admin or an app's writer), in the terminal as the palette command Data, or through the MCP tool `data`. An admin or a writer of an app gets the page (an empty state until a database exists); a guest or a reader gets a 404, no palette entry, and nothing from the tool.
	4
	5	- `listSources(db, dataDir, viewer)` — `forge.db` for admins; for each app the viewer can write, every `.db` under each environment's data dir. `findSource` resolves an id only through that list, so a reader gets `undefined` (a 404), and no input becomes a path.
	6	- `tables(source)`, `rows(source, table, {filter, order, page})` — 50 rows a page; table and column names are checked against the schema.
	7	- `query(source, sql, timeoutMs?)` — one `SELECT`/`WITH`, read-only connection (`query_only`), 1,000 rows, 5 s. It runs in a worker that is terminated at the cap, so even a query that never yields stops.
	8	- Masking (`mask.ts`): columns named like token, secret, password, hash, sealed, key, cookie, … show `••••••`. Queries read through temporary views that shadow each table, so aliases and subqueries are masked too; `main.` qualifiers are refused. Masked columns cannot be searched or ordered. `ALLOW` lists `table.column` exceptions per source id.
	9	- Connections open per call and close after; they never take a writer lock.
	10
	11	Views are masked whole when their text mentions a masked column (an alias cannot launder a secret). `FORCE` masks generic columns by source (`forge_setting.value`, …); `test/data-browser.test.ts` audits every `forge.db` column against the name rule and lists the ones reviewed as clear.
	12
	13	Limits on work: `query` runs at most 2 workers at once (a third is told it is busy), each with a 128 MB heap cap (`hard_heap_limit`), a 5 s timer, 1,000 rows and about 250 KB of result; every cell is cut to 1,000 characters inside SQLite and a blob shows as its size. `rows()` cuts cells the same way.
		sentinelcrew 2d ago outdated 'Not done: a terminal screen' is a miss against 'both doors'. Add the `src/tui/` screen, its entry, and a test, then drop this line.
	14
	15	Terminal: `src/tui/screens/data.ts` (files → tables → rows, `n`/`p` for pages), offered by `maySeeData`.
	16
	17	Not done: a SQL editor on the page or in the terminal (SQL is MCP only).
		@@ -89,6 +89,7 @@ export const SYSTEM_NAMES: readonly string[] = [
89	89	"mcp",
90	90	"metrics",
91	91	"contract",
	92	"data",
92	93	"migrate",
93	94	"mode",
94	95	"new",
		@@ -0,0 +1,142 @@
	1	/**
	2	* Tables and rows of one source, read on a connection opened for the call
	3	* and closed after it. Read-only: it takes no writer lock and a busy WAL
	4	* answers in milliseconds or not at all.
	5	*/
	6
	7	import { Database } from "bun:sqlite";
	8	import { cell, maskPlan, MASK } from "./mask.ts";
	9	import type { Source } from "./sources.ts";
	10
	11	export const PAGE_SIZE = 50;
	12	/** Deepest page served: a huge OFFSET is a slow scan. */
	13	export const MAX_PAGE = 1000;
	14
	15	export interface TableInfo {
	16	readonly name: string;
	17	readonly rows: number;
	18	}
	19
	20	export interface RowFilter {
	21	/** Rows where any column (or just `column`) contains `text`. */
	22	readonly text: string;
	23	readonly column?: string;
	24	}
	25
	26	export interface RowsOptions {
	27	readonly filter?: RowFilter;
	28	readonly order?: { readonly column: string; readonly dir?: "asc" \| "desc" };
	29	/** Zero-based. */
	30	readonly page?: number;
	31	}
	32
	33	export interface RowsResult {
	34	readonly columns: readonly string[];
	35	readonly rows: readonly (readonly (string \| number \| null)[])[];
	36	readonly page: number;
	37	readonly total: number;
	38	}
	39
	40	export function openReadOnly(path: string): Database {
	41	const db = new Database(path, { readonly: true });
	42	db.exec("PRAGMA busy_timeout = 250");
	43	db.exec("PRAGMA query_only = 1");
	44	return db;
	45	}
	46
	47	const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
	48
	49	function withDb<T>(source: Source, fn: (db: Database) => T): T {
	50	const db = openReadOnly(source.path);
	51	try {
	52	return fn(db);
	53	} finally {
	54	db.close();
	55	}
	56	}
	57
	58	const tableNames = (db: Database): string[] =>
	59	db
	60	.query<{ name: string }, []>(
	61	"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
	62	)
	63	.all()
	64	.map((r) => r.name);
	65
	66	export const tables = (source: Source): TableInfo[] =>
	67	withDb(source, (db) =>
	68	tableNames(db).map((name) => ({
	69	name,
	70	rows:
	71	db
	72	.query<{ n: number }, []>(`SELECT count(*) AS n FROM ${quote(name)}`)
	73	.get()?.n ?? 0,
	74	})),
	75	);
	76
	77	/** Rows of `table`, or `undefined` when the table does not exist. */
	78	export function rows(
	79	source: Source,
	80	table: string,
	81	opts: RowsOptions = {},
	82	): RowsResult \| undefined {
	83	return withDb(source, (db) => {
	84	if (!tableNames(db).includes(table)) return undefined;
	85	const columns = db
	86	.query<{ name: string }, []>(`PRAGMA table_info(${quote(table)})`)
	87	.all()
	88	.map((c) => c.name);
	89	const masked = maskPlan(db, source.id).get(table) ?? new Set<string>();
	90	const where: string[] = [];
	91	const args: string[] = [];
	92	if (opts.filter && opts.filter.text !== "") {
	93	const col = opts.filter.column;
	94	if (col !== undefined && !columns.includes(col)) return undefined;
	95	// Searching a masked column would leak it a character at a time.
	96	const searchable = (col === undefined ? columns : [col]).filter(
	97	(c) => !masked.has(c),
	98	);
	99	const like = opts.filter.text.replace(/[\\%_]/g, (m) => `\\${m}`);
	100	where.push(
	101	searchable.length === 0
	102	? "0"
	103	: searchable
	104	.map((c) => `CAST(${quote(c)} AS TEXT) LIKE ? ESCAPE '\\'`)
	105	.join(" OR "),
	106	);
	107	for (let i = 0; i < searchable.length; i++) args.push(`%${like}%`);
	108	}
	109	let order = "";
	110	if (opts.order) {
	111	if (!columns.includes(opts.order.column)) return undefined;
	112	if (masked.has(opts.order.column)) return undefined;
	113	order = ` ORDER BY ${quote(opts.order.column)} ${opts.order.dir === "desc" ? "DESC" : "ASC"}`;
	114	}
	115	const clause = where.length ? ` WHERE ${where.join(" AND ")}` : "";
	116	const wanted = Number(opts.page ?? 0);
	117	const page = Number.isFinite(wanted)
	118	? Math.min(MAX_PAGE, Math.max(0, Math.floor(wanted)))
	119	: 0;
	120	const total =
	121	db
	122	.query<{ n: number }, string[]>(
	123	`SELECT count(*) AS n FROM ${quote(table)}${clause}`,
	124	)
	125	.get(...args)?.n ?? 0;
	126	const raw = db
	127	.query<unknown[], string[]>(
	128	`SELECT ${columns.map(quote).join(", ")} FROM ${quote(table)}${clause}${order} LIMIT ${PAGE_SIZE} OFFSET ${page * PAGE_SIZE}`,
	129	)
	130	.values(...args);
	131	return {
	132	columns,
	133	page,
	134	total,
	135	rows: raw.map((r) =>
	136	r.map((v, i) =>
	137	masked.has(columns[i]!) && v !== null ? MASK : cell(v),
	138	),
	139	),
	140	};
	141	});
	142	}
		@@ -0,0 +1,134 @@
	1	import type { Database } from "bun:sqlite";
	2
	3	/**
	4	* Which columns hold secrets, by rule. A masked cell shows that a value is
	5	* there and nothing of it. A source may allow a column by name
	6	* (`table.column`) when its content is known to be harmless.
	7	*/
	8
	9	export const MASK = "••••••";
	10
	11	const SECRET =
	12	/_enc$\|verifier\|nonce\|challenge\|(^\|_)code(_hash)?$\|token\|secret\|passw(or)?d\|passhash\|(^\|_)hash$\|^hash$\|sealed\|api_?key\|(^\|_)key$\|^key$\|private\|credential\|cookie\|(^\|_)salt$\|(^\|_)sig(nature)?$\|^pw/i;
	13
	14	/** Per-source allowlist: source id → `table.column` entries shown in clear. */
	15	export const ALLOW: Record<string, readonly string[]> = {};
	16
	17	/**
	18	* Per-source columns masked whatever their name: generic ones that hold
	19	* secrets in some rows (a setting's `value`, a stored response `body`).
	20	*/
	21	export const FORCE: Readonly<Record<string, readonly string[]>> = {
	22	forge: [
	23	"forge_setting.value",
	24	"forge_setting_change.from_value",
	25	"forge_setting_change.to_value",
	26	"front_setting.value",
	27	"user_setting.value",
	28	"origin_cursor.value",
	29	"sentinel_kept.value",
	30	"idempotency.body",
	31	"event.payload",
	32	"sentinel_finding.evidence",
	33	],
	34	};
	35
	36	export const isSecretColumn = (column: string): boolean => SECRET.test(column);
	37
	38	/** Whether `table.column` of a source shows only the mask. */
	39	export function masker(sourceId: string): (table: string, column: string) => boolean {
	40	const allow = new Set(ALLOW[sourceId] ?? []);
	41	const force = new Set(FORCE[sourceId] ?? []);
	42	return (table, column) => {
	43	const key = `${table}.${column}`;
	44	if (allow.has(key)) return false;
	45	return force.has(key) \|\| SECRET.test(column);
	46	};
	47	}
	48
	49	const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
	50
	51	/**
	52	* The masked columns of every table and view in `db`. A view is judged by
	53	* its text, and masked whole when that text uses `*` (which can pull a
	54	* secret column without naming it, under any name), names any masked
	55	* column of the database, or names another view masked whole. So neither
	56	* an alias, a column list nor a view over a view can launder a secret.
	57	*/
	58	export function maskPlan(db: Database, sourceId: string): Map<string, Set<string>> {
	59	const masked = masker(sourceId);
	60	const plan = new Map<string, Set<string>>();
	61	const objects = db
	62	.query<{ name: string; type: string; sql: string \| null }, []>(
	63	"SELECT name, type, sql FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'",
	64	)
	65	.all();
	66	const columnsOf = (name: string): string[] =>
	67	db
	68	.query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
	69	.all()
	70	.map((c) => c.name);
	71	const secretNames = new Set<string>();
	72	const cols = new Map<string, string[]>();
	73	for (const o of objects) {
	74	const c = columnsOf(o.name);
	75	cols.set(o.name, c);
	76	if (o.type === "table")
	77	plan.set(o.name, new Set(c.filter((x) => masked(o.name, x))));
	78	for (const x of plan.get(o.name) ?? []) secretNames.add(x.toLowerCase());
		sentinelcrew 2d ago outdated High: a view is masked only if its SQL text names a masked column, or if its own column name matches the rule. `CREATE VIEW allusers AS SELECT * FROM users` never names `api_token` in its text. A view with renamed columns, such as `CREATE VIEW v(a,b) AS SELECT * FROM users`, matches nothing either. For a FORCE column like `forge_setting.value`, a view column still named `value` is not in FORCE under the view name. Result: `rows(src,"allusers")` and `SELECT * FROM allusers` show the secret in clear. Fix: treat a view containing `*` as fully masked, or resolve each view column to its origin table.column and apply masker to that. Add a test.
	79	}
	80	const names = (text: string, name: string): boolean => {
	81	const n = name.toLowerCase();
	82	let at = text.indexOf(n);
	83	while (at >= 0) {
	84	const before = text[at - 1] ?? " ";
	85	const after = text[at + n.length] ?? " ";
	86	if (!/[a-z0-9_$]/.test(before) && !/[a-z0-9_$]/.test(after)) return true;
	87	at = text.indexOf(n, at + 1);
	88	}
	89	return false;
	90	};
	91	const views = objects.filter((o) => o.type === "view");
	92	const text = new Map(views.map((o) => [o.name, (o.sql ?? "").toLowerCase()]));
	93	// The part after the view's own name and column list: its SELECT.
	94	const body = (name: string): string => {
	95	const t = text.get(name)!;
	96	const as = t.search(/\bas\b/);
	97	return as >= 0 ? t.slice(as) : t;
	98	};
	99	const whole = new Set<string>();
	100	for (const o of views) {
	101	const b = body(o.name);
	102	if (b.includes("*") \|\| [...secretNames].some((n) => names(b, n))) whole.add(o.name);
	103	}
	104	// A view over a view masked whole is masked whole, to any depth.
	105	for (let grew = true; grew; ) {
	106	grew = false;
	107	for (const o of views) {
	108	if (whole.has(o.name)) continue;
	109	if ([...whole].some((w) => names(body(o.name), w))) {
	110	whole.add(o.name);
	111	grew = true;
	112	}
	113	}
	114	}
	115	for (const o of views) {
	116	const c = cols.get(o.name)!;
	117	plan.set(o.name, new Set(whole.has(o.name) ? c : c.filter((x) => masked(o.name, x))));
	118	}
	119	return plan;
	120	}
	121
	122	/** Longest text a cell shows; the rest is cut and marked. */
	123	export const CELL_CAP = 1000;
	124
	125	/** A cell as plain JSON-safe data. */
	126	export function cell(value: unknown): string \| number \| null {
	127	if (value === null \|\| value === undefined) return null;
	128	if (typeof value === "bigint") return value.toString();
	129	if (typeof value === "number") return value;
	130	if (typeof value === "string")
	131	return value.length > CELL_CAP ? `${value.slice(0, CELL_CAP)}…` : value;
	132	if (value instanceof Uint8Array) return `<blob ${value.byteLength} bytes>`;
	133	return String(value);
	134	}
		@@ -0,0 +1,97 @@
	1	/**
	2	* Read-only SQL against one source: `SELECT` or `WITH`, a read-only
	3	* connection, a time cap (the worker is ended, which stops even a query
	4	* that never yields) and a row cap.
	5	*/
	6
	7	import { checkQuery } from "./sql.ts";
	8	import type { Source } from "./sources.ts";
	9
	10	export const QUERY_TIMEOUT_MS = 5000;
	11	export const QUERY_ROW_CAP = 1000;
	12	/** Characters of result kept (a quarter megabyte). */
	13	export const QUERY_BYTE_CAP = 250_000;
	14	/** Memory a query may use. */
	15	export const QUERY_HEAP = 128 * 1024 * 1024;
	16	/** Queries running at once; one more is told the forge is busy. */
	17	export const QUERY_WORKERS = 2;
	18	let running = 0;
	19
	20	export type QueryResult =
	21	\| {
	22	readonly ok: true;
	23	readonly columns: readonly string[];
	24	readonly rows: readonly (readonly (string \| number \| null)[])[];
	25	readonly truncated: boolean;
	26	}
	27	\| {
	28	readonly ok: false;
	29	readonly error: "empty" \| "not_select" \| "multiple" \| "schema" \| "timeout" \| "sql" \| "busy";
	30	readonly message: string;
	31	};
	32
	33	const REFUSALS = {
	34	empty: "Write a query.",
	35	not_select: "Only SELECT and WITH statements are allowed.",
	36	multiple: "One statement at a time.",
	37	schema: "Name tables without a schema prefix.",
	38	} as const;
	39
	40	/** Queries running now (for tests). */
	41	export const queriesRunning = (): number => running;
	42
	43	export function query(
	44	source: Source,
	45	sql: string,
	46	timeoutMs = QUERY_TIMEOUT_MS,
	47	): Promise<QueryResult> {
	48	const verdict = checkQuery(sql);
	49	if (verdict !== "ok")
	50	return Promise.resolve({ ok: false, error: verdict, message: REFUSALS[verdict] });
	51	if (running >= QUERY_WORKERS)
	52	return Promise.resolve({
	53	ok: false,
	54	error: "busy",
	55	message: "Busy with other queries: try again in a moment.",
	56	});
	57	running++;
	58	return new Promise((resolve) => {
	59	const worker = new Worker(new URL("./query_worker.ts", import.meta.url).href);
	60	let finished = false;
	61	const done = (r: QueryResult) => {
	62	if (finished) return;
	63	finished = true;
	64	clearTimeout(timer);
	65	worker.terminate();
	66	running--;
	67	resolve(r);
	68	};
	69	const timer = setTimeout(
	70	() =>
	71	done({
	72	ok: false,
	73	error: "timeout",
	74	message: `Stopped after ${timeoutMs / 1000} s.`,
	75	}),
	76	timeoutMs,
	77	);
	78	worker.onmessage = (e: MessageEvent) => {
	79	const m = e.data;
	80	done(
	81	m.ok
	82	? { ok: true, columns: m.columns, rows: m.rows, truncated: m.truncated }
	83	: { ok: false, error: "sql", message: String(m.message) },
	84	);
	85	};
	86	worker.onerror = (e) =>
	87	done({ ok: false, error: "sql", message: e.message \|\| "query failed" });
	88	worker.postMessage({
	89	path: source.path,
	90	sourceId: source.id,
	91	sql,
	92	limit: QUERY_ROW_CAP,
	93	bytes: QUERY_BYTE_CAP,
	94	heap: QUERY_HEAP,
	95	});
	96	});
	97	}
		@@ -0,0 +1,83 @@
	1	/**
	2	* Runs one query off the main thread so that a slow one can be stopped by
	3	* ending the thread. Secret columns are shadowed by temporary views that
	4	* show the mask, so an alias or a subquery cannot read them.
	5	*/
	6
	7	import { Database } from "bun:sqlite";
	8	import { CELL_CAP, cell, maskPlan, MASK } from "./mask.ts";
	9
	10	declare const self: Worker;
	11
	12	interface Job {
	13	readonly path: string;
	14	readonly sourceId: string;
	15	readonly sql: string;
	16	readonly limit: number;
	17	/** Bytes of result text kept before the rest is dropped. */
	18	readonly bytes: number;
	19	/** Memory SQLite may hold for this query. */
	20	readonly heap: number;
	21	}
	22
	23	const quote = (name: string): string => `"${name.replaceAll('"', '""')}"`;
	24
	25	self.onmessage = (event: MessageEvent<Job>) => {
	26	const job = event.data;
	27	let db: Database \| undefined;
	28	try {
	29	db = new Database(job.path, { readonly: true });
	30	db.exec("PRAGMA busy_timeout = 250");
	31	for (const [name, hide] of maskPlan(db, job.sourceId)) {
	32	if (hide.size === 0) continue;
	33	const cols = db
	34	.query<{ name: string }, []>(`PRAGMA table_info(${quote(name)})`)
	35	.all();
	36	const select = cols
	37	.map((c) =>
	38	hide.has(c.name)
	39	? `CASE WHEN main.${quote(name)}.${quote(c.name)} IS NULL THEN NULL ELSE '${MASK}' END AS ${quote(c.name)}`
	40	: `main.${quote(name)}.${quote(c.name)} AS ${quote(c.name)}`,
	41	)
	42	.join(", ");
	43	db.exec(`CREATE TEMP VIEW ${quote(name)} AS SELECT ${select} FROM main.${quote(name)}`);
	44	}
	45	db.exec("PRAGMA query_only = 1");
	46	db.exec(`PRAGMA hard_heap_limit = ${Math.floor(job.heap)}`);
	47	const columns = db.query(job.sql).columnNames;
	48	// Cut every cell inside SQLite, so a giant blob or string never reaches
	49	// this thread whole.
	50	const names = columns.map((_, i) => `c${i}`);
	51	const shown = names
	52	.map(
	53	(n) =>
	54	`CASE typeof(${n}) WHEN 'blob' THEN '<blob ' \|\| length(${n}) \|\| ' bytes>' WHEN 'text' THEN substr(${n}, 1, ${CELL_CAP + 1}) ELSE ${n} END`,
	55	)
	56	.join(", ");
	57	const body = job.sql.trim().replace(/;+\s*$/, "");
	58	const stmt =
	59	columns.length === 0
	60	? db.query(job.sql)
	61	: db.query(`WITH q(${names.join(", ")}) AS (\n${body}\n) SELECT ${shown} FROM q`);
	62	const out: (string \| number \| null)[][] = [];
	63	let truncated = false;
	64	let size = 0;
	65	for (const row of stmt.values() as unknown[][]) {
	66	const cells = row.map(cell);
	67	size += cells.reduce<number>((n, v) => n + (typeof v === "string" ? v.length : 8), 0);
	68	if (out.length >= job.limit \|\| size > job.bytes) {
	69	truncated = true;
	70	break;
	71	}
	72	out.push(cells);
	73	}
	74	self.postMessage({ ok: true, columns, rows: out, truncated });
	75	} catch (error) {
	76	self.postMessage({
	77	ok: false,
	78	message: error instanceof Error ? error.message : String(error),
	79	});
	80	} finally {
	81	db?.close();
	82	}
	83	};
		@@ -0,0 +1,93 @@
	1	/**
	2	* The SQLite files a viewer may see: the forge's own (admins), and each
	3	* app's per environment (people who can write the app's repository). An id
	4	* is only ever resolved by looking it up in this list, so no input becomes
	5	* a path.
	6	*/
	7
	8	import type { Database } from "bun:sqlite";
	9	import { existsSync, readdirSync } from "node:fs";
	10	import { join } from "node:path";
	11	import { listApps } from "../core/apps.ts";
	12	import { getRepoById, writableRepoIds } from "../core/repos.ts";
	13	import type { User } from "../core/users.ts";
	14	import { envDataDir, isEnvName } from "../apps/data.ts";
	15
	16	export interface Source {
	17	/** `forge`, or `app:<appId>:<env>:<file>`. */
	18	readonly id: string;
	19	readonly label: string;
	20	readonly kind: "forge" \| "app";
	21	readonly path: string;
	22	}
	23
	24	const DB_FILE = /^[\w.-]+\.db$/;
	25
	26	export function listSources(
	27	db: Database,
	28	dataDir: string,
	29	viewer: User \| undefined,
	30	): Source[] {
	31	const out: Source[] = [];
	32	if (viewer === undefined) return out;
	33	if (viewer.isAdmin) {
	34	const path = join(dataDir, "forge.db");
	35	if (existsSync(path))
	36	out.push({ id: "forge", label: "forge", kind: "forge", path });
	37	}
	38	const apps = listApps(db);
	39	const writable = writableRepoIds(
	40	db,
	41	viewer,
	42	apps.map((a) => a.repoId),
	43	);
	44	for (const app of apps) {
	45	if (!writable.has(app.repoId) \|\| getRepoById(db, app.repoId) === undefined)
	46	continue;
	47	let envs: string[];
	48	try {
	49	envs = readdirSync(join(dataDir, "apps", app.id)).filter(isEnvName);
	50	} catch {
	51	continue;
	52	}
	53	for (const env of envs.sort()) {
	54	if (!isEnvName(env)) continue;
	55	const dir = envDataDir(dataDir, app.id, env);
	56	let files: string[];
	57	try {
	58	files = readdirSync(dir).filter((f) => DB_FILE.test(f));
	59	} catch {
	60	continue;
	61	}
	62	for (const file of files.sort())
	63	out.push({
	64	id: `app:${app.id}:${env}:${file}`,
	65	label: `${app.slug} · ${env} · ${file}`,
	66	kind: "app",
	67	path: join(dir, file),
	68	});
	69	}
	70	}
	71	return out;
	72	}
	73
	74	/**
	75	* Whether the viewer has a door to the data browser at all: an admin, or a
	76	* writer of an app, whether or not any database file exists yet. A guest and
	77	* a reader have none (a 404 on the web, no entry in the terminal).
	78	*/
	79	export function maySeeData(db: Database, viewer: User \| undefined): boolean {
	80	if (viewer === undefined) return false;
	81	if (viewer.isAdmin) return true;
	82	const ids = listApps(db).map((a) => a.repoId);
	83	return ids.length > 0 && writableRepoIds(db, viewer, ids).size > 0;
	84	}
	85
	86	/** The source `id` names, if this viewer may see it; else `undefined` (a 404). */
	87	export const findSource = (
	88	db: Database,
	89	dataDir: string,
	90	viewer: User \| undefined,
	91	id: string,
	92	): Source \| undefined =>
	93	listSources(db, dataDir, viewer).find((s) => s.id === id);
		@@ -0,0 +1,32 @@
	1	/** What a read-only query may be: one SELECT or WITH statement, nothing else. */
	2
	3	/** The statement with comments and string literals blanked; identifiers stay. */
	4	function skeleton(sql: string): string {
	5	let out = "";
	6	for (let i = 0; i < sql.length; i++) {
	7	const c = sql[i]!;
	8	if (c === "-" && sql[i + 1] === "-") {
	9	while (i < sql.length && sql[i] !== "\n") i++;
	10	out += " ";
	11	} else if (c === "/" && sql[i + 1] === "*") {
	12	const end = sql.indexOf("*/", i + 2);
	13	i = end < 0 ? sql.length : end + 1;
	14	out += " ";
	15	} else if (c === "'") {
	16	i++;
	17	while (i < sql.length && sql[i] !== "'") i++;
	18	out += "''";
	19	} else out += c;
	20	}
	21	return out;
	22	}
	23
	24	export function checkQuery(sql: string): "ok" \| "empty" \| "not_select" \| "multiple" \| "schema" {
	25	const s = skeleton(sql).trim().replace(/;+\s*$/, "").trim();
	26	if (s === "") return "empty";
	27	if (s.includes(";")) return "multiple";
	28	if (!/^(select\|with)\b/i.test(s)) return "not_select";
	29	// `main.t` would step round the masking views that shadow `t`.
	30	if (/(^\|[^\w])("main"\|\[main\]\|`main`\|main)\s*\./i.test(s)) return "schema";
	31	return "ok";
	32	}