Public: anyone can read this; the crew can change it.
Boards, step 6d: a stages source and the Throughput starting board #2475
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/2461-boards-step-6d-a-stages-source-and-the-t → main 2 commits fork 1 thread · 1 unresolved
9 files
changed +183
−11
README.md +1 −1
| @@ -1247,7 +1247,7 @@ work here. | ||
| 1247 | 1247 | `forge/boards`, a repo the forge owns and makes on the first boot with an admin; every save is a commit |
| 1248 | 1248 | by the admin who made it, and a file that does not validate is not shown. Make, rename, |
| 1249 | 1249 | duplicate, delete, add/edit/remove/reorder panels, or pin the query on `/boards/data`. |
| 1250 | The first boot also writes three starting boards, **Usage**, **Crew** and **Security** (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`; no token text, hash, payload or IP), once each: `.offered` in that repo records them, so an edit stays and a deletion sticks. | |
| 1250 | The first boot also writes four starting boards, **Usage**, **Crew**, **Security** (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`; no token text, hash, payload or IP) and **Throughput** (items entering each stage from issue to live, per hour, from the `stages` source), once each: `.offered` in that repo records them, so an edit stays and a deletion sticks. | |
| 1251 | 1251 | A board with a refresh redraws its grid on that beat through the live-region swap |
| 1252 | 1252 | (`/boards/:slug/live`, etag'd). The same doors are in the API: `GET /api/v1/boards`, `GET|PUT|DELETE /api/v1/boards/:slug` |
| 1253 | 1253 | (an agent or anyone but an admin is 403; a panel whose query cannot run is refused with 422). The same four doors are MCP tools (`boards`, `board_write`, `board_delete`; `boards` with a `slug` reads one) and, for the same person, a **Boards** screen in the terminal that lists boards and shows a board's panels as text. No dependency was added. |
docs/APPS.md +9
| @@ -1198,6 +1198,15 @@ leaves its series absent, never zero, and `/boards/data?source=machine` | ||
| 1198 | 1198 | names each one. Its rows belong to no repo, so only an every-repo scope |
| 1199 | 1199 | reads them; the sources list carries each column's unit in `docs`. |
| 1200 | 1200 | |
| 1201 | The `stages` source counts items moving from issue to live: one row per | |
| 1202 | item entering a stage, with `stage` one of `1 issue` (opened), `2 run` (a | |
| 1203 | builder first started on the issue; a retry is not a second item), `3 pull` | |
| 1204 | (opened), `4 merged` (into the default branch) or `5 live` (a production | |
| 1205 | deploy served), and `at` when it entered. It is a fixed view over `issue`, | |
| 1206 | `run` and `pull` — the same moves `/flow` counts (`flow/throughput.ts`) — | |
| 1207 | and keeps no table of its own; the **Throughput** starting board reads it | |
| 1208 | by the hour. | |
| 1209 | ||
| 1201 | 1210 | POST /api/v1/boards/query |
| 1202 | 1211 | {"source": "runs", "aggregate": "count", "groupBy": "state"} |
| 1203 | 1212 | -> {"rows": [{"group": "failed", "value": 1}, {"group": "ok", "value": 7}]} |
src/boards/query.ts +2 −1
| @@ -136,8 +136,9 @@ export function runQuery(db: Database, q: unknown, scope: Scope): Answer { | ||
| 136 | 136 | } |
| 137 | 137 | const grouped = [groupSql === "NULL" ? "" : "1", seriesSql === "" ? "" : "3"].filter(Boolean).join(", "); |
| 138 | 138 | |
| 139 | const from = src.select === undefined ? ident(src.table) : `(${src.select}) AS ${ident(src.table)}`; | |
| 139 | 140 | const sql = |
| 140 | `SELECT ${groupSql} AS "group", ${valueSql} AS "value"${seriesSql} FROM ${ident(src.table)}` + | |
| 141 | `SELECT ${groupSql} AS "group", ${valueSql} AS "value"${seriesSql} FROM ${from}` + | |
| 141 | 142 | (where.length ? ` WHERE ${where.join(" AND ")}` : "") + |
| 142 | 143 | (grouped === "" ? "" : ` GROUP BY ${grouped} ORDER BY ${grouped}`) + |
| 143 | 144 | ` LIMIT ${MAX_ROWS}`; |
src/boards/sources.ts +32
| @@ -19,6 +19,11 @@ export type Source = { | ||
| 19 | 19 | * reads them. |
| 20 | 20 | */ |
| 21 | 21 | owner: "repo_id" | "issue" | "admin"; |
| 22 | /** | |
| 23 | * A fixed SELECT read in place of a table, named `table` inside the | |
| 24 | * query; no input in it. For a source that is a view over several tables. | |
| 25 | */ | |
| 26 | select?: string; | |
| 22 | 27 | /** Fixed predicate narrowing the table to this source; no input in it. */ |
| 23 | 28 | where?: string; |
| 24 | 29 | /** The column `from`, `to` and `{bucket}` apply to. */ |
| @@ -174,4 +179,31 @@ export const SOURCES: Readonly<Record<string, Source>> = { | ||
| 174 | 179 | celsius_max: "°C, the highest 15-s point in the step", |
| 175 | 180 | }, |
| 176 | 181 | }, |
| 182 | // Items entering each stage from issue to live, one row per entry, read | |
| 183 | // from issue, run and pull as `flow/throughput.ts` reads them: an issue | |
| 184 | // opens, a builder first starts on it (a retry is not a second item), its | |
| 185 | // pull opens, the pull merges into the default branch, a production | |
| 186 | // deploy serves. The stage names are numbered so they sort in that order. | |
| 187 | stages: { | |
| 188 | table: "stage", | |
| 189 | owner: "repo_id", | |
| 190 | select: | |
| 191 | "SELECT '1 issue' AS stage, repo_id, id AS issue_id, created_at AS at FROM issue WHERE kind = 'issue' " + | |
| 192 | "UNION ALL SELECT '2 run', x.repo_id, x.issue_id, MIN(x.started_at) FROM run x JOIN issue i ON i.id = x.issue_id " + | |
| 193 | "WHERE x.kind = 'agent' AND x.reason != 'review' AND i.kind = 'issue' AND x.started_at IS NOT NULL GROUP BY x.issue_id " + | |
| 194 | "UNION ALL SELECT '3 pull', i.repo_id, i.id, i.created_at FROM issue i JOIN pull p ON p.issue_id = i.id " + | |
| 195 | "UNION ALL SELECT '4 merged', i.repo_id, i.id, p.merged_at FROM pull p JOIN issue i ON i.id = p.issue_id " + | |
| 196 | "JOIN repo r ON r.id = i.repo_id WHERE p.merged_at IS NOT NULL AND p.base_branch = r.default_branch " + | |
| 197 | "UNION ALL SELECT '5 live', repo_id, NULL, serving_at FROM run " + | |
| 198 | "WHERE kind = 'deploy' AND env = 'production' AND serving_at IS NOT NULL", | |
| 199 | time: "at", | |
| 200 | columns: { stage: "text", repo_id: "text", issue_id: "text", at: "time" }, | |
| 201 | docs: { | |
| 202 | stage: | |
| 203 | "1 issue (opened), 2 run (a builder first started on the issue), 3 pull (opened), " + | |
| 204 | "4 merged (into the default branch) or 5 live (a production deploy served)", | |
| 205 | issue_id: "the issue or pull that entered; null for 5 live, which is a deploy", | |
| 206 | at: "when the item entered the stage, UTC", | |
| 207 | }, | |
| 208 | }, | |
| 177 | 209 | }; |
src/boards/starting.ts +12 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | /** |
| 2 | * The boards the forge offers a new install: Usage, Crew and Security. They are plain | |
| 2 | * The boards the forge offers a new install: Usage, Crew, Security and Throughput. They are plain | |
| 3 | 3 | * boards in `forge/boards` once written, so a person edits or deletes them |
| 4 | 4 | * like any other; `.offered` (a JSON list of slugs, committed with the |
| 5 | 5 | * boards) records what was offered so a deletion sticks and an edit is |
| @@ -17,6 +17,7 @@ const panel = (title: string, chart: Panel["chart"], query: Record<string, unkno | ||
| 17 | 17 | }); |
| 18 | 18 | const MINUTE = { bucket: "minute" }; |
| 19 | 19 | const HOUR = { bucket: "hour" }; |
| 20 | const STAGES = { source: "stages", aggregate: "count" }; | |
| 20 | 21 | const machine = (op: "eq" | "starts", value: string, column: string) => ({ |
| 21 | 22 | source: "machine", aggregate: "avg", column, groupBy: MINUTE, filter: [{ column: "target", op, value }], |
| 22 | 23 | }); |
| @@ -67,6 +68,16 @@ export const STARTING_BOARDS: Readonly<Record<string, Board>> = { | ||
| 67 | 68 | panel("Admin actions by kind", "bar", { source: "adminActions", aggregate: "count", groupBy: "kind" }, 2), |
| 68 | 69 | ], |
| 69 | 70 | }, |
| 71 | throughput: { | |
| 72 | title: "Throughput", | |
| 73 | range: "7d", | |
| 74 | refresh: "1m", | |
| 75 | panels: [ | |
| 76 | panel("Items entering each stage per hour", "bar", { ...STAGES, groupBy: HOUR, series: "stage" }, 3), | |
| 77 | panel("Entered each stage", "bar", { ...STAGES, groupBy: "stage" }), | |
| 78 | panel("Stages by hour", "heat", { ...STAGES, groupBy: HOUR, series: "stage" }, 2), | |
| 79 | ], | |
| 80 | }, | |
| 70 | 81 | }; |
| 71 | 82 | |
| 72 | 83 | /** The slugs already offered, from the repo; none when it has no such file yet. */ |
src/main.ts +1 −1
| @@ -354,7 +354,7 @@ const snapshots = startSnapshots( | ||
| 354 | 354 | // node, every 15 s into a ring the `machine` board source reads. |
| 355 | 355 | const sampler = startSampler(app); |
| 356 | 356 | const memorySampler = startMemorySampler(app); |
| 357 | // The Usage, Crew and Security starting boards, once each (actions/boards.ts). | |
| 357 | // The Usage, Crew, Security and Throughput starting boards, once each (actions/boards.ts). | |
| 358 | 358 | void offerStartingBoards(app) |
| 359 | 359 | .then((slugs) => slugs.length > 0 && app.log.info("starting boards offered", { slugs: slugs.join(", ") })) |
| 360 | 360 | .catch((error) => app.log.warn("starting boards not offered", { error: String(error) })); |
test/boards-query.test.ts +3 −1
| @@ -235,7 +235,9 @@ describe("sources", () => { | ||
| 235 | 235 | test("every listed column exists in the table", () => { |
| 236 | 236 | for (const [name, s] of Object.entries(SOURCES)) { |
| 237 | 237 | const real = new Set( |
| 238 | (db.query(`PRAGMA table_info("${s.table}")`).all() as { name: string }[]).map((c) => c.name), | |
| 238 | s.select === undefined | |
| 239 | ? (db.query(`PRAGMA table_info("${s.table}")`).all() as { name: string }[]).map((c) => c.name) | |
| 240 | : db.query(`SELECT * FROM (${s.select}) LIMIT 0`).columnNames, | |
| 239 | 241 | ); |
| 240 | 242 | for (const c of Object.keys(s.columns)) expect(real.has(c), `${name}.${c}`).toBe(true); |
| 241 | 243 | expect(s.columns[s.time]).toBe("time"); |
test/boards-stages.test.ts added +97
| @@ -0,0 +1,97 @@ | ||
| 1 | /** | |
| 2 | * The `stages` source: items entering each stage from issue to live, read | |
| 3 | * from issue, run and pull with no table of its own. With seeded rows the | |
| 4 | * counts are exact, per stage and per hour, and a repo scope narrows them. | |
| 5 | */ | |
| 6 | import { beforeAll, expect, test } from "bun:test"; | |
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { runQuery, type Scope } from "../src/boards/query.ts"; | |
| 9 | import { migrations } from "../src/core/schema.ts"; | |
| 10 | import { openDb } from "../src/kernel/db.ts"; | |
| 11 | ||
| 12 | let db: Database; | |
| 13 | const ALL: Scope = { repoIds: "all" }; | |
| 14 | const count = (q: Record<string, unknown>, scope: Scope = ALL) => runQuery(db, { source: "stages", aggregate: "count", ...q }, scope); | |
| 15 | ||
| 16 | beforeAll(() => { | |
| 17 | db = openDb(":memory:", migrations); | |
| 18 | db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1','u','u@x','x')"); | |
| 19 | db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1','u1','one'), ('r2','u1','two')"); | |
| 20 | const issue = db.prepare("INSERT INTO issue (id, repo_id, number, kind, title, author_id, created_at) VALUES (?, ?, ?, ?, 't', 'u1', ?)"); | |
| 21 | const pull = db.prepare("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?, ?, 'h', 'o', 'o', ?)"); | |
| 22 | const run = db.prepare( | |
| 23 | "INSERT INTO run (id, repo_id, kind, issue_id, env, reason, state, started_at, serving_at) VALUES (?, ?, ?, ?, ?, ?, 'ok', ?, ?)", | |
| 24 | ); | |
| 25 | // r1: two issues opened in the 10:00 hour, one in 11:00. | |
| 26 | issue.run("i1", "r1", 1, "issue", "2026-10-01T10:05:00.000Z"); | |
| 27 | issue.run("i2", "r1", 2, "issue", "2026-10-01T10:55:00.000Z"); | |
| 28 | issue.run("i3", "r1", 3, "issue", "2026-10-01T11:00:00.000Z"); | |
| 29 | // A builder starts on i1 twice (a retry) and on i2 once: two items start, at their first start. | |
| 30 | run.run("a1", "r1", "agent", "i1", "", "", "2026-10-01T11:10:00.000Z", null); | |
| 31 | run.run("a2", "r1", "agent", "i1", "", "", "2026-10-01T12:30:00.000Z", null); | |
| 32 | run.run("a3", "r1", "agent", "i2", "", "", "2026-10-01T11:20:00.000Z", null); | |
| 33 | // Queued, never started; a review run; a check: none of them is a start. | |
| 34 | run.run("a4", "r1", "agent", "i3", "", "", null, null); | |
| 35 | run.run("a5", "r1", "agent", "i1", "", "review", "2026-10-01T11:40:00.000Z", null); | |
| 36 | run.run("c1", "r1", "check", "i1", "", "", "2026-10-01T11:45:00.000Z", null); | |
| 37 | // Two pulls open at 12:xx; p1 merges into main, p2 into another branch (not a landing), p3 stays open. | |
| 38 | issue.run("p1", "r1", 4, "pull", "2026-10-01T12:00:00.000Z"); | |
| 39 | issue.run("p2", "r1", 5, "pull", "2026-10-01T12:15:00.000Z"); | |
| 40 | issue.run("p3", "r1", 6, "pull", "2026-10-01T13:05:00.000Z"); | |
| 41 | pull.run("p1", "main", "2026-10-01T13:10:00.000Z"); | |
| 42 | pull.run("p2", "side", "2026-10-01T13:20:00.000Z"); | |
| 43 | pull.run("p3", "main", null); | |
| 44 | // One production deploy serves at 13:30; a preview's and one never served are not live. | |
| 45 | run.run("d1", "r1", "deploy", null, "production", "", "2026-10-01T13:25:00.000Z", "2026-10-01T13:30:00.000Z"); | |
| 46 | run.run("d2", "r1", "deploy", null, "preview-4", "", "2026-10-01T13:25:00.000Z", "2026-10-01T13:31:00.000Z"); | |
| 47 | run.run("d3", "r1", "deploy", null, "production", "", "2026-10-01T13:40:00.000Z", null); | |
| 48 | // r2: one issue and one live deploy, in other hours. | |
| 49 | issue.run("j1", "r2", 1, "issue", "2026-10-01T09:00:00.000Z"); | |
| 50 | run.run("e1", "r2", "deploy", null, "production", "", "2026-10-01T14:00:00.000Z", "2026-10-01T14:05:00.000Z"); | |
| 51 | }); | |
| 52 | ||
| 53 | test("counts each stage exactly", () => { | |
| 54 | expect(count({ groupBy: "stage" })).toEqual({ | |
| 55 | ok: true, | |
| 56 | rows: [ | |
| 57 | { group: "1 issue", value: 4 }, | |
| 58 | { group: "2 run", value: 2 }, | |
| 59 | { group: "3 pull", value: 3 }, | |
| 60 | { group: "4 merged", value: 1 }, | |
| 61 | { group: "5 live", value: 2 }, | |
| 62 | ], | |
| 63 | }); | |
| 64 | }); | |
| 65 | ||
| 66 | test("counts each stage per hour, at the moment it entered", () => { | |
| 67 | expect(count({ groupBy: { bucket: "hour" }, series: "stage" }, { repoIds: ["r1"] })).toEqual({ | |
| 68 | ok: true, | |
| 69 | rows: [ | |
| 70 | { group: "2026-10-01T10:00:00Z", series: "1 issue", value: 2 }, | |
| 71 | { group: "2026-10-01T11:00:00Z", series: "1 issue", value: 1 }, | |
| 72 | { group: "2026-10-01T11:00:00Z", series: "2 run", value: 2 }, | |
| 73 | { group: "2026-10-01T12:00:00Z", series: "3 pull", value: 2 }, | |
| 74 | { group: "2026-10-01T13:00:00Z", series: "3 pull", value: 1 }, | |
| 75 | { group: "2026-10-01T13:00:00Z", series: "4 merged", value: 1 }, | |
| 76 | { group: "2026-10-01T13:00:00Z", series: "5 live", value: 1 }, | |
| 77 | ], | |
| 78 | }); | |
| 79 | }); | |
| 80 | ||
| 81 | test("a repo scope and a range narrow it", () => { | |
| 82 | expect(count({ groupBy: "stage" }, { repoIds: ["r2"] })).toEqual({ | |
| 83 | ok: true, | |
| 84 | rows: [{ group: "1 issue", value: 1 }, { group: "5 live", value: 1 }], | |
| 85 | }); | |
| 86 | // The retry at 12:30 is not a start: i1 entered at 11:10, outside this range. | |
| 87 | expect(count({ from: "2026-10-01T12:00:00Z", to: "2026-10-01T13:00:00Z", groupBy: "stage" })).toEqual({ | |
| 88 | ok: true, | |
| 89 | rows: [{ group: "3 pull", value: 2 }], | |
| 90 | }); | |
| 91 | expect(count({ filter: [{ column: "stage", op: "eq", value: "5 live" }] })).toEqual({ ok: true, rows: [{ group: null, value: 2 }] }); | |
| 92 | }); | |
| 93 | ||
| 94 | test("answers on a fresh forge", () => { | |
| 95 | const fresh = openDb(":memory:", migrations); | |
| 96 | expect(runQuery(fresh, { source: "stages", aggregate: "count", groupBy: "stage" }, ALL)).toEqual({ ok: true, rows: [] }); | |
| 97 | }); |
test/boards-starting.test.ts +26 −6
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | /** |
| 2 | * The forge offers the Usage, Crew and Security boards on boot: every panel runs, an | |
| 2 | * The forge offers the Usage, Crew, Security and Throughput boards on boot: every panel runs, an | |
| 3 | 3 | * edit survives the next boot, and a deletion is not undone. |
| 4 | 4 | */ |
| 5 | 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; |
| @@ -10,7 +10,10 @@ import { changeBoard, deleteBoard, offerStartingBoards } from "../src/actions/bo | ||
| 10 | 10 | import { createApp } from "../src/app.ts"; |
| 11 | 11 | import { runQuery } from "../src/boards/query.ts"; |
| 12 | 12 | import { STARTING_BOARDS } from "../src/boards/starting.ts"; |
| 13 | import { listBoards, readBoard } from "../src/boards/store.ts"; | |
| 13 | import { OFFERED_FILE } from "../src/boards/starting.ts"; | |
| 14 | import { BOARDS_OWNER, BOARDS_REPO, listBoards, readBoard } from "../src/boards/store.ts"; | |
| 15 | import { repoDir } from "../src/git/repos.ts"; | |
| 16 | import { commitFiles } from "../src/git/write.ts"; | |
| 14 | 17 | import { createUser, type User } from "../src/core/users.ts"; |
| 15 | 18 | |
| 16 | 19 | let dir: string; |
| @@ -33,10 +36,10 @@ test("a forge with no person to found the repo writes nothing", async () => { | ||
| 33 | 36 | expect(await offerStartingBoards(bare)).toEqual([]); |
| 34 | 37 | }); |
| 35 | 38 | |
| 36 | test("a fresh boot writes Usage, Crew and Security, and every panel runs", async () => { | |
| 37 | expect((await offerStartingBoards(app)).sort()).toEqual(["crew", "security", "usage"]); | |
| 39 | test("a fresh boot writes Usage, Crew, Security and Throughput, and every panel runs", async () => { | |
| 40 | expect((await offerStartingBoards(app)).sort()).toEqual(["crew", "security", "throughput", "usage"]); | |
| 38 | 41 | const boards = await listBoards(app.reposRoot); |
| 39 | expect(boards.map((b) => b.slug)).toEqual(["crew", "security", "usage"]); | |
| 42 | expect(boards.map((b) => b.slug)).toEqual(["crew", "security", "throughput", "usage"]); | |
| 40 | 43 | let panels = 0; |
| 41 | 44 | for (const { board } of boards) { |
| 42 | 45 | for (const p of board.panels) { |
| @@ -45,7 +48,7 @@ test("a fresh boot writes Usage, Crew and Security, and every panel runs", async | ||
| 45 | 48 | panels++; |
| 46 | 49 | } |
| 47 | 50 | } |
| 48 | expect(panels).toBe(STARTING_BOARDS.usage!.panels.length + STARTING_BOARDS.crew!.panels.length + STARTING_BOARDS.security!.panels.length); | |
| 51 | expect(panels).toBe(Object.values(STARTING_BOARDS).reduce((n, b) => n + b.panels.length, 0)); | |
| 49 | 52 | }); |
| 50 | 53 | |
| 51 | 54 | test("a second boot leaves an edit in place", async () => { |
| @@ -60,3 +63,20 @@ test("a deleted starting board stays deleted", async () => { | ||
| 60 | 63 | expect(await offerStartingBoards(app)).toEqual([]); |
| 61 | 64 | expect(await readBoard(app.reposRoot, "crew")).toBeUndefined(); |
| 62 | 65 | }); |
| 66 | ||
| 67 | test("a forge offered Usage, Crew and Security before gets Throughput, and the rest stay as they are", async () => { | |
| 68 | // As a forge from before Throughput existed: it is not in `.offered` and has no file. | |
| 69 | expect((await deleteBoard(app, admin, "throughput")).ok).toBe(true); | |
| 70 | const rewound = await commitFiles(repoDir(app.reposRoot, BOARDS_OWNER, BOARDS_REPO), { | |
| 71 | branch: "main", | |
| 72 | message: "as offered before Throughput", | |
| 73 | files: [{ path: OFFERED_FILE, content: `${JSON.stringify(["usage", "crew", "security"])}\n` }], | |
| 74 | author: { name: "forge", email: "[email protected]" }, | |
| 75 | }); | |
| 76 | expect(rewound.ok).toBe(true); | |
| 77 | expect(await offerStartingBoards(app)).toEqual(["throughput"]); | |
| 78 | expect((await readBoard(app.reposRoot, "throughput"))?.title).toBe("Throughput"); | |
| 79 | expect((await readBoard(app.reposRoot, "usage"))?.title).toBe("My usage"); | |
| 80 | expect(await readBoard(app.reposRoot, "crew")).toBeUndefined(); | |
| 81 | expect(await offerStartingBoards(app)).toEqual([]); | |
| 82 | }); |
new above
offline
Each stage can show up as its own part of the board in plain words (Opened, Started, Pull, Merged, Live) once the owner-facing labels are decided. The numeric prefixes are fine for sorting but read nerdy on a chart legend.