Public: anyone can read this; the crew can change it.
MCP system {kind:"routes"} answers the slowest routes against their budgets, as the performance watcher sees them #11490
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/11476-mcp-system-kind-routes-answers-the-slowe → main 9 commits fork 2 threads · 2 unresolved
7 files
changed +149
−9
docs/MCP.md +6 −1
| @@ -209,7 +209,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`; | ||
| 209 | 209 | | `subscription_wake` | — | Admin: wake a resting subscription | 168 | |
| 210 | 210 | | `subscriptions` | — | Claude accounts: scopes, budgets | 158 | |
| 211 | 211 | | `sync` | — | Sync a mirror now; `originId` reconnects it | 197 | |
| 212 | | `system` | `csp` | The forge machine now: CPU, memory, disk, network | 277 | | |
| 212 | | `system` | `csp`, `routes` | The forge machine now: CPU, memory, disk, network | 286 | | |
| 213 | 213 | | `take_back` | — | Open the pull that undoes a shipped pull | 219 | |
| 214 | 214 | | `take_updates` | — | Take the parent forge's updates into a fork | 176 | |
| 215 | 215 | | `teams` | — | Crew teams for person admins: charter, dial (0–100) and member agents | 338 | |
| @@ -314,6 +314,11 @@ the blocked host counted last, read by `violationsByDirective` in | ||
| 314 | 314 | `src/http/csp_report.ts` as the security page reads its week. A forge admin |
| 315 | 315 | only; anyone else is `forbidden`. |
| 316 | 316 | |
| 317 | `system` with `{ kind: "routes" }` (#11476): up to 10 budgeted routes as | |
| 318 | `{ route, p95Ms, budgetMs, n, slowestPhase }` over the watcher's 15-minute | |
| 319 | record, the most over budget first; route patterns only. An admin or an agent | |
| 320 | granted `telemetry:read`. | |
| 321 | ||
| 317 | 322 | `flow` with `{ q, action }` is the fleet's write door (#450): the plan |
| 318 | 323 | `POST /api/v1/fleet/ops` answers for a dry run, `{ dryRun: true, query, kind, |
| 319 | 324 | verb, value, reversible, rows, counts }`, with nothing changed and only a read |
docs/features/performance-watcher.md +1 −1
| @@ -4,4 +4,4 @@ | ||
| 4 | 4 | |
| 5 | 5 | `history.ts` (#11477): every tick the duty folds the record's new samples into an in-memory ledger under the hour each sample happened in (so 10:50 traffic is never stamped 11:00) and upserts one row per route pattern (route, hour, n, p50, p95) into `route_hour`, so a row is the hour's p50 and p95, not the last 15 minutes'. At most 5,000 samples per route and hour are kept (past it the row stands on the first), at most 200 routes (the busiest) and 168 hours; older rows go in the same write. A restart starts an empty ledger: an empty record writes nothing, and a row is never replaced by one with fewer requests, so the hours before the restart keep the fuller row. Admins read them at `GET /api/v1/telemetry/routes?hours=N` (1 to 168, default 24; else 400). |
| 6 | 6 | |
| 7 | Not yet: a synthetic probe for routes with no traffic (a client-written label would let anyone hide requests; it needs an admin-only door first), route-hour stats in the terminal and MCP (findings reach it as issues), browser timings, the board page, release before/after. | |
| 7 | Not yet: a synthetic probe for routes with no traffic (a client-written label would let anyone hide requests; it needs an admin-only door first), route-hour stats in the terminal (findings reach it as issues; the MCP `system` `kind: routes` reads the live routes), browser timings, the board page, release before/after. |
src/http/mcp/tools/system.ts +11 −4
| @@ -4,20 +4,27 @@ import { scriptErrorCounts } from "../../script_errors.ts"; | ||
| 4 | 4 | import { violationsByDirective, windowStart } from "../../csp_report.ts"; |
| 5 | 5 | import { countRead } from "../../../system/telemetry_reads.ts"; |
| 6 | 6 | import { stallHour, TOP_ROUTES } from "../../../metrics/stalls.ts"; |
| 7 | import { lagMinutes, loopWatch } from "../../../perf/live.ts"; | |
| 7 | import { lagMinutes, loopWatch, perfRecord } from "../../../perf/live.ts"; | |
| 8 | 8 | import { LAG_BAR_MS, overBar } from "../../../perf/lag_minutes.ts"; |
| 9 | import { slowestRoutes } from "../../../perf/watch.ts"; | |
| 9 | 10 | import { good, bad, type Tool } from "../shared.ts"; |
| 10 | 11 | |
| 11 | 12 | export const tool: Tool = { |
| 12 | 13 | name: "system", |
| 13 | doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only", | |
| 14 | doc: "the machine the forge runs on, now: CPU, load, memory, disk, network, processes, `stalls`: the last hour's main-thread stalls by route (`<route> (in flight)` when still running at the wake), or `job:<name>`, as {cause, ms, count}, `readings`: the last 20 minutes' worst loop lag as {at, ms, what} beside `barMs` (100) and `over`, and `readers`: who polls the telemetry and how fast, `{ kind: \"routes\" }`: the 10 slowest budgeted routes as {route, p95Ms, budgetMs, n, slowestPhase}, over budget first; a forge admin, or an agent granted `telemetry:read`. `{ kind: \"csp\" }`: the last 24 h of Content-Security-Policy violations browsers reported, `{ directive, n, newest }` per directive (`newest` the blocked host last counted), most first, counted per UTC hour from `since`, and `scriptErrors` `{ page, where, message, n, last }`, as the security page reads them; a forge admin only", | |
| 14 | 15 | description: |
| 15 | 16 | "The forge machine now: CPU, memory, disk, network. Admin or telemetry:read; `kind: csp`: 24 h of CSP violations (admin).", |
| 16 | inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp"] } } }, | |
| 17 | inputSchema: { type: "object", properties: { kind: { type: "string", enum: ["csp", "routes"] } } }, | |
| 17 | 18 | writes: false, |
| 18 | 19 | run(ctx, user, args) { |
| 19 | 20 | if (args["kind"] !== undefined) { |
| 20 | if (args["kind"] !== "csp") return bad("invalid", "kind is csp, or absent for the machine"); | |
| 21 | if (args["kind"] === "routes") { | |
| 22 | const reader = telemetryReader(ctx.app, user); | |
| 23 | if (!reader) return bad("forbidden", "only an admin of this forge, or an agent granted telemetry:read, reads its route timings"); | |
| 24 | noteRead(ctx.app, user, reader, "system"); | |
| 25 | return good({ routes: slowestRoutes(perfRecord, Date.now()) }); | |
| 26 | } | |
| 27 | if (args["kind"] !== "csp") return bad("invalid", "kind is csp, routes, or absent for the machine"); | |
outdated
Copy of WRAPPERS in src/perf/watch.ts. Share one exported helper instead, and move this row-building into src/perf. |
||
| 21 | 28 | if (!user.isAdmin) return bad("forbidden", "only an admin of this forge reads its CSP violations"); |
| 22 | 29 | const now = new Date(); |
| 23 | 30 | return good({ hours: 24, since: windowStart(24, now).toISOString(), directives: violationsByDirective(ctx.app.db, 24, now), scriptErrors: scriptErrorCounts(ctx.app.db, 24, now) }); |
src/perf/watch.ts +21 −2
| @@ -9,7 +9,7 @@ import { found, type Finding } from "../hunt/finding.ts"; | ||
| 9 | 9 | import type { Database } from "bun:sqlite"; |
| 10 | 10 | import { dayOf } from "./day.ts"; |
| 11 | 11 | import { budgetFor, gitBudgetFor, PAGE_BUDGET } from "./budgets.ts"; |
| 12 | import { percentile, summarize, WINDOW_MS, type PerfRecord } from "./record.ts"; | |
| 12 | import { percentile, summarize, WINDOW_MS, type PerfRecord, type RouteStat } from "./record.ts"; | |
| 13 | 13 | |
| 14 | 14 | export { WINDOW_MS }; |
| 15 | 15 | export const MIN_REQUESTS = 20; |
| @@ -19,6 +19,25 @@ const CACHE_PHASE = /(cache|kpis|panel|vitals)/; | ||
| 19 | 19 | /** `app`, `db` and `route` wrap the others; the blame goes to a part. */ |
| 20 | 20 | const WRAPPERS = new Set(["app", "db", "route", "compress"]); |
| 21 | 21 | |
| 22 | /** The phase the blame goes to: the slowest part of a route's p95, never a wrapper; undefined when there is none. */ | |
| 23 | export function slowestPhase(stat: RouteStat): RouteStat["phases"][number] | undefined { | |
| 24 | return stat.phases.find((p) => !WRAPPERS.has(p.name)); | |
| 25 | } | |
| 26 | ||
| 27 | export type SlowRoute = { route: string; p95Ms: number; budgetMs: number; n: number; slowestPhase: string | null }; | |
| 28 | ||
| 29 | /** The `limit` budgeted routes furthest over their p95 budget (ratio p95 / budget) in the window ending at `now`; routes with no budget are left out. */ | |
| 30 | export function slowestRoutes(record: PerfRecord, now: number, limit = 10): SlowRoute[] { | |
| 31 | const rows = record.patterns().flatMap((route) => { | |
| 32 | const stat = summarize(record.traffic(route, now - WINDOW_MS)); | |
| 33 | if (stat === null) return []; | |
| 34 | const budget = budgetFor(route); | |
| 35 | if (budget === undefined) return []; | |
| 36 | return [{ over: stat.p95 / budget.p95Ms, row: { route, p95Ms: Math.round(stat.p95 * 10) / 10, budgetMs: budget.p95Ms, n: stat.n, slowestPhase: slowestPhase(stat)?.name ?? null } }]; | |
| 37 | }); | |
| 38 | return rows.sort((a, b) => b.over - a.over).slice(0, limit).map((r) => r.row); | |
| 39 | } | |
| 40 | ||
| 22 | 41 | export function evaluate(record: PerfRecord, now: number): Finding[] { |
| 23 | 42 | const out: Finding[] = []; |
| 24 | 43 | for (const pattern of record.patterns()) { |
| @@ -29,7 +48,7 @@ export function evaluate(record: PerfRecord, now: number): Finding[] { | ||
| 29 | 48 | if (stat === null || stat.n < MIN_REQUESTS || stat.p95 <= budget.p95Ms) continue; |
| 30 | 49 | // Over budget "for 15 min": the route must have been watched most of the window. |
| 31 | 50 | if (now - (record.first(pattern) ?? now) < WINDOW_MS * 0.9) continue; |
| 32 | const slow = stat.phases.find((p) => !WRAPPERS.has(p.name)); | |
| 51 | const slow = slowestPhase(stat); | |
| 33 | 52 | const phase = slow?.name ?? "app"; |
| 34 | 53 | const lines = [ |
| 35 | 54 | `Budget: ${budget.words} (${budget.p95Ms} ms).`, |
test/mcp-system-routes.test.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | /** MCP `system` `{kind:"routes"}` answers the slowest budgeted routes, worst over budget first (#11476). */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createAgent } from "../src/actions/crew.ts"; | |
| 8 | import { setTelemetryGrant } from "../src/actions/telemetry.ts"; | |
| 9 | import { createToken } from "../src/core/tokens.ts"; | |
| 10 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import { perfRecord } from "../src/perf/live.ts"; | |
| 13 | import { quietLogs } from "./helpers/globals.ts"; | |
| 14 | ||
| 15 | quietLogs(); | |
| 16 | ||
| 17 | let dataDir = ""; | |
| 18 | let app: App; | |
| 19 | let servers: Servers; | |
| 20 | let admin = ""; | |
| 21 | let plain = ""; | |
| 22 | ||
| 23 | beforeAll(async () => { | |
| 24 | dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-routes-")); | |
| 25 | app = createApp({ dataDir, port: 0 }); | |
| 26 | servers = await serve(app); | |
| 27 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "correct horse battery 9", isAdmin: true }); | |
| 28 | const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "correct horse battery 9" }); | |
| 29 | if (!a.ok || !b.ok) throw new Error("users"); | |
| 30 | admin = createToken(app.db, a.value.id, "t", "admin").token; | |
| 31 | plain = createToken(app.db, b.value.id, "t", "admin").token; | |
| 32 | }, 30_000); | |
| 33 | ||
| 34 | afterAll(async () => { | |
| 35 | servers.stop(); | |
| 36 | app.close(); | |
| 37 | await rm(dataDir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | const call = async (token: string) => { | |
| 41 | const res = await fetch(`${servers.url}/mcp`, { | |
| 42 | method: "POST", | |
| 43 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, | |
| 44 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "system", arguments: { kind: "routes" } } }), | |
| 45 | }); | |
| 46 | return (await res.json()) as any; | |
| 47 | }; | |
| 48 | ||
| 49 | test("routes lists the over-budget route first, with its budget and slowest phase; a non-admin is refused", async () => { | |
| 50 | const at = Date.now(); | |
| 51 | for (let i = 0; i < 25; i++) { | |
| 52 | perfRecord.add("/boards", { at, ms: 100, signedIn: true, phases: [{ name: "app", ms: 100, hit: false }] }); | |
| 53 | perfRecord.add("/explore", { at, ms: 2000, signedIn: true, phases: [{ name: "app", ms: 2000, hit: false }, { name: "read", ms: 1500, hit: false }] }); | |
| 54 | } | |
| 55 | const out = JSON.parse((await call(admin)).result.content[0].text); | |
| 56 | const mine = out.routes.filter((r: any) => r.route === "/boards" || r.route === "/explore"); | |
| 57 | expect(out.routes.length).toBeLessThanOrEqual(10); | |
| 58 | expect(mine[0]).toEqual({ route: "/explore", p95Ms: 2000, budgetMs: 800, n: 25, slowestPhase: "read" }); | |
| 59 | expect(mine[1]).toEqual({ route: "/boards", p95Ms: 100, budgetMs: 800, n: 25, slowestPhase: null }); | |
| 60 | expect(out.routes[0].route).toBe("/explore"); | |
| 61 | ||
| 62 | const refused = await call(plain); | |
| 63 | expect(refused.result.isError).toBe(true); | |
| 64 | expect(refused.result.content[0].text).toContain("forbidden"); | |
| 65 | }); | |
| 66 | ||
| 67 | test("an agent granted telemetry:read is allowed; without the grant it is refused", async () => { | |
| 68 | const alice = getUserByName(app.db, "alice")!; | |
| 69 | const bot = await createAgent(app, alice, { username: "routebot" }); | |
| 70 | if (!bot.ok) throw new Error("bot"); | |
| 71 | const token = createToken(app.db, bot.value.user.id, "t", "ro").token; | |
| 72 | expect((await call(token)).result.isError).toBe(true); | |
| 73 | expect(setTelemetryGrant(app, alice, "routebot", true).ok).toBe(true); | |
| 74 | const out = await call(token); | |
| 75 | expect(out.result.isError).not.toBe(true); | |
| 76 | expect(Array.isArray(JSON.parse(out.result.content[0].text).routes)).toBe(true); | |
| 77 | }); |
test/perf-slowest-routes.test.ts added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | /** slowestRoutes: budgeted routes only, worst over budget first, capped (#11476). */ | |
| 2 | import { expect, test } from "bun:test"; | |
| 3 | import { createPerfRecord } from "../src/perf/record.ts"; | |
| 4 | import { slowestRoutes } from "../src/perf/watch.ts"; | |
| 5 | ||
| 6 | const at = Date.now(); | |
| 7 | const fill = (rec: ReturnType<typeof createPerfRecord>, route: string, ms: number) => { | |
| 8 | for (let i = 0; i < 5; i++) rec.add(route, { at, ms, signedIn: true, phases: [{ name: "app", ms, hit: false }, { name: "read", ms: ms / 2, hit: false }] }); | |
| 9 | }; | |
| 10 | ||
| 11 | test("a route with no budget is omitted; the rest are ordered by p95 over budget", () => { | |
| 12 | const rec = createPerfRecord(() => true); | |
| 13 | fill(rec, "/boards", 100); | |
| 14 | fill(rec, "/explore", 2000); | |
| 15 | fill(rec, "/no-such-budgeted-route", 99999); | |
| 16 | const rows = slowestRoutes(rec, at + 1); | |
| 17 | expect(rows.map((r) => r.route)).toEqual(["/explore", "/boards"]); | |
| 18 | expect(rows[0]).toEqual({ route: "/explore", p95Ms: 2000, budgetMs: 800, n: 5, slowestPhase: "read" }); | |
| 19 | }); | |
| 20 | ||
| 21 | test("the list is capped at 10", () => { | |
| 22 | const rec = createPerfRecord(() => true); | |
| 23 | const routes = ["/boards", "/explore", "/", "/issues", "/pulls", "/crew", "/system", "/telemetry", "/security", "/search", "/forks", "/notes"]; | |
| 24 | for (const r of routes) fill(rec, r, 5000); | |
| 25 | const budgeted = slowestRoutes(rec, at + 1, 100).length; | |
| 26 | expect(budgeted).toBeGreaterThan(2); | |
| 27 | expect(slowestRoutes(rec, at + 1, 2).length).toBe(Math.min(2, budgeted)); | |
| 28 | expect(slowestRoutes(rec, at + 1).length).toBeLessThanOrEqual(10); | |
| 29 | }); |
test/stall-in-flight.test.ts +4 −1
| @@ -176,6 +176,9 @@ function describe_server() { | ||
| 176 | 176 | expect(inFlight.size()).toBe(0); |
| 177 | 177 | |
| 178 | 178 | const out = await system(); |
| 179 | expect(out.stalls).toContainEqual({ cause: `${named[0]} (in flight)`, ms: LIMIT_MS + 10, count: 1 }); | |
| 179 | // A loaded machine may charge a real stall to the same cause as well: at least ours is there. | |
| 180 | const row = out.stalls.find((r: { cause: string }) => r.cause === `${named[0]} (in flight)`); | |
| 181 | expect(row.count).toBeGreaterThanOrEqual(1); | |
| 182 | expect(row.ms).toBeGreaterThanOrEqual(LIMIT_MS + 10); | |
| 180 | 183 | }); |
| 181 | 184 | } |
Duplicate import from perf/live.ts; merge with line 8's.