Public: anyone can read this; the crew can change it.
Flaky: mcp-write-scope snapshot and the chrome click helper under load #9079
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9031-flaky-mcp-write-scope-snapshot-and-the-c → main 3 commits fork 4 threads · 4 unresolved
5 files
changed +214
−22
test/chrome-retry.test.ts added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { retryStep } from "./helpers/chrome.ts"; | |
| 3 | ||
| 4 | test("a step that fails until the element appears is retried until it runs", async () => { | |
| 5 | let calls = 0; | |
| 6 | await retryStep(async () => (++calls < 4 ? "no such element" : undefined), Date.now() + 2_000, 1); | |
| 7 | expect(calls).toBe(4); | |
| 8 | }); | |
| 9 | ||
| 10 | test("a step that never succeeds throws its last failure at the deadline", async () => { | |
| 11 | await expect(retryStep(async () => "no such element", Date.now() + 50, 5)).rejects.toThrow( | |
| 12 | "step never ran: no such element", | |
| 13 | ); | |
| 14 | }); |
test/helpers/chrome.ts +29 −2
| @@ -339,6 +339,25 @@ export interface Followed { | ||
| 339 | 339 | readonly errors: string[]; |
| 340 | 340 | } |
| 341 | 341 | |
| 342 | /** | |
| 343 | * Run `attempt` until it reports no failure (`undefined`) or `deadline` | |
| 344 | * passes, pausing `pauseMs` between tries: a click on an element the page | |
| 345 | * has not drawn yet throws, and under load that is only "not yet". Throws | |
| 346 | * the last failure when time runs out. | |
| 347 | */ | |
| 348 | export async function retryStep( | |
| 349 | attempt: () => Promise<string | undefined>, | |
| 350 | deadline: number, | |
| 351 | pauseMs = 50, | |
| 352 | ): Promise<void> { | |
| 353 | for (;;) { | |
| 354 | const failure = await attempt(); | |
| 355 | if (failure === undefined) return; | |
| 356 | if (Date.now() >= deadline) throw new Error(`step never ran: ${failure}`); | |
| 357 | await Bun.sleep(pauseMs); | |
| 358 | } | |
| 359 | } | |
| 360 | ||
| 342 | 361 | /** |
| 343 | 362 | * Load `url` with a session cookie, then run each of `steps` (an |
| 344 | 363 | * expression evaluated in the page — a click) after the page before it has |
| @@ -432,8 +451,16 @@ export async function clickAndFollow( | ||
| 432 | 451 | for (const step of input.steps) { |
| 433 | 452 | if (reached()) break; |
| 434 | 453 | loaded = cdp.once("Page.loadEventFired"); |
| 435 | // As a person's click: a user gesture, so a link that opens a tab does. | |
| 436 | await cdp.send("Runtime.evaluate", { expression: step, userGesture: true }); | |
| 454 | await retryStep( | |
| 455 | async () => { | |
| 456 | // As a person's click: a user gesture, so a link that opens a tab does. | |
| 457 | const r = await cdp.send("Runtime.evaluate", { expression: step, userGesture: true }); | |
| 458 | return r.exceptionDetails === undefined | |
| 459 | ? undefined | |
| 460 | : String(r.exceptionDetails.exception?.description ?? r.exceptionDetails.text); | |
| 461 | }, | |
| 462 | deadline, | |
| 463 | ); | |
| 437 | 464 | await Promise.race([loaded, Bun.sleep(Math.max(0, deadline - Date.now()))]); |
| 438 | 465 | } |
| 439 | 466 | while (!reached() && Date.now() < deadline) await Bun.sleep(100); |
test/helpers/write-snapshot.ts added +64
| @@ -0,0 +1,64 @@ | ||
| 1 | /** | |
| 2 | * What a database holds, read so that two reads around a run of refused | |
| 3 | * writes can be compared (#9031). Every table is held to its row count, | |
| 4 | * except the two counters a timer flushes on its own (`route_timing`, | |
| 5 | * `usage_count`): there the `/mcp` rows, which count the test's own | |
| 6 | * requests, are left out of the snapshot and summed by `ownCount` instead, | |
| 7 | * and every other row is held exactly, in primary-key order. | |
| 8 | * | |
| 9 | * Why the exact rows cannot flake: a row only appears in these tables for a | |
| 10 | * request the server answered, keyed by its route pattern. A server that | |
| 11 | * only ever answers `POST /mcp` has nothing pending under any other pattern, | |
| 12 | * so a flush between two reads touches `/mcp` rows alone; an hour turning | |
| 13 | * over adds a new `/mcp` row, which is also left out; and the 30-day prune | |
| 14 | * finds nothing in a database made minutes ago. | |
| 15 | */ | |
| 16 | import type { Database } from "bun:sqlite"; | |
| 17 | ||
| 18 | /** The pattern the router files every MCP call under (`mcpRoutes` in src/http/mcp.ts). */ | |
| 19 | export const MCP_PATTERN = "/mcp"; | |
| 20 | ||
| 21 | /** Each timer-flushed counter, with its primary key in order (see its migration). */ | |
| 22 | export const COUNTERS: Record<string, readonly string[]> = { | |
| 23 | route_timing: ["hour", "pattern", "bucket"], | |
| 24 | usage_count: ["hour", "pattern", "method", "status", "role", "surface"], | |
| 25 | }; | |
| 26 | ||
| 27 | export type Snapshot = Record<string, number | unknown[]>; | |
| 28 | ||
| 29 | export function snapshot(db: Database): Snapshot { | |
| 30 | const tables = db | |
| 31 | .query<{ name: string }, []>( | |
| 32 | "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", | |
| 33 | ) | |
| 34 | .all() | |
| 35 | .map((r) => r.name); | |
| 36 | return Object.fromEntries( | |
| 37 | tables.map((n) => { | |
| 38 | const key = COUNTERS[n]; | |
| 39 | return [ | |
| 40 | n, | |
| 41 | key | |
| 42 | ? db | |
| 43 | .query(`SELECT * FROM "${n}" WHERE pattern IS NOT ?1 ORDER BY ${key.join(", ")}`) | |
| 44 | .all(MCP_PATTERN) | |
| 45 | : db.query<{ c: number }, []>(`SELECT count(*) AS c FROM "${n}"`).get()!.c, | |
| 46 | ]; | |
| 47 | }), | |
| 48 | ); | |
| 49 | } | |
| 50 | ||
| 51 | /** Total counted under `/mcp` across the counters: it may rise, never fall. */ | |
| 52 | export function ownCount(db: Database): number { | |
| 53 | let sum = 0; | |
| 54 | for (const n of Object.keys(COUNTERS)) { | |
| 55 | const exists = db | |
| 56 | .query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1") | |
| 57 | .get(n); | |
| 58 | if (!exists) continue; | |
| 59 | sum += db | |
| 60 | .query<{ t: number }, [string]>(`SELECT coalesce(sum(n), 0) AS t FROM "${n}" WHERE pattern = ?1`) | |
| 61 | .get(MCP_PATTERN)!.t; | |
| 62 | } | |
| 63 | return sum; | |
| 64 | } |
test/mcp-write-scope.test.ts +11 −20
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * Every MCP tool that writes, called with a read-only token, is refused |
| 3 | 3 | * before it writes; every tool that needs an admin token is refused to an |
| 4 | * `rw` one. The event log and every table's row count stay as they were. | |
| 4 | * `rw` one. The event log, every table's row count and every counter row | |
| 5 | * outside `/mcp` stay as they were (see test/helpers/write-snapshot.ts). | |
| 5 | 6 | */ |
| 6 | 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 7 | 8 | import { mkdtemp, rm } from "node:fs/promises"; |
| @@ -13,6 +14,7 @@ import { createRepo } from "../src/actions/repos.ts"; | ||
| 13 | 14 | import { createUser } from "../src/core/users.ts"; |
| 14 | 15 | import { toolRegistry, toolNames } from "../src/http/mcp.ts"; |
| 15 | 16 | import { serve, type Servers } from "../src/http/server.ts"; |
| 17 | import { ownCount, snapshot } from "./helpers/write-snapshot.ts"; | |
| 16 | 18 | |
| 17 | 19 | /** Tools this test cannot call as a write, by name, and why. */ |
| 18 | 20 | const SKIPPED: Record<string, string> = {}; |
| @@ -102,21 +104,6 @@ describe("a token that may only read cannot write through any tool", () => { | ||
| 102 | 104 | return args; |
| 103 | 105 | }; |
| 104 | 106 | |
| 105 | const snapshot = () => { | |
| 106 | const tables = app.db | |
| 107 | .query<{ name: string }, []>( | |
| 108 | "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", | |
| 109 | ) | |
| 110 | .all() | |
| 111 | .map((r) => r.name); | |
| 112 | return Object.fromEntries( | |
| 113 | tables.map((n) => [ | |
| 114 | n, | |
| 115 | app.db.query<{ c: number }, []>(`SELECT count(*) AS c FROM "${n}"`).get()!.c, | |
| 116 | ]), | |
| 117 | ); | |
| 118 | }; | |
| 119 | ||
| 120 | 107 | test("the list of skipped tools names only tools that exist", () => { |
| 121 | 108 | const names = new Set(toolNames()); |
| 122 | 109 | for (const name of Object.keys(SKIPPED)) expect(names.has(name)).toBe(true); |
| @@ -125,7 +112,8 @@ describe("a token that may only read cannot write through any tool", () => { | ||
| 125 | 112 | test("each write is refused for a read-only token, and nothing changes", async () => { |
| 126 | 113 | const writes = toolRegistry().filter((t) => t.writes && !(t.name in SKIPPED)); |
| 127 | 114 | expect(writes.length).toBeGreaterThan(50); |
| 128 | const before = snapshot(); | |
| 115 | const before = snapshot(app.db); | |
| 116 | const counted = ownCount(app.db); | |
| 129 | 117 | const events = app.db.query("SELECT * FROM event ORDER BY rowid").all(); |
| 130 | 118 | const unrefused: string[] = []; |
| 131 | 119 | for (const t of writes) { |
| @@ -139,7 +127,8 @@ describe("a token that may only read cannot write through any tool", () => { | ||
| 139 | 127 | if (!refused) unrefused.push(t.name); |
| 140 | 128 | } |
| 141 | 129 | expect(unrefused).toEqual([]); |
| 142 | expect(snapshot()).toEqual(before); | |
| 130 | expect(snapshot(app.db)).toEqual(before); | |
| 131 | expect(ownCount(app.db)).toBeGreaterThanOrEqual(counted); | |
| 143 | 132 | expect(app.db.query("SELECT * FROM event ORDER BY rowid").all()).toEqual(events); |
| 144 | 133 | }, 60_000); |
| 145 | 134 | |
| @@ -148,7 +137,8 @@ describe("a token that may only read cannot write through any tool", () => { | ||
| 148 | 137 | expect(admin.length).toBeGreaterThan(0); |
| 149 | 138 | // A new admin-scope tool must be given valid arguments here. |
| 150 | 139 | expect(admin.map((t) => t.name).sort()).toEqual(Object.keys(ADMIN_ARGS).sort()); |
| 151 | const before = snapshot(); | |
| 140 | const before = snapshot(app.db); | |
| 141 | const counted = ownCount(app.db); | |
| 152 | 142 | const events = app.db.query("SELECT * FROM event ORDER BY rowid").all(); |
| 153 | 143 | const answered: string[] = []; |
| 154 | 144 | for (const t of admin) { |
| @@ -156,7 +146,8 @@ describe("a token that may only read cannot write through any tool", () => { | ||
| 156 | 146 | if (!notListed(body)) answered.push(t.name); |
| 157 | 147 | } |
| 158 | 148 | expect(answered).toEqual([]); |
| 159 | expect(snapshot()).toEqual(before); | |
| 149 | expect(snapshot(app.db)).toEqual(before); | |
| 150 | expect(ownCount(app.db)).toBeGreaterThanOrEqual(counted); | |
| 160 | 151 | expect(app.db.query("SELECT * FROM event ORDER BY rowid").all()).toEqual(events); |
| 161 | 152 | }, 60_000); |
| 162 | 153 | }); |
test/write-snapshot.test.ts added +96
| @@ -0,0 +1,96 @@ | ||
| 1 | /** | |
| 2 | * The snapshot mcp-write-scope compares (#9031): a stray write to a | |
| 3 | * timer-flushed counter outside `/mcp` makes it differ, and the test's own | |
| 4 | * `/mcp` rows growing does not. Held to the real router and the real schemas. | |
| 5 | */ | |
| 6 | import { Database } from "bun:sqlite"; | |
| 7 | import { describe, expect, test } from "bun:test"; | |
| 8 | import routeTiming from "../src/core/migrations/20261019140000-route-timing.ts"; | |
| 9 | import usageCount from "../src/core/migrations/20261016130000-usage-count.ts"; | |
| 10 | import { mcpRoutes } from "../src/http/mcp.ts"; | |
| 11 | import { Router } from "../src/http/router.ts"; | |
| 12 | import { COUNTERS, MCP_PATTERN, ownCount, snapshot } from "./helpers/write-snapshot.ts"; | |
| 13 | ||
| 14 | const H = "2026-11-03T10"; | |
| 15 | ||
| 16 | function seeded(): Database { | |
| 17 | const db = new Database(":memory:"); | |
| 18 | db.exec(routeTiming); | |
| 19 | db.exec(usageCount); | |
| 20 | db.exec("CREATE TABLE note (id INTEGER PRIMARY KEY, body TEXT)"); | |
| 21 | db.query("INSERT INTO route_timing VALUES (?1, ?2, 2, 1, 900)").run(H, "/api/v1/repos/:owner/:name"); | |
| 22 | db.query("INSERT INTO route_timing VALUES (?1, ?2, 1, 3, 1200)").run(H, MCP_PATTERN); | |
| 23 | db.query("INSERT INTO usage_count VALUES (?1, ?2, 'GET', '2xx', 'user', 'api', 4)").run(H, "/api/v1/user"); | |
| 24 | db.query("INSERT INTO usage_count VALUES (?1, ?2, 'POST', '2xx', 'user', 'mcp', 5)").run(H, MCP_PATTERN); | |
| 25 | return db; | |
| 26 | } | |
| 27 | ||
| 28 | describe("the write-scope snapshot", () => { | |
| 29 | test("its key for MCP traffic is the pattern the router files /mcp under", () => { | |
| 30 | const router = new Router<any>(); | |
| 31 | mcpRoutes(router); | |
| 32 | expect(router.match("POST", "/mcp")?.pattern).toBe(MCP_PATTERN); | |
| 33 | }); | |
| 34 | ||
| 35 | test("it orders each counter by that table's real primary key", () => { | |
| 36 | const db = seeded(); | |
| 37 | for (const [table, key] of Object.entries(COUNTERS)) { | |
| 38 | const pk = db | |
| 39 | .query<{ name: string; pk: number }, []>(`SELECT name, pk FROM pragma_table_info('${table}') WHERE pk > 0 ORDER BY pk`) | |
| 40 | .all() | |
| 41 | .map((c) => c.name); | |
| 42 | expect(key).toEqual(pk); | |
| 43 | } | |
| 44 | }); | |
| 45 | ||
| 46 | test("an inserted row outside /mcp makes it differ", () => { | |
| 47 | for (const insert of [ | |
| 48 | "INSERT INTO route_timing VALUES ('2026-11-03T10', '/healthz', 0, 1, 10)", | |
| 49 | "INSERT INTO usage_count VALUES ('2026-11-03T10', '/api/v1/user', 'POST', '2xx', 'user', 'api', 1)", | |
| 50 | ]) { | |
| 51 | const db = seeded(); | |
| 52 | const before = snapshot(db); | |
| 53 | db.exec(insert); | |
| 54 | expect(snapshot(db)).not.toEqual(before); | |
| 55 | } | |
| 56 | }); | |
| 57 | ||
| 58 | test("a changed or removed row outside /mcp makes it differ", () => { | |
| 59 | for (const write of [ | |
| 60 | "UPDATE route_timing SET n = n + 1 WHERE pattern <> '/mcp'", | |
| 61 | "UPDATE usage_count SET n = n + 1 WHERE pattern <> '/mcp'", | |
| 62 | "DELETE FROM usage_count WHERE pattern <> '/mcp'", | |
| 63 | ]) { | |
| 64 | const db = seeded(); | |
| 65 | const before = snapshot(db); | |
| 66 | db.exec(write); | |
| 67 | expect(snapshot(db)).not.toEqual(before); | |
| 68 | } | |
| 69 | }); | |
| 70 | ||
| 71 | test("a row whose pattern only contains mcp is held exactly, not let grow", () => { | |
| 72 | const db = seeded(); | |
| 73 | db.query("INSERT INTO route_timing VALUES (?1, '/api/v1/mcp-tools', 0, 1, 10)").run(H); | |
| 74 | const before = snapshot(db); | |
| 75 | db.exec("UPDATE route_timing SET n = n + 1 WHERE pattern = '/api/v1/mcp-tools'"); | |
| 76 | expect(snapshot(db)).not.toEqual(before); | |
| 77 | }); | |
| 78 | ||
| 79 | test("the /mcp rows growing, or a new hour's /mcp row, leave it equal and raise the own count", () => { | |
| 80 | const db = seeded(); | |
| 81 | const before = snapshot(db); | |
| 82 | const counted = ownCount(db); | |
| 83 | db.exec("UPDATE route_timing SET n = n + 2, sum_us = sum_us + 50 WHERE pattern = '/mcp'"); | |
| 84 | db.exec("UPDATE usage_count SET n = n + 2 WHERE pattern = '/mcp'"); | |
| 85 | db.query("INSERT INTO route_timing VALUES ('2026-11-03T11', ?1, 1, 1, 400)").run(MCP_PATTERN); | |
| 86 | expect(snapshot(db)).toEqual(before); | |
| 87 | expect(ownCount(db)).toBe(counted + 5); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a new row in any other table changes its count", () => { | |
| 91 | const db = seeded(); | |
| 92 | const before = snapshot(db); | |
| 93 | db.exec("INSERT INTO note (body) VALUES ('x')"); | |
| 94 | expect(snapshot(db)).not.toEqual(before); | |
| 95 | }); | |
| 96 | }); |
new above
offline
Excluding these tables removes them from the no-write guarantee. Keep them in the snapshot but compare them tolerantly (e.g. counts may only change by the timer flush, or snapshot the non-counter columns), so a tool writing to them still fails the test.