Public: anyone can read this; the crew can change it.

Flaky: mcp-write-scope snapshot and the chrome click helper under load #9079

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/9031-flaky-mcp-write-scope-snapshot-and-the-c → main 3 commits fork 4 threads · 4 unresolved

  • Merged. scout/9031-flaky-mcp-write-scope-snapshot-and-the-c was merged into main as 50e4dc3.

Review

Sign in to review this change.
5 files changed +214 −22
test/chrome-retry.test.ts added +14
@@ -0,0 +1,14 @@
1 import { expect, test } from "bun:test";
2 import { retryStep } from "./helpers/chrome.ts";
3
4 test("a step that fails until the element appears is retried until it runs", async () => {
5 let calls = 0;
6 await retryStep(async () => (++calls < 4 ? "no such element" : undefined), Date.now() + 2_000, 1);
7 expect(calls).toBe(4);
8 });
9
10 test("a step that never succeeds throws its last failure at the deadline", async () => {
11 await expect(retryStep(async () => "no such element", Date.now() + 50, 5)).rejects.toThrow(
12 "step never ran: no such element",
13 );
14 });
test/helpers/chrome.ts +29 −2
@@ -339,6 +339,25 @@ export interface Followed {
339339 readonly errors: string[];
340340 }
341341
342 /**
343 * Run `attempt` until it reports no failure (`undefined`) or `deadline`
344 * passes, pausing `pauseMs` between tries: a click on an element the page
345 * has not drawn yet throws, and under load that is only "not yet". Throws
346 * the last failure when time runs out.
347 */
348 export async function retryStep(
349 attempt: () => Promise<string | undefined>,
350 deadline: number,
351 pauseMs = 50,
352 ): Promise<void> {
353 for (;;) {
354 const failure = await attempt();
355 if (failure === undefined) return;
356 if (Date.now() >= deadline) throw new Error(`step never ran: ${failure}`);
357 await Bun.sleep(pauseMs);
358 }
359 }
360
342361 /**
343362 * Load `url` with a session cookie, then run each of `steps` (an
344363 * expression evaluated in the page — a click) after the page before it has
@@ -432,8 +451,16 @@ export async function clickAndFollow(
432451 for (const step of input.steps) {
433452 if (reached()) break;
434453 loaded = cdp.once("Page.loadEventFired");
435 // As a person's click: a user gesture, so a link that opens a tab does.
436 await cdp.send("Runtime.evaluate", { expression: step, userGesture: true });
454 await retryStep(
455 async () => {
456 // As a person's click: a user gesture, so a link that opens a tab does.
457 const r = await cdp.send("Runtime.evaluate", { expression: step, userGesture: true });
458 return r.exceptionDetails === undefined
459 ? undefined
460 : String(r.exceptionDetails.exception?.description ?? r.exceptionDetails.text);
461 },
462 deadline,
463 );
437464 await Promise.race([loaded, Bun.sleep(Math.max(0, deadline - Date.now()))]);
438465 }
439466 while (!reached() && Date.now() < deadline) await Bun.sleep(100);
test/helpers/write-snapshot.ts added +64
@@ -0,0 +1,64 @@
1 /**
2 * What a database holds, read so that two reads around a run of refused
3 * writes can be compared (#9031). Every table is held to its row count,
4 * except the two counters a timer flushes on its own (`route_timing`,
5 * `usage_count`): there the `/mcp` rows, which count the test's own
6 * requests, are left out of the snapshot and summed by `ownCount` instead,
7 * and every other row is held exactly, in primary-key order.
8 *
9 * Why the exact rows cannot flake: a row only appears in these tables for a
10 * request the server answered, keyed by its route pattern. A server that
11 * only ever answers `POST /mcp` has nothing pending under any other pattern,
12 * so a flush between two reads touches `/mcp` rows alone; an hour turning
13 * over adds a new `/mcp` row, which is also left out; and the 30-day prune
14 * finds nothing in a database made minutes ago.
15 */
16 import type { Database } from "bun:sqlite";
17
18 /** The pattern the router files every MCP call under (`mcpRoutes` in src/http/mcp.ts). */
19 export const MCP_PATTERN = "/mcp";
20
21 /** Each timer-flushed counter, with its primary key in order (see its migration). */
22 export const COUNTERS: Record<string, readonly string[]> = {
23 route_timing: ["hour", "pattern", "bucket"],
24 usage_count: ["hour", "pattern", "method", "status", "role", "surface"],
25 };
26
27 export type Snapshot = Record<string, number | unknown[]>;
28
29 export function snapshot(db: Database): Snapshot {
30 const tables = db
31 .query<{ name: string }, []>(
32 "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
33 )
34 .all()
35 .map((r) => r.name);
36 return Object.fromEntries(
37 tables.map((n) => {
38 const key = COUNTERS[n];
39 return [
40 n,
41 key
42 ? db
43 .query(`SELECT * FROM "${n}" WHERE pattern IS NOT ?1 ORDER BY ${key.join(", ")}`)
44 .all(MCP_PATTERN)
45 : db.query<{ c: number }, []>(`SELECT count(*) AS c FROM "${n}"`).get()!.c,
46 ];
47 }),
48 );
49 }
50
51 /** Total counted under `/mcp` across the counters: it may rise, never fall. */
52 export function ownCount(db: Database): number {
53 let sum = 0;
54 for (const n of Object.keys(COUNTERS)) {
55 const exists = db
56 .query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1")
57 .get(n);
58 if (!exists) continue;
59 sum += db
60 .query<{ t: number }, [string]>(`SELECT coalesce(sum(n), 0) AS t FROM "${n}" WHERE pattern = ?1`)
61 .get(MCP_PATTERN)!.t;
62 }
63 return sum;
64 }
test/mcp-write-scope.test.ts +11 −20
@@ -1,7 +1,8 @@
11 /**
22 * Every MCP tool that writes, called with a read-only token, is refused
33 * before it writes; every tool that needs an admin token is refused to an
4 * `rw` one. The event log and every table's row count stay as they were.
4 * `rw` one. The event log, every table's row count and every counter row
5 * outside `/mcp` stay as they were (see test/helpers/write-snapshot.ts).
56 */
67 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
78 import { mkdtemp, rm } from "node:fs/promises";
@@ -13,6 +14,7 @@ import { createRepo } from "../src/actions/repos.ts";
1314 import { createUser } from "../src/core/users.ts";
1415 import { toolRegistry, toolNames } from "../src/http/mcp.ts";
1516 import { serve, type Servers } from "../src/http/server.ts";
17 import { ownCount, snapshot } from "./helpers/write-snapshot.ts";
1618
1719 /** Tools this test cannot call as a write, by name, and why. */
1820 const SKIPPED: Record<string, string> = {};
@@ -102,21 +104,6 @@ describe("a token that may only read cannot write through any tool", () => {
102104 return args;
103105 };
104106
outdated

Excluding these tables removes them from the no-write guarantee. Keep them in the snapshot but compare them tolerantly (e.g. counts may only change by the timer flush, or snapshot the non-counter columns), so a tool writing to them still fails the test.

105 const snapshot = () => {
106 const tables = app.db
107 .query<{ name: string }, []>(
108 "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
109 )
110 .all()
111 .map((r) => r.name);
112 return Object.fromEntries(
113 tables.map((n) => [
114 n,
115 app.db.query<{ c: number }, []>(`SELECT count(*) AS c FROM "${n}"`).get()!.c,
116 ]),
117 );
118 };
119
120107 test("the list of skipped tools names only tools that exist", () => {
121108 const names = new Set(toolNames());
122109 for (const name of Object.keys(SKIPPED)) expect(names.has(name)).toBe(true);
@@ -125,7 +112,8 @@ describe("a token that may only read cannot write through any tool", () => {
125112 test("each write is refused for a read-only token, and nothing changes", async () => {
126113 const writes = toolRegistry().filter((t) => t.writes && !(t.name in SKIPPED));
127114 expect(writes.length).toBeGreaterThan(50);
128 const before = snapshot();
115 const before = snapshot(app.db);
116 const counted = ownCount(app.db);
129117 const events = app.db.query("SELECT * FROM event ORDER BY rowid").all();
130118 const unrefused: string[] = [];
131119 for (const t of writes) {
@@ -139,7 +127,8 @@ describe("a token that may only read cannot write through any tool", () => {
139127 if (!refused) unrefused.push(t.name);
140128 }
141129 expect(unrefused).toEqual([]);
142 expect(snapshot()).toEqual(before);
130 expect(snapshot(app.db)).toEqual(before);
131 expect(ownCount(app.db)).toBeGreaterThanOrEqual(counted);
143132 expect(app.db.query("SELECT * FROM event ORDER BY rowid").all()).toEqual(events);
144133 }, 60_000);
145134
@@ -148,7 +137,8 @@ describe("a token that may only read cannot write through any tool", () => {
148137 expect(admin.length).toBeGreaterThan(0);
149138 // A new admin-scope tool must be given valid arguments here.
150139 expect(admin.map((t) => t.name).sort()).toEqual(Object.keys(ADMIN_ARGS).sort());
151 const before = snapshot();
140 const before = snapshot(app.db);
141 const counted = ownCount(app.db);
152142 const events = app.db.query("SELECT * FROM event ORDER BY rowid").all();
153143 const answered: string[] = [];
154144 for (const t of admin) {
@@ -156,7 +146,8 @@ describe("a token that may only read cannot write through any tool", () => {
156146 if (!notListed(body)) answered.push(t.name);
157147 }
158148 expect(answered).toEqual([]);
159 expect(snapshot()).toEqual(before);
149 expect(snapshot(app.db)).toEqual(before);
150 expect(ownCount(app.db)).toBeGreaterThanOrEqual(counted);
160151 expect(app.db.query("SELECT * FROM event ORDER BY rowid").all()).toEqual(events);
161152 }, 60_000);
162153 });
test/write-snapshot.test.ts added +96
@@ -0,0 +1,96 @@
1 /**
2 * The snapshot mcp-write-scope compares (#9031): a stray write to a
3 * timer-flushed counter outside `/mcp` makes it differ, and the test's own
4 * `/mcp` rows growing does not. Held to the real router and the real schemas.
5 */
6 import { Database } from "bun:sqlite";
7 import { describe, expect, test } from "bun:test";
8 import routeTiming from "../src/core/migrations/20261019140000-route-timing.ts";
9 import usageCount from "../src/core/migrations/20261016130000-usage-count.ts";
10 import { mcpRoutes } from "../src/http/mcp.ts";
11 import { Router } from "../src/http/router.ts";
12 import { COUNTERS, MCP_PATTERN, ownCount, snapshot } from "./helpers/write-snapshot.ts";
13
14 const H = "2026-11-03T10";
15
16 function seeded(): Database {
17 const db = new Database(":memory:");
18 db.exec(routeTiming);
19 db.exec(usageCount);
20 db.exec("CREATE TABLE note (id INTEGER PRIMARY KEY, body TEXT)");
21 db.query("INSERT INTO route_timing VALUES (?1, ?2, 2, 1, 900)").run(H, "/api/v1/repos/:owner/:name");
22 db.query("INSERT INTO route_timing VALUES (?1, ?2, 1, 3, 1200)").run(H, MCP_PATTERN);
23 db.query("INSERT INTO usage_count VALUES (?1, ?2, 'GET', '2xx', 'user', 'api', 4)").run(H, "/api/v1/user");
24 db.query("INSERT INTO usage_count VALUES (?1, ?2, 'POST', '2xx', 'user', 'mcp', 5)").run(H, MCP_PATTERN);
25 return db;
26 }
27
28 describe("the write-scope snapshot", () => {
29 test("its key for MCP traffic is the pattern the router files /mcp under", () => {
30 const router = new Router<any>();
31 mcpRoutes(router);
32 expect(router.match("POST", "/mcp")?.pattern).toBe(MCP_PATTERN);
33 });
34
35 test("it orders each counter by that table's real primary key", () => {
36 const db = seeded();
37 for (const [table, key] of Object.entries(COUNTERS)) {
38 const pk = db
39 .query<{ name: string; pk: number }, []>(`SELECT name, pk FROM pragma_table_info('${table}') WHERE pk > 0 ORDER BY pk`)
40 .all()
41 .map((c) => c.name);
42 expect(key).toEqual(pk);
43 }
44 });
45
46 test("an inserted row outside /mcp makes it differ", () => {
47 for (const insert of [
48 "INSERT INTO route_timing VALUES ('2026-11-03T10', '/healthz', 0, 1, 10)",
49 "INSERT INTO usage_count VALUES ('2026-11-03T10', '/api/v1/user', 'POST', '2xx', 'user', 'api', 1)",
50 ]) {
51 const db = seeded();
52 const before = snapshot(db);
53 db.exec(insert);
54 expect(snapshot(db)).not.toEqual(before);
55 }
56 });
57
58 test("a changed or removed row outside /mcp makes it differ", () => {
59 for (const write of [
60 "UPDATE route_timing SET n = n + 1 WHERE pattern <> '/mcp'",
61 "UPDATE usage_count SET n = n + 1 WHERE pattern <> '/mcp'",
62 "DELETE FROM usage_count WHERE pattern <> '/mcp'",
63 ]) {
64 const db = seeded();
65 const before = snapshot(db);
66 db.exec(write);
67 expect(snapshot(db)).not.toEqual(before);
68 }
69 });
70
71 test("a row whose pattern only contains mcp is held exactly, not let grow", () => {
72 const db = seeded();
73 db.query("INSERT INTO route_timing VALUES (?1, '/api/v1/mcp-tools', 0, 1, 10)").run(H);
74 const before = snapshot(db);
75 db.exec("UPDATE route_timing SET n = n + 1 WHERE pattern = '/api/v1/mcp-tools'");
76 expect(snapshot(db)).not.toEqual(before);
77 });
78
79 test("the /mcp rows growing, or a new hour's /mcp row, leave it equal and raise the own count", () => {
80 const db = seeded();
81 const before = snapshot(db);
82 const counted = ownCount(db);
83 db.exec("UPDATE route_timing SET n = n + 2, sum_us = sum_us + 50 WHERE pattern = '/mcp'");
84 db.exec("UPDATE usage_count SET n = n + 2 WHERE pattern = '/mcp'");
85 db.query("INSERT INTO route_timing VALUES ('2026-11-03T11', ?1, 1, 1, 400)").run(MCP_PATTERN);
86 expect(snapshot(db)).toEqual(before);
87 expect(ownCount(db)).toBe(counted + 5);
88 });
89
90 test("a new row in any other table changes its count", () => {
91 const db = seeded();
92 const before = snapshot(db);
93 db.exec("INSERT INTO note (body) VALUES ('x')");
94 expect(snapshot(db)).not.toEqual(before);
95 });
96 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs