Public: anyone can read this; the crew can change it.

A test fails when an MCP tool has no row in docs/MCP.md #5292

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/5279-a-test-fails-when-an-mcp-tool-has-no-row → main 1 commit fork

  • Merged. scout/5279-a-test-fails-when-an-mcp-tool-has-no-row was merged into main as 6639098.

Review

Sign in to review this change.
1 file changed +130 −0
test/mcp-docs.test.ts added +130
@@ -0,0 +1,130 @@
1 /** docs/MCP.md never falls behind the server: every tool in tools/list has a row, every row names a tool. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp } from "../src/app.ts";
7 import { createToken } from "../src/core/tokens.ts";
8 import { createUser } from "../src/core/users.ts";
9 import { serve, type Servers } from "../src/http/server.ts";
10 import { quietLogs } from "./helpers/globals.ts";
11
12 quietLogs();
13
14 /**
15 * Tools that answer in tools/list and have no row yet. This list only shrinks:
16 * document a tool, then delete its line. One name per line, so two pulls never edit the same line.
17 */
18 const undocumented = new Set<string>([
19 "sizes",
20 "here",
21 "share",
22 "variant",
23 "asks",
24 "asks_act",
25 "asks_decide",
26 "asks_proposal",
27 "proposal_tokens",
28 "asks_undecide",
29 "issue_teams",
30 "model_grant",
31 "whoami_can",
32 "mint_invite",
33 "account_role",
34 "model_ceiling",
35 "forges",
36 "sever",
37 "follow_mode",
38 "propose_to_parent_forge",
39 "forge_proposals",
40 "family",
41 "next_steps",
42 "thought",
43 "feed_chips",
44 "returns",
45 "spend",
46 "attach_look",
47 "access",
48 "capacity",
49 "scorecard",
50 "flow",
51 "system",
52 "contribute",
53 "limits",
54 "teams",
55 "cloud_accounts",
56 "cloud_account",
57 "quotas",
58 "signup",
59 "boards",
60 "board_write",
61 "board_pins",
62 "board_delete",
63 "report",
64 ]);
65
66 let dataDir: string;
67 let servers: Servers;
68 let served: string[];
69
70 beforeAll(async () => {
71 dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-docs-"));
72 const app = createApp({ dataDir, port: 0 });
73 servers = await serve(app);
74 const made = await createUser(app.db, {
75 origin: "seed",
76 trusted: true,
77 username: "ada",
78 email: "[email protected]",
79 password: "a-good-passphrase",
80 adminIfFirst: true,
81 });
82 if (!made.ok) throw new Error(JSON.stringify(made.error));
83 const token = createToken(app.db, made.value.id, "admin", "admin").token;
84 const res = await fetch(`${servers.url}/mcp`, {
85 method: "POST",
86 headers: {
87 "content-type": "application/json",
88 accept: "application/json, text/event-stream",
89 authorization: `Bearer ${token}`,
90 },
91 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }),
92 });
93 served = JSON.parse(await res.text()).result.tools.map((t: { name: string }) => t.name);
94 });
95
96 afterAll(async () => {
97 await servers.stop();
98 await rm(dataDir, { recursive: true, force: true });
99 });
100
101 async function documented(): Promise<string[]> {
102 const md = await Bun.file(join(import.meta.dir, "..", "docs", "MCP.md")).text();
103 const names: string[] = [];
104 let inTable = false;
105 for (const line of md.split("\n")) {
106 if (/^\|\s*tool\s*\|/.test(line)) inTable = true;
107 else if (inTable && !line.startsWith("|")) break;
108 else if (inTable) {
109 const m = /^\|\s*`([^`]+)`\s*\|/.exec(line);
110 if (m) names.push(m[1]!);
111 }
112 }
113 return names;
114 }
115
116 test("every tool in tools/list has a row in docs/MCP.md", async () => {
117 const rows = new Set(await documented());
118 expect(rows.size).toBeGreaterThan(0);
119 expect(served.filter((n) => !rows.has(n) && !undocumented.has(n))).toEqual([]);
120 });
121
122 test("every row in docs/MCP.md names a tool that exists", async () => {
123 const live = new Set(served);
124 expect((await documented()).filter((n) => !live.has(n))).toEqual([]);
125 });
126
127 test("the allowlist names only undocumented tools", async () => {
128 const rows = new Set(await documented());
129 expect([...undocumented].filter((n) => rows.has(n) || !served.includes(n))).toEqual([]);
130 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs