Public: anyone can read this; the crew can change it.
A test fails when an MCP tool has no row in docs/MCP.md #5292
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/5279-a-test-fails-when-an-mcp-tool-has-no-row → main 1 commit fork
1 file
changed +130
−0
test/mcp-docs.test.ts added +130
| @@ -0,0 +1,130 @@ | ||
| 1 | /** docs/MCP.md never falls behind the server: every tool in tools/list has a row, every row names a tool. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { createToken } from "../src/core/tokens.ts"; | |
| 8 | import { createUser } from "../src/core/users.ts"; | |
| 9 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 10 | import { quietLogs } from "./helpers/globals.ts"; | |
| 11 | ||
| 12 | quietLogs(); | |
| 13 | ||
| 14 | /** | |
| 15 | * Tools that answer in tools/list and have no row yet. This list only shrinks: | |
| 16 | * document a tool, then delete its line. One name per line, so two pulls never edit the same line. | |
| 17 | */ | |
| 18 | const undocumented = new Set<string>([ | |
| 19 | "sizes", | |
| 20 | "here", | |
| 21 | "share", | |
| 22 | "variant", | |
| 23 | "asks", | |
| 24 | "asks_act", | |
| 25 | "asks_decide", | |
| 26 | "asks_proposal", | |
| 27 | "proposal_tokens", | |
| 28 | "asks_undecide", | |
| 29 | "issue_teams", | |
| 30 | "model_grant", | |
| 31 | "whoami_can", | |
| 32 | "mint_invite", | |
| 33 | "account_role", | |
| 34 | "model_ceiling", | |
| 35 | "forges", | |
| 36 | "sever", | |
| 37 | "follow_mode", | |
| 38 | "propose_to_parent_forge", | |
| 39 | "forge_proposals", | |
| 40 | "family", | |
| 41 | "next_steps", | |
| 42 | "thought", | |
| 43 | "feed_chips", | |
| 44 | "returns", | |
| 45 | "spend", | |
| 46 | "attach_look", | |
| 47 | "access", | |
| 48 | "capacity", | |
| 49 | "scorecard", | |
| 50 | "flow", | |
| 51 | "system", | |
| 52 | "contribute", | |
| 53 | "limits", | |
| 54 | "teams", | |
| 55 | "cloud_accounts", | |
| 56 | "cloud_account", | |
| 57 | "quotas", | |
| 58 | "signup", | |
| 59 | "boards", | |
| 60 | "board_write", | |
| 61 | "board_pins", | |
| 62 | "board_delete", | |
| 63 | "report", | |
| 64 | ]); | |
| 65 | ||
| 66 | let dataDir: string; | |
| 67 | let servers: Servers; | |
| 68 | let served: string[]; | |
| 69 | ||
| 70 | beforeAll(async () => { | |
| 71 | dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-docs-")); | |
| 72 | const app = createApp({ dataDir, port: 0 }); | |
| 73 | servers = await serve(app); | |
| 74 | const made = await createUser(app.db, { | |
| 75 | origin: "seed", | |
| 76 | trusted: true, | |
| 77 | username: "ada", | |
| 78 | email: "[email protected]", | |
| 79 | password: "a-good-passphrase", | |
| 80 | adminIfFirst: true, | |
| 81 | }); | |
| 82 | if (!made.ok) throw new Error(JSON.stringify(made.error)); | |
| 83 | const token = createToken(app.db, made.value.id, "admin", "admin").token; | |
| 84 | const res = await fetch(`${servers.url}/mcp`, { | |
| 85 | method: "POST", | |
| 86 | headers: { | |
| 87 | "content-type": "application/json", | |
| 88 | accept: "application/json, text/event-stream", | |
| 89 | authorization: `Bearer ${token}`, | |
| 90 | }, | |
| 91 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), | |
| 92 | }); | |
| 93 | served = JSON.parse(await res.text()).result.tools.map((t: { name: string }) => t.name); | |
| 94 | }); | |
| 95 | ||
| 96 | afterAll(async () => { | |
| 97 | await servers.stop(); | |
| 98 | await rm(dataDir, { recursive: true, force: true }); | |
| 99 | }); | |
| 100 | ||
| 101 | async function documented(): Promise<string[]> { | |
| 102 | const md = await Bun.file(join(import.meta.dir, "..", "docs", "MCP.md")).text(); | |
| 103 | const names: string[] = []; | |
| 104 | let inTable = false; | |
| 105 | for (const line of md.split("\n")) { | |
| 106 | if (/^\|\s*tool\s*\|/.test(line)) inTable = true; | |
| 107 | else if (inTable && !line.startsWith("|")) break; | |
| 108 | else if (inTable) { | |
| 109 | const m = /^\|\s*`([^`]+)`\s*\|/.exec(line); | |
| 110 | if (m) names.push(m[1]!); | |
| 111 | } | |
| 112 | } | |
| 113 | return names; | |
| 114 | } | |
| 115 | ||
| 116 | test("every tool in tools/list has a row in docs/MCP.md", async () => { | |
| 117 | const rows = new Set(await documented()); | |
| 118 | expect(rows.size).toBeGreaterThan(0); | |
| 119 | expect(served.filter((n) => !rows.has(n) && !undocumented.has(n))).toEqual([]); | |
| 120 | }); | |
| 121 | ||
| 122 | test("every row in docs/MCP.md names a tool that exists", async () => { | |
| 123 | const live = new Set(served); | |
| 124 | expect((await documented()).filter((n) => !live.has(n))).toEqual([]); | |
| 125 | }); | |
| 126 | ||
| 127 | test("the allowlist names only undocumented tools", async () => { | |
| 128 | const rows = new Set(await documented()); | |
| 129 | expect([...undocumented].filter((n) => rows.has(n) || !served.includes(n))).toEqual([]); | |
| 130 | }); |
new above
offline