Public: anyone can read this; the crew can change it.
docs/MCP.md names every tool the MCP lists: /.well-known/mcp counts 202 tools, a test compares the doc to tools/list #14397
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14393-docs-mcp-md-names-every-tool-the-mcp-lis → main 1 commit fork 1 thread · 1 unresolved
1 file
changed +90
−0
test/mcp-doc-matches-tools.test.ts added +90
| @@ -0,0 +1,90 @@ | ||
| 1 | /** | |
| 2 | * The manifest, the doc and the server say the same thing about what an agent | |
| 3 | * can do (#14393): every tool `tools/list` names has a row in the table of | |
| 4 | * docs/MCP.md, every row names a listed tool, and `/.well-known/mcp` counts | |
| 5 | * the same tools. A failure names the tool. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { readFileSync } from "node:fs"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { createToken } from "../src/core/tokens.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 16 | ||
| 17 | let dataDir: string; | |
| 18 | let app: App; | |
| 19 | let servers: Servers; | |
| 20 | let token: string; | |
| 21 | ||
| 22 | beforeAll(async () => { | |
| 23 | dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-doc-")); | |
| 24 | app = createApp({ dataDir, port: 0 }); | |
| 25 | servers = await serve(app); | |
| 26 | // The first user is a forge admin, who is listed every tool. | |
| 27 | const made = await createUser(app.db, { | |
| 28 | origin: "seed", | |
| 29 | trusted: true, | |
| 30 | username: "alice", | |
| 31 | email: "[email protected]", | |
| 32 | password: "a-good-passphrase", | |
| 33 | adminIfFirst: true, | |
| 34 | }); | |
| 35 | if (!made.ok) throw new Error(made.error); | |
| 36 | token = createToken(app.db, made.value.id, "admin", "admin").token; | |
| 37 | }, 30_000); | |
| 38 | ||
| 39 | afterAll(async () => { | |
| 40 | servers.stop(); | |
| 41 | app.close(); | |
| 42 | await rm(dataDir, { recursive: true, force: true }); | |
| 43 | }); | |
| 44 | ||
| 45 | async function listed(): Promise<string[]> { | |
| 46 | const names: string[] = []; | |
| 47 | let cursor: string | undefined; | |
| 48 | do { | |
| 49 | const res = await fetch(`${servers.url}/mcp`, { | |
| 50 | method: "POST", | |
| 51 | headers: { | |
| 52 | "content-type": "application/json", | |
| 53 | accept: "application/json, text/event-stream", | |
| 54 | authorization: `Bearer ${token}`, | |
| 55 | }, | |
| 56 | body: JSON.stringify({ | |
| 57 | jsonrpc: "2.0", | |
| 58 | id: 1, | |
| 59 | method: "tools/list", | |
| 60 | params: { all: true, ...(cursor === undefined ? {} : { cursor }) }, | |
| 61 | }), | |
| 62 | }); | |
| 63 | const json: any = await res.json(); | |
| 64 | names.push(...json.result.tools.map((t: { name: string }) => t.name)); | |
| 65 | cursor = json.result.nextCursor; | |
| 66 | } while (cursor !== undefined); | |
| 67 | return names; | |
| 68 | } | |
| 69 | ||
| 70 | /** The tool names in the first column of the "## Tools" table. */ | |
| 71 | function documented(): string[] { | |
| 72 | const doc = readFileSync(join(import.meta.dir, "..", "docs", "MCP.md"), "utf8"); | |
| 73 | const section = doc.split(/^## Tools$/m)[1]!.split(/^## /m)[0]!; | |
| 74 | return [...section.matchAll(/^\| `([a-z0-9_]+)` \|/gm)].map((m) => m[1]!); | |
| 75 | } | |
| 76 | ||
| 77 | test("docs/MCP.md has a row for every tool tools/list names, and no other", async () => { | |
| 78 | const names = new Set(await listed()); | |
| 79 | // `ssh_keys` waits on the SSH switch, which is off here; the doc keeps its row. | |
| 80 | const rows = new Set(documented().filter((n) => n !== "ssh_keys")); | |
| 81 | expect(names.size).toBeGreaterThan(100); | |
| 82 | expect([...names].filter((n) => !rows.has(n)).map((n) => `${n} is listed but not in docs/MCP.md`)).toEqual([]); | |
| 83 | expect([...rows].filter((n) => !names.has(n)).map((n) => `${n} is in docs/MCP.md but not listed`)).toEqual([]); | |
| 84 | }); | |
| 85 | ||
| 86 | test("/.well-known/mcp counts the tools tools/list names", async () => { | |
| 87 | const door: any = await (await fetch(`${servers.url}/.well-known/mcp`)).json(); | |
| 88 | const names = await listed(); | |
| 89 | expect(door.tools, "tools in /.well-known/mcp vs tools/list").toBe(names.length); | |
| 90 | }); |
new above
offline
The hardcoded ssh_keys exclusion means the SSH-on count (202) is never compared. Tie it to the switch, or run once with it on.