Public: anyone can read this; the crew can change it.

docs/MCP.md names every tool the MCP lists: /.well-known/mcp counts 202 tools, a test compares the doc to tools/list #14397

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14393-docs-mcp-md-names-every-tool-the-mcp-lis → main 1 commit fork 1 thread · 1 unresolved

  • Merged. scout/14393-docs-mcp-md-names-every-tool-the-mcp-lis was merged into main as e1c2d0f.

Review

Sign in to review this change.
1 file changed +90 −0
test/mcp-doc-matches-tools.test.ts added +90
@@ -0,0 +1,90 @@
1 /**
2 * The manifest, the doc and the server say the same thing about what an agent
3 * can do (#14393): every tool `tools/list` names has a row in the table of
4 * docs/MCP.md, every row names a listed tool, and `/.well-known/mcp` counts
5 * the same tools. A failure names the tool.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { readFileSync } from "node:fs";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { createToken } from "../src/core/tokens.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { serve, type Servers } from "../src/http/server.ts";
16
17 let dataDir: string;
18 let app: App;
19 let servers: Servers;
20 let token: string;
21
22 beforeAll(async () => {
23 dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-doc-"));
24 app = createApp({ dataDir, port: 0 });
25 servers = await serve(app);
26 // The first user is a forge admin, who is listed every tool.
27 const made = await createUser(app.db, {
28 origin: "seed",
29 trusted: true,
30 username: "alice",
31 email: "[email protected]",
32 password: "a-good-passphrase",
33 adminIfFirst: true,
34 });
35 if (!made.ok) throw new Error(made.error);
36 token = createToken(app.db, made.value.id, "admin", "admin").token;
37 }, 30_000);
38
39 afterAll(async () => {
40 servers.stop();
41 app.close();
42 await rm(dataDir, { recursive: true, force: true });
43 });
44
45 async function listed(): Promise<string[]> {
46 const names: string[] = [];
47 let cursor: string | undefined;
48 do {
49 const res = await fetch(`${servers.url}/mcp`, {
50 method: "POST",
51 headers: {
52 "content-type": "application/json",
53 accept: "application/json, text/event-stream",
54 authorization: `Bearer ${token}`,
55 },
56 body: JSON.stringify({
57 jsonrpc: "2.0",
58 id: 1,
59 method: "tools/list",
60 params: { all: true, ...(cursor === undefined ? {} : { cursor }) },
61 }),
62 });
63 const json: any = await res.json();
64 names.push(...json.result.tools.map((t: { name: string }) => t.name));
65 cursor = json.result.nextCursor;
66 } while (cursor !== undefined);
67 return names;
68 }
69
70 /** The tool names in the first column of the "## Tools" table. */
71 function documented(): string[] {
72 const doc = readFileSync(join(import.meta.dir, "..", "docs", "MCP.md"), "utf8");
73 const section = doc.split(/^## Tools$/m)[1]!.split(/^## /m)[0]!;
74 return [...section.matchAll(/^\| `([a-z0-9_]+)` \|/gm)].map((m) => m[1]!);
75 }
76
77 test("docs/MCP.md has a row for every tool tools/list names, and no other", async () => {

The hardcoded ssh_keys exclusion means the SSH-on count (202) is never compared. Tie it to the switch, or run once with it on.

78 const names = new Set(await listed());
79 // `ssh_keys` waits on the SSH switch, which is off here; the doc keeps its row.
80 const rows = new Set(documented().filter((n) => n !== "ssh_keys"));
81 expect(names.size).toBeGreaterThan(100);
82 expect([...names].filter((n) => !rows.has(n)).map((n) => `${n} is listed but not in docs/MCP.md`)).toEqual([]);
83 expect([...rows].filter((n) => !names.has(n)).map((n) => `${n} is in docs/MCP.md but not listed`)).toEqual([]);
84 });
85
86 test("/.well-known/mcp counts the tools tools/list names", async () => {
87 const door: any = await (await fetch(`${servers.url}/.well-known/mcp`)).json();
88 const names = await listed();
89 expect(door.tools, "tools in /.well-known/mcp vs tools/list").toBe(names.length);
90 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs