Public: anyone can read this; the crew can change it.
llms.txt and /.well-known/mcp.json name the same MCP tools, and a test fails when they drift #10940
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10937-llms-txt-and-well-known-mcp-json-name-th → main 1 commit fork 2 threads · 2 unresolved
1 file
changed +95
−0
test/mcp-doors.test.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | /** An agent reading llms.txt, /.well-known/mcp.json or the MCP's own tools/list sees the same tools; a drift names the tool and the door that lacks it. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | ||
| 11 | type Door = { name: string; tools: Set<string> }; | |
| 12 | ||
| 13 | /** One message per tool missing from a door, or none when the doors agree. */ | |
| 14 | export function drift(doors: Door[]): string[] { | |
| 15 | const all = new Set(doors.flatMap((d) => [...d.tools])); | |
| 16 | return [...all].sort().flatMap((tool) => doors.filter((d) => !d.tools.has(tool)).map((d) => `${tool}: missing from ${d.name}`)); | |
| 17 | } | |
| 18 | ||
| 19 | describe("the doors onto the MCP tools", () => { | |
| 20 | let dataDir: string; | |
| 21 | let servers: Servers; | |
| 22 | let admin: string; | |
| 23 | let member: string; | |
| 24 | ||
| 25 | beforeAll(async () => { | |
| 26 | dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-doors-")); | |
| 27 | const app = createApp({ dataDir, port: 0 }); | |
| 28 | servers = await serve(app); | |
| 29 | const make = async (username: string, adminIfFirst: boolean) => { | |
| 30 | const made = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst }); | |
| 31 | if (!made.ok) throw new Error(made.error); | |
| 32 | return createToken(app.db, made.value.id, "doors", adminIfFirst ? "admin" : "rw").token; | |
| 33 | }; | |
| 34 | admin = await make("alice", true); | |
| 35 | member = await make("bob", false); | |
| 36 | }); | |
| 37 | afterAll(async () => { | |
| 38 | await servers.stop(); | |
| 39 | await rm(dataDir, { recursive: true, force: true }); | |
| 40 | }); | |
| 41 | ||
| 42 | const listed = async (token: string | undefined) => { | |
| 43 | const res = await fetch(`${servers.url}/llms.txt`, { headers: token ? { authorization: `Bearer ${token}` } : {} }); | |
| 44 | const lines = (await res.text()).split("\n"); | |
| 45 | const start = lines.indexOf("## MCP tools"); | |
| 46 | expect(start).toBeGreaterThan(-1); | |
| 47 | const end = lines.findIndex((l, i) => i > start && l.startsWith("## ")); | |
| 48 | return lines | |
| 49 | .slice(start + 1, end) | |
| 50 | .map((l) => /^- ([a-z][\w-]*): (.*)$/.exec(l)) | |
| 51 | .filter((m): m is RegExpExecArray => m !== null) | |
| 52 | .map((m) => ({ name: m[1]!, admin: m[2]!.includes("(admin)") })); | |
| 53 | }; | |
| 54 | const served = async (token: string) => { | |
| 55 | const names: string[] = []; | |
| 56 | let cursor: string | undefined; | |
| 57 | do { | |
| 58 | const res = await fetch(`${servers.url}/mcp`, { | |
| 59 | method: "POST", | |
| 60 | headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, | |
| 61 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: cursor ? { cursor } : {} }), | |
| 62 | }); | |
| 63 | const r = ((await res.json()) as any).result; | |
| 64 | names.push(...r.tools.map((t: any) => t.name)); | |
| 65 | cursor = r.nextCursor; | |
| 66 | } while (cursor !== undefined); | |
| 67 | return names; | |
| 68 | }; | |
| 69 | const card = async () => ((await (await fetch(`${servers.url}/.well-known/mcp.json`)).json()) as any).tokenFreeTools as string[]; | |
| 70 | ||
| 71 | test("an admin sees every tool at tools/list and in llms.txt", async () => { | |
| 72 | const llms = new Set((await listed(admin)).map((t) => t.name)); | |
| 73 | const mcp = new Set(await served(admin)); | |
| 74 | expect(drift([{ name: "llms.txt", tools: llms }, { name: "tools/list", tools: mcp }])).toEqual([]); | |
| 75 | }); | |
| 76 | ||
| 77 | test("a member's tools/list is llms.txt without the tools it marks (admin)", async () => { | |
| 78 | const llms = await listed(member); | |
| 79 | const open = new Set(llms.filter((t) => !t.admin).map((t) => t.name)); | |
| 80 | const mcp = new Set(await served(member)); | |
| 81 | expect(drift([{ name: "llms.txt (not admin)", tools: open }, { name: "tools/list", tools: mcp }])).toEqual([]); | |
| 82 | }); | |
| 83 | ||
Hard to read: build the filtered tools/list door in a named const, then pass it to drift. |
||
| 84 | test("a stranger sees the same token-free tools in llms.txt and mcp.json, and a member's tools/list carries them", async () => { | |
| 85 | const llms = new Set((await listed(undefined)).map((t) => t.name)); | |
| 86 | const free = new Set(await card()); | |
| 87 | expect(drift([{ name: "llms.txt", tools: llms }, { name: "mcp.json", tools: free }])).toEqual([]); | |
| 88 | const mcp = new Set(await served(member)); | |
| 89 | expect(drift([{ name: "tools/list", tools: mcp }, { name: "mcp.json", tools: free }].map((d) => (d.name === "tools/list" ? { ...d, tools: new Set([...d.tools].filter((n) => free.has(n))) } : d)))).toEqual([]); | |
| 90 | }); | |
| 91 | ||
| 92 | test("a tool missing from one door is named with that door", () => { | |
| 93 | expect(drift([{ name: "llms.txt", tools: new Set(["a", "b"]) }, { name: "mcp.json", tools: new Set(["a"]) }])).toEqual(["b: missing from mcp.json"]); | |
| 94 | }); | |
| 95 | }); |
Nothing imports
drift; dropexport.