Public: anyone can read this; the crew can change it.

llms.txt and /.well-known/mcp.json name the same MCP tools, and a test fails when they drift #10940

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/10937-llms-txt-and-well-known-mcp-json-name-th → main 1 commit fork 2 threads · 2 unresolved

  • Merged. scout/10937-llms-txt-and-well-known-mcp-json-name-th was merged into main as 704c797.

Review

Sign in to review this change.
1 file changed +95 −0
test/mcp-doors.test.ts added +95
@@ -0,0 +1,95 @@
1 /** An agent reading llms.txt, /.well-known/mcp.json or the MCP's own tools/list sees the same tools; a drift names the tool and the door that lacks it. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import { createToken } from "../src/core/tokens.ts";
9 import { createUser } from "../src/core/users.ts";
10
11 type Door = { name: string; tools: Set<string> };
12
13 /** One message per tool missing from a door, or none when the doors agree. */
14 export function drift(doors: Door[]): string[] {

Nothing imports drift; drop export.

15 const all = new Set(doors.flatMap((d) => [...d.tools]));
16 return [...all].sort().flatMap((tool) => doors.filter((d) => !d.tools.has(tool)).map((d) => `${tool}: missing from ${d.name}`));
17 }
18
19 describe("the doors onto the MCP tools", () => {
20 let dataDir: string;
21 let servers: Servers;
22 let admin: string;
23 let member: string;
24
25 beforeAll(async () => {
26 dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-doors-"));
27 const app = createApp({ dataDir, port: 0 });
28 servers = await serve(app);
29 const make = async (username: string, adminIfFirst: boolean) => {
30 const made = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst });
31 if (!made.ok) throw new Error(made.error);
32 return createToken(app.db, made.value.id, "doors", adminIfFirst ? "admin" : "rw").token;
33 };
34 admin = await make("alice", true);
35 member = await make("bob", false);
36 });
37 afterAll(async () => {
38 await servers.stop();
39 await rm(dataDir, { recursive: true, force: true });
40 });
41
42 const listed = async (token: string | undefined) => {
43 const res = await fetch(`${servers.url}/llms.txt`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
44 const lines = (await res.text()).split("\n");
45 const start = lines.indexOf("## MCP tools");
46 expect(start).toBeGreaterThan(-1);
47 const end = lines.findIndex((l, i) => i > start && l.startsWith("## "));
48 return lines
49 .slice(start + 1, end)
50 .map((l) => /^- ([a-z][\w-]*): (.*)$/.exec(l))
51 .filter((m): m is RegExpExecArray => m !== null)
52 .map((m) => ({ name: m[1]!, admin: m[2]!.includes("(admin)") }));
53 };
54 const served = async (token: string) => {
55 const names: string[] = [];
56 let cursor: string | undefined;
57 do {
58 const res = await fetch(`${servers.url}/mcp`, {
59 method: "POST",
60 headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
61 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: cursor ? { cursor } : {} }),
62 });
63 const r = ((await res.json()) as any).result;
64 names.push(...r.tools.map((t: any) => t.name));
65 cursor = r.nextCursor;
66 } while (cursor !== undefined);
67 return names;
68 };
69 const card = async () => ((await (await fetch(`${servers.url}/.well-known/mcp.json`)).json()) as any).tokenFreeTools as string[];
70
71 test("an admin sees every tool at tools/list and in llms.txt", async () => {
72 const llms = new Set((await listed(admin)).map((t) => t.name));
73 const mcp = new Set(await served(admin));
74 expect(drift([{ name: "llms.txt", tools: llms }, { name: "tools/list", tools: mcp }])).toEqual([]);
75 });
76
77 test("a member's tools/list is llms.txt without the tools it marks (admin)", async () => {
78 const llms = await listed(member);
79 const open = new Set(llms.filter((t) => !t.admin).map((t) => t.name));
80 const mcp = new Set(await served(member));
81 expect(drift([{ name: "llms.txt (not admin)", tools: open }, { name: "tools/list", tools: mcp }])).toEqual([]);
82 });
83

Hard to read: build the filtered tools/list door in a named const, then pass it to drift.

84 test("a stranger sees the same token-free tools in llms.txt and mcp.json, and a member's tools/list carries them", async () => {
85 const llms = new Set((await listed(undefined)).map((t) => t.name));
86 const free = new Set(await card());
87 expect(drift([{ name: "llms.txt", tools: llms }, { name: "mcp.json", tools: free }])).toEqual([]);
88 const mcp = new Set(await served(member));
89 expect(drift([{ name: "tools/list", tools: mcp }, { name: "mcp.json", tools: free }].map((d) => (d.name === "tools/list" ? { ...d, tools: new Set([...d.tools].filter((n) => free.has(n))) } : d)))).toEqual([]);
90 });
91
92 test("a tool missing from one door is named with that door", () => {
93 expect(drift([{ name: "llms.txt", tools: new Set(["a", "b"]) }, { name: "mcp.json", tools: new Set(["a"]) }])).toEqual(["b: missing from mcp.json"]);
94 });
95 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs