Public: anyone can read this; the crew can change it.
/llms.txt is token-efficient for a stranger: the 170-line MCP tool list is folded to what they can use, plus one line for the rest #8294
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8289-llms-txt-is-token-efficient-for-a-strang → main 4 commits fork 3 threads · 3 unresolved
5 files
changed +137
−16
src/http/server.ts +1 −1
| @@ -1186,7 +1186,7 @@ export function buildRouter(): Router<Handler> { | ||
| 1186 | 1186 | // reclaim a path under `/api/`. |
| 1187 | 1187 | // The forge as an MCP server: the studio's verbs for an assistant. |
| 1188 | 1188 | mcpRoutes(router); |
| 1189 | setMcpTools([...TOOLS, REPORT_TOOL]); | |
| 1189 | setMcpTools(TOOLS, [REPORT_TOOL]); | |
| 1190 | 1190 | // The public API as OpenAPI, for any client that reads it. |
| 1191 | 1191 | openapiRoute(router); |
| 1192 | 1192 | oauthRoutes(router); |
src/web/llms_tools.ts +23 −4
| @@ -3,7 +3,8 @@ | ||
| 3 | 3 | * index and `report`, `name: doc` (its `description` when it has no `doc`), |
| 4 | 4 | * then `(admin)` and `(writes)` where they apply. The live list of every tool, |
| 5 | 5 | * so no committed file has to be regenerated when a tool changes (#8304). |
| 6 | * Built once at boot and the same for every reader, whoever they are. | |
| 6 | * Built once at boot; a reader with an account gets all of it, a stranger the | |
| 7 | * token-free tools and one line for the rest, so /llms.txt varies by reader. | |
| 7 | 8 | */ |
| 8 | 9 | import type { Tool } from "../http/mcp/shared.ts"; |
| 9 | 10 | |
| @@ -25,10 +26,28 @@ export const toolsSection = (tools: readonly Listed[]): string[] => [ | ||
| 25 | 26 | "", |
| 26 | 27 | ]; |
| 27 | 28 | |
| 29 | /** | |
| 30 | * What a stranger can call, in full, then one line for the rest: the tools an | |
| 31 | * account opens are counted, not listed, and the line says where the list is. | |
| 32 | */ | |
| 33 | export const anonymousToolsSection = (open: readonly Listed[], more: number, origin: string): string[] => [ | |
| 34 | "## MCP tools", | |
| 35 | "", | |
| 36 | ...open.map((t) => toolLine(t)), | |
| 37 | `- ${more} more tools with an account: call tools/list on ${origin}/mcp once signed in, or read ${origin}/contract/openapi.json.`, | |
| 38 | "", | |
| 39 | ]; | |
| 40 | ||
| 28 | 41 | // The index is handed in at boot by the server rather than imported here: |
| 29 | 42 | // importing it from a page module closes an import cycle through the actions. |
| 43 | let toolCount = 0; | |
| 44 | let open: readonly Listed[] = []; | |
| 30 | 45 | let section: readonly string[] = []; |
| 31 | export const setMcpTools = (tools: readonly Listed[]): void => { | |
| 32 | section = toolsSection(tools); | |
| 46 | export const setMcpTools = (all: readonly Listed[], tokenFree: readonly Listed[] = []): void => { | |
| 47 | toolCount = all.length; | |
| 48 | open = tokenFree; | |
| 49 | section = toolsSection([...all, ...tokenFree]); | |
| 33 | 50 | }; |
| 34 | export const mcpToolsSection = (): readonly string[] => section; | |
| 51 | /** The signed-in reader's section is the whole index; a stranger's is folded. */ | |
| 52 | export const mcpToolsSection = (origin: string, signedIn: boolean): readonly string[] => | |
| 53 | signedIn ? section : anonymousToolsSection(open, toolCount, origin); |
src/web/standards.ts +12 −8
| @@ -339,7 +339,7 @@ function runningSentence(): string { | ||
| 339 | 339 | } |
| 340 | 340 | |
| 341 | 341 | /** llms.txt and its ETag per origin, valid while the listing read, the build and the tell command are the same. */ |
| 342 | const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; body: string; etag: string }>(); | |
| 342 | const llmsBodies = new Map<string, { signedIn: boolean; repos: LlmsListing; running: string; tell: string | undefined; body: string; etag: string }>(); | |
| 343 | 343 | |
| 344 | 344 | /** |
| 345 | 345 | * GET /llms.txt — the forge as an agent should meet it (llmstxt.org): what |
| @@ -352,26 +352,30 @@ export function llmsTxt(ctx: Ctx): Response { | ||
| 352 | 352 | const tell = sshTellCommand(ctx.app); |
| 353 | 353 | const running = runningSentence(); |
| 354 | 354 | const repos = timePhase(ctx.request, "repos", () => llmsRepos(ctx.app.db)); |
| 355 | let kept = llmsBodies.get(o); | |
| 355 | // The tools section depends on who asks, so the signed-in and the anonymous body are kept apart. | |
| 356 | const signedIn = ctx.auth.user !== undefined; | |
| 357 | const key = `${o} ${signedIn}`; | |
| 358 | let kept = llmsBodies.get(key); | |
| 356 | 359 | if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell) { |
| 357 | const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos)); | |
| 360 | const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos, signedIn)); | |
| 358 | 361 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); |
| 359 | kept = { repos, running, tell, body, etag }; | |
| 362 | kept = { signedIn, repos, running, tell, body, etag }; | |
| 360 | 363 | // Origins come from the Host header: keep a handful, not one per guess. |
| 361 | 364 | if (llmsBodies.size >= 16) llmsBodies.clear(); |
| 362 | llmsBodies.set(o, kept); | |
| 365 | llmsBodies.set(key, kept); | |
| 363 | 366 | } |
| 364 | 367 | const cache = "public, max-age=3600"; |
| 365 | 368 | const sent = ctx.request.headers.get("if-none-match"); |
| 366 | 369 | if (sent !== null && sent.split(",").some((t) => t.trim() === kept.etag)) |
| 367 | return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache } }); | |
| 370 | return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache, vary: "Cookie, Authorization" } }); | |
| 368 | 371 | const res = text(kept.body, "text/markdown; charset=utf-8"); |
| 369 | 372 | res.headers.set("etag", kept.etag); |
| 373 | res.headers.append("vary", "Cookie, Authorization"); | |
| 370 | 374 | return res; |
| 371 | 375 | } |
| 372 | 376 | |
| 373 | 377 | /** The llms.txt document for one origin. */ |
| 374 | function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing): string { | |
| 378 | function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing, signedIn: boolean): string { | |
| 375 | 379 | const mcp = mcpPointers(o); |
| 376 | 380 | return ( |
| 377 | 381 | [ |
| @@ -424,7 +428,7 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s | ||
| 424 | 428 | `- API: ${o}/api/v1, Authorization: Bearer <token>`, |
| 425 | 429 | `- OpenAPI: GET ${o}/api/v1/openapi.json (redirects to ${o}/contract/openapi.json)`, |
| 426 | 430 | "", |
| 427 | ...mcpToolsSection(), | |
| 431 | ...mcpToolsSection(o, signedIn), | |
| 428 | 432 | "## Public repositories", |
| 429 | 433 | "", |
| 430 | 434 | // Names and addresses only: a description is whatever its owner |
test/llms-anonymous.test.ts added +84
| @@ -0,0 +1,84 @@ | ||
| 1 | /** A stranger's /llms.txt folds the MCP tool list to what they can call. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { TOOLS } from "../src/http/mcp/tools.ts"; | |
| 11 | ||
| 12 | let dataDir: string; | |
| 13 | let app: ReturnType<typeof createApp>; | |
| 14 | let servers: Servers; | |
| 15 | let token: string; | |
| 16 | ||
| 17 | beforeAll(async () => { | |
| 18 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-anon-")); | |
| 19 | app = createApp({ dataDir, port: 0 }); | |
| 20 | servers = await serve(app); | |
| 21 | const made = await createUser(app.db, { | |
| 22 | origin: "seed", | |
| 23 | trusted: true, | |
| 24 | username: "alice", | |
| 25 | email: "[email protected]", | |
| 26 | password: "a-good-passphrase", | |
| 27 | adminIfFirst: true, | |
| 28 | }); | |
| 29 | if (!made.ok) throw new Error(made.error); | |
| 30 | token = createToken(app.db, made.value.id, "rw", "rw").token; | |
| 31 | }, 30_000); | |
| 32 | ||
| 33 | afterAll(async () => { | |
| 34 | servers.stop(); | |
| 35 | app.close(); | |
| 36 | await rm(dataDir, { recursive: true, force: true }); | |
| 37 | }); | |
| 38 | ||
| 39 | const section = (body: string): string[] => { | |
| 40 | const lines = body.split("\n"); | |
| 41 | const start = lines.indexOf("## MCP tools"); | |
| 42 | const end = lines.findIndex((l, i) => i > start && l.startsWith("## ")); | |
| 43 | return lines.slice(start + 1, end).filter((l) => l.startsWith("- ")); | |
| 44 | }; | |
| 45 | ||
| 46 | test("the anonymous text is small, varies by reader, and lists report plus one line", async () => { | |
| 47 | const res = await fetch(`${servers.url}/llms.txt`); | |
| 48 | const body = await res.text(); | |
| 49 | expect(Buffer.byteLength(body)).toBeLessThan(5 * 1024); | |
| 50 | expect(res.headers.get("vary")).toContain("Cookie"); | |
| 51 | expect(res.headers.get("vary")).toContain("Authorization"); | |
| 52 | const listed = section(body); | |
| 53 | expect(listed).toHaveLength(2); | |
| 54 | expect(listed[0]).toStartWith("- report: "); | |
| 55 | expect(listed[1]).toContain(`${TOOLS.length} more tools with an account`); | |
| 56 | expect(listed[1]).toContain("tools/list"); | |
| 57 | expect(listed[1]).toContain("/contract/openapi.json"); | |
| 58 | }); | |
| 59 | ||
| 60 | test("every tool the anonymous text names is one tools/list returns", async () => { | |
| 61 | const names = section(await fetch(`${servers.url}/llms.txt`).then((r) => r.text())) | |
| 62 | .map((l) => /^- ([a-z_]+): /.exec(l)?.[1]) | |
| 63 | .filter((n): n is string => n !== undefined); | |
| 64 | expect(names).toEqual(["report"]); | |
| 65 | const returned: string[] = []; | |
| 66 | let cursor: string | undefined; | |
| 67 | do { | |
| 68 | const list = await fetch(`${servers.url}/mcp`, { | |
| 69 | method: "POST", | |
| 70 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, | |
| 71 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", ...(cursor === undefined ? {} : { params: { cursor } }) }), | |
| 72 | }).then((r) => r.json() as Promise<any>); | |
| 73 | returned.push(...list.result.tools.map((t: { name: string }) => t.name)); | |
| 74 | cursor = list.result.nextCursor; | |
| 75 | } while (cursor !== undefined); | |
| 76 | for (const n of names) expect(returned).toContain(n); | |
| 77 | }); | |
| 78 | ||
| 79 | test("a signed-in reader still gets every tool and report", async () => { | |
| 80 | const body = await fetch(`${servers.url}/llms.txt`, { | |
| 81 | headers: { authorization: `Bearer ${token}` }, | |
| 82 | }).then((r) => r.text()); | |
| 83 | expect(section(body)).toHaveLength(TOOLS.length + 1); | |
| 84 | }); |
test/llms-tools.test.ts +17 −3
| @@ -7,23 +7,37 @@ import { createApp } from "../src/app.ts"; | ||
| 7 | 7 | import { serve, type Servers } from "../src/http/server.ts"; |
| 8 | 8 | import { REPORT_TOOL } from "../src/http/mcp.ts"; |
| 9 | 9 | import { TOOLS } from "../src/http/mcp/tools.ts"; |
| 10 | import { createToken } from "../src/core/tokens.ts"; | |
| 11 | import { createUser } from "../src/core/users.ts"; | |
| 10 | 12 | import type { Listed } from "../src/web/llms_tools.ts"; |
| 11 | 13 | |
| 12 | 14 | describe("/llms.txt MCP tools", () => { |
| 13 | 15 | let dataDir: string; |
| 14 | 16 | let servers: Servers; |
| 17 | let token: string; | |
| 15 | 18 | |
| 16 | 19 | beforeAll(async () => { |
| 17 | 20 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-tools-")); |
| 18 | servers = await serve(createApp({ dataDir, port: 0 })); | |
| 21 | const app = createApp({ dataDir, port: 0 }); | |
| 22 | servers = await serve(app); | |
| 23 | const made = await createUser(app.db, { | |
| 24 | origin: "seed", | |
| 25 | trusted: true, | |
| 26 | username: "alice", | |
| 27 | email: "[email protected]", | |
| 28 | password: "a-good-passphrase", | |
| 29 | adminIfFirst: true, | |
| 30 | }); | |
| 31 | if (!made.ok) throw new Error(made.error); | |
| 32 | token = createToken(app.db, made.value.id, "rw", "rw").token; | |
| 19 | 33 | }); |
| 20 | 34 | afterAll(async () => { |
| 21 | 35 | await servers.stop(); |
| 22 | 36 | await rm(dataDir, { recursive: true, force: true }); |
| 23 | 37 | }); |
| 24 | 38 | |
| 25 | test("lists exactly the tools in the index and `report`, by their `doc`, with (admin) and (writes)", async () => { | |
| 26 | const res = await fetch(`${servers.url}/llms.txt`); | |
| 39 | test("lists, for a signed-in reader, exactly the tools in the index and `report`, by their `doc`, with (admin) and (writes)", async () => { | |
| 40 | const res = await fetch(`${servers.url}/llms.txt`, { headers: { authorization: `Bearer ${token}` } }); | |
| 27 | 41 | expect(res.status).toBe(200); |
| 28 | 42 | expect(res.headers.get("cache-control")).toContain("max-age"); |
| 29 | 43 | const lines = (await res.text()).split("\n"); |
new above
offline
Drop the
Linealias and useListeddirectly. Also keep only the tool count here, not the whole array, sincetoolsis read only for.length.