Public: anyone can read this; the crew can change it.
llms.txt is cacheable and role-aware: public by default, richer with a token, with an ETag that changes only when its words do #12334
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12322-llms-txt-is-cacheable-and-role-aware-pub → main 7 commits fork 2 threads · 2 unresolved
7 files
changed +155
−23
docs/MCP.md +1 −1
| @@ -253,7 +253,7 @@ client's OAuth sign-in); the open tools are named in `/llms.txt`, the server | ||
| 253 | 253 | card (`tokenFreeTools`) and `/api/v1/here`. |
| 254 | 254 | |
| 255 | 255 | The complete list, with schemas, is MCP `tools/list` called with a token. |
| 256 | `/llms.txt`, under "## MCP tools", lists only the tools open without a token (one line each, from its `doc`) and counts the rest, pointing on to `tools/list`; signed in, it lists every tool. | |
| 256 | `/llms.txt`, under "## MCP tools", lists only the tools open without a token (one line each, from its `doc`) and counts the rest, pointing on to `tools/list`; signed in, it lists the tools that token may call (a read-only token omits the writes it cannot make; admin tools need an admin account and an admin-scope token). It carries an ETag that changes only when its words do; a stranger's read is public for five minutes and answers `If-None-Match` with 304, a token's is private. | |
| 257 | 257 | |
| 258 | 258 | Sending a failed webhook delivery again (`redeliver`, #8075) is the terminal's `r` on a hook's Deliveries screen only; the `hooks` tool has no action for it yet. |
| 259 | 259 |
src/web/llms_tools.ts +37 −6
| @@ -8,9 +8,10 @@ | ||
| 8 | 8 | */ |
| 9 | 9 | import type { Tool } from "../http/mcp/shared.ts"; |
| 10 | 10 | import { TOKEN_FREE_TOOLS } from "../core/mcp_card.ts"; |
| 11 | import { scopeAllows, type TokenScope } from "../core/tokens.ts"; | |
| 11 | 12 | |
| 12 | 13 | /** What a line reads: `report` is not a `Tool`, having no `run`, but is listed. */ |
| 13 | export type Listed = Pick<Tool, "name" | "description" | "doc" | "writes" | "adminOnly" | "needsAdminScope">; | |
| 14 | export type Listed = Pick<Tool, "name" | "description" | "doc" | "writes" | "adminOnly" | "needsAdminScope" | "dryByDefault" | "readsWithout" | "readsWhen">; | |
| 14 | 15 | |
| 15 | 16 | const oneLine = (s: string): string => s.replace(/\s+/g, " ").trim(); |
| 16 | 17 | |
| @@ -43,19 +44,49 @@ export const anonymousToolsSection = (open: readonly Listed[], more: number, ori | ||
| 43 | 44 | "", |
| 44 | 45 | ]; |
| 45 | 46 | |
| 47 | /** Who is reading: a stranger, or an account with the scope its token (or session) holds. */ | |
| 48 | export interface Reader { readonly scope: TokenScope; readonly isAdmin: boolean } | |
| 49 | ||
| 50 | /** | |
| 51 | * Whether this reader may call the tool: the rule the MCP dispatch applies | |
| 52 | * (http/mcp.ts) — admin-only tools need an admin account, admin-scope tools an | |
| 53 | * admin-scope token, and a write needs `rw` unless a call can only plan or read. | |
| 54 | */ | |
| 55 | export function callableBy(t: Listed, r: Reader): boolean { | |
| 56 | if (t.adminOnly === true && !r.isAdmin) return false; | |
| 57 | if (t.needsAdminScope === true && !scopeAllows(r.scope, "admin")) return false; | |
| 58 | if (!t.writes || scopeAllows(r.scope, "rw")) return true; | |
| 59 | return t.dryByDefault === true || t.readsWithout !== undefined || t.readsWhen !== undefined; | |
| 60 | } | |
| 61 | ||
| 46 | 62 | // The index is handed in at boot by the server rather than imported here: |
| 47 | 63 | // importing it from a page module closes an import cycle through the actions. |
| 48 | 64 | let toolCount = 0; |
| 49 | 65 | let open: readonly Listed[] = []; |
| 50 | let section: readonly string[] = []; | |
| 66 | let every: readonly Listed[] = []; | |
| 67 | let sections = new Map<string, readonly string[]>(); | |
| 68 | let version = 0; | |
| 69 | /** Bumps whenever the list is set, so a kept /llms.txt body knows it is stale. */ | |
| 70 | export const mcpToolsVersion = (): number => version; | |
| 51 | 71 | export const setMcpTools = (all: readonly Listed[], tokenFree: readonly Listed[] = []): void => { |
| 72 | version++; | |
| 52 | 73 | // The open tools in TOOLS (explore, search) are listed with `report` and `start`, |
| 53 | 74 | // not counted among the rest. |
| 54 | 75 | const free = all.filter((t) => TOKEN_FREE_TOOLS.includes(t.name)); |
| 55 | 76 | toolCount = all.length - free.length; |
| 56 | 77 | open = [...tokenFree, ...free]; |
| 57 | section = toolsSection([...all, ...tokenFree]); | |
| 78 | every = [...all, ...tokenFree]; | |
| 79 | sections = new Map(); // every kept per-reader list goes with the old index | |
| 58 | 80 | }; |
| 59 | /** The signed-in reader's section is the whole index; a stranger's is folded. */ | |
| 60 | export const mcpToolsSection = (origin: string, signedIn: boolean): readonly string[] => | |
| 61 | signedIn ? section : anonymousToolsSection(open, toolCount, origin); | |
| 81 | ||
| 82 | /** The kept-body key for a reader: tools differ by scope and by admin, nothing else. */ | |
| 83 | export const readerKey = (r: Reader | undefined): string => (r === undefined ? "anon" : `${r.scope}${r.isAdmin ? "+admin" : ""}`); | |
| 84 | ||
| 85 | /** A signed-in reader's section is the tools they may call; a stranger's is folded. */ | |
| 86 | export function mcpToolsSection(origin: string, reader: Reader | undefined): readonly string[] { | |
| 87 | if (reader === undefined) return anonymousToolsSection(open, toolCount, origin); | |
| 88 | const k = readerKey(reader); | |
| 89 | let s = sections.get(k); | |
| 90 | if (s === undefined) sections.set(k, (s = toolsSection(every.filter((t) => callableBy(t, reader))))); | |
| 91 | return s; | |
| 92 | } |
src/web/standards.ts +17 −12
| @@ -47,7 +47,7 @@ import { offerTable } from "./offer.ts"; | ||
| 47 | 47 | import { getPlans } from "../settings/plans.ts"; |
| 48 | 48 | import { SITE_PATH, SITE_LINE } from "./site_line.ts"; |
| 49 | 49 | import { holds } from "./feeds.ts"; |
| 50 | import { mcpToolsSection } from "./llms_tools.ts"; | |
| 50 | import { mcpToolsSection, mcpToolsVersion, readerKey, type Reader } from "./llms_tools.ts"; | |
| 51 | 51 | import { threeDoors } from "./doors_words.ts"; |
| 52 | 52 | import { agentCard, toolsKey, apiCatalog, cardsLine, DISCOVERY, mcpCard, offeredPlugins, mcpPointers, offeredPluginRepo, pluginSource } from "../core/mcp_card.ts"; |
| 53 | 53 | import { forgeDocument } from "../core/standards_index.ts"; |
| @@ -399,7 +399,7 @@ function runningSentence(): string { | ||
| 399 | 399 | } |
| 400 | 400 | |
| 401 | 401 | /** llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and the plugin source being public are the same. */ |
| 402 | const llmsBodies = new Map<string, { signedIn: boolean; repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; body: string; etag: string }>(); | |
| 402 | const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; tools: number; body: string; etag: string }>(); | |
| 403 | 403 | |
| 404 | 404 | /** |
| 405 | 405 | * GET /llms.txt — the forge as an agent should meet it (llmstxt.org): what |
outdated
The signed-in branch still serves the whole tool index keyed only by signedIn. Issue box 2 asks for the tools this token may call: filter by the token scope (key the kept body by scope, not just signedIn) and add a test that a read-only token sees fewer tools than an rw one. If no scope rule exists, extract one small module that owns it rather than leaving the box open. |
||
| @@ -409,8 +409,11 @@ const llmsBodies = new Map<string, { signedIn: boolean; repos: LlmsListing; runn | ||
| 409 | 409 | */ |
| 410 | 410 | export function llmsTxt(ctx: Ctx): Response { |
| 411 | 411 | const kept = llmsBody(ctx); |
| 412 | // The third line names the running commit: a client revalidates (one small 304) rather than keep an old build for an hour. | |
| 413 | const cache = "private, no-cache"; | |
| 412 | // The third line names the running commit: a client revalidates (one small 304) | |
| 413 | // rather than keep an old build for an hour. | |
| 414 | // A stranger's text holds nothing of the reader's: it may be shared and kept five | |
| 415 | // minutes. A reader with an account gets more and stays private. | |
| 416 | const cache = ctx.auth.user === undefined ? "public, max-age=300" : "private, no-cache"; | |
| 414 | 417 | const sent = ctx.request.headers.get("if-none-match"); |
| 415 | 418 | if (sent !== null && sent.split(",").some((t) => t.trim() === kept.etag)) |
| 416 | 419 | return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache, vary: "Cookie, Authorization" } }); |
| @@ -428,15 +431,17 @@ export function llmsBody(ctx: Ctx): { body: string; etag: string } { | ||
| 428 | 431 | const running = runningSentence(); |
| 429 | 432 | const plugins = offeredPluginRepo(ctx.app.db, ctx.app.config.forgeRepo, UPSTREAM_REPO); |
| 430 | 433 | const repos = timePhase(ctx.request, "repos", () => llmsRepos(ctx.app.db)); |
| 431 | // The tools section depends on who asks, so the signed-in and the anonymous body are kept apart. | |
| 432 | const signedIn = ctx.auth.user !== undefined; | |
| 433 | const key = `${o} ${signedIn}`; | |
| 434 | // The tools section depends on who asks (scope and admin): each reader kind keeps its body. | |
| 435 | // A session acts with full authority; a token with its stored scope. | |
| 436 | const reader: Reader | undefined = ctx.auth.user === undefined ? undefined : { scope: ctx.auth.scope ?? "admin", isAdmin: ctx.auth.user.isAdmin }; | |
| 437 | const key = `${o} ${readerKey(reader)}`; | |
| 434 | 438 | const plans = JSON.stringify(getPlans(ctx.app.db)); |
| 439 | const tools = mcpToolsVersion(); | |
| 435 | 440 | let kept = llmsBodies.get(key); |
| 436 | if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell || kept.plugins !== plugins || kept.plans !== plans) { | |
| 437 | const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos, signedIn, plugins)); | |
| 441 | if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell || kept.plugins !== plugins || kept.plans !== plans || kept.tools !== tools) { | |
| 442 | const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos, reader, plugins)); | |
| 438 | 443 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); |
| 439 | kept = { signedIn, repos, running, tell, plugins, plans, body, etag }; | |
| 444 | kept = { repos, running, tell, plugins, plans, tools, body, etag }; | |
| 440 | 445 | // Origins come from the Host header: keep a handful, not one per guess. |
| 441 | 446 | if (llmsBodies.size >= 16) llmsBodies.clear(); |
| 442 | 447 | llmsBodies.set(key, kept); |
| @@ -454,7 +459,7 @@ function pluginLines(o: string, plugins: string | undefined): string[] { | ||
| 454 | 459 | return plugins === undefined ? [...lines, "- Terminal agent plugin: not offered here (no public source)"] : lines; |
| 455 | 460 | } |
| 456 | 461 | |
| 457 | function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing, signedIn: boolean, plugins: string | undefined): string { | |
| 462 | function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing, reader: Reader | undefined, plugins: string | undefined): string { | |
| 458 | 463 | const mcp = mcpPointers(o); |
| 459 | 464 | return ( |
| 460 | 465 | [ |
| @@ -524,7 +529,7 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s | ||
| 524 | 529 | `- API catalog (RFC 9727, one linkset of the above): ${o}${DISCOVERY.catalog}`, |
| 525 | 530 | `- OpenAPI: GET ${o}${DISCOVERY.openapi}, or one slice with ?tag= (${TAGS.join(", ")})`, |
| 526 | 531 | "", |
| 527 | ...mcpToolsSection(o, signedIn), | |
| 532 | ...mcpToolsSection(o, reader), | |
| 528 | 533 | cardsLine(), |
| 529 | 534 | "", |
| 530 | 535 | "## Public repositories", |
test/llms-anonymous.test.ts +1 −1
| @@ -27,7 +27,7 @@ beforeAll(async () => { | ||
| 27 | 27 | adminIfFirst: true, |
| 28 | 28 | }); |
| 29 | 29 | if (!made.ok) throw new Error(made.error); |
| 30 | token = createToken(app.db, made.value.id, "rw", "rw").token; | |
| 30 | token = createToken(app.db, made.value.id, "rw", "admin").token; | |
| 31 | 31 | }, 30_000); |
| 32 | 32 | |
| 33 | 33 | afterAll(async () => { |
test/llms-etag.test.ts added +96
| @@ -0,0 +1,96 @@ | ||
| 1 | /** /llms.txt: public and conditional for a stranger, private and richer with a token. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { callableBy, setMcpTools, type Listed } from "../src/web/llms_tools.ts"; | |
| 11 | import { TOOLS } from "../src/http/mcp/tools.ts"; | |
| 12 | import { REPORT_TOOL, START_TOOL } from "../src/http/mcp.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: ReturnType<typeof createApp>; | |
| 16 | let servers: Servers; | |
| 17 | let token: string; | |
| 18 | let roToken: string; | |
| 19 | ||
| 20 | beforeAll(async () => { | |
| 21 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-etag-")); | |
| 22 | app = createApp({ dataDir, port: 0 }); | |
| 23 | servers = await serve(app); | |
| 24 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 25 | if (!made.ok) throw new Error(made.error); | |
| 26 | token = createToken(app.db, made.value.id, "rw", "rw").token; | |
| 27 | roToken = createToken(app.db, made.value.id, "ro", "ro").token; | |
| 28 | }, 30_000); | |
| 29 | ||
| 30 | afterAll(async () => { | |
| 31 | servers.stop(); | |
| 32 | app.close(); | |
| 33 | await rm(dataDir, { recursive: true, force: true }); | |
| 34 | }); | |
| 35 | ||
| 36 | const get = (headers: Record<string, string> = {}) => fetch(`${servers.url}/llms.txt`, { headers }); | |
| 37 | ||
| 38 | test("an anonymous read is public for five minutes with a strong ETag that two reads share, and If-None-Match answers 304", async () => { | |
| 39 | const a = await get(); | |
| 40 | const b = await get(); | |
| 41 | expect(a.headers.get("cache-control")).toBe("public, max-age=300"); | |
| 42 | const etag = a.headers.get("etag")!; | |
| 43 | expect(etag).toMatch(/^"[0-9a-f]+"$/); | |
| 44 | expect(b.headers.get("etag")).toBe(etag); | |
| 45 | expect(await a.text()).toBe(await b.text()); | |
| 46 | const cond = await get({ "if-none-match": etag }); | |
| 47 | expect(cond.status).toBe(304); | |
| 48 | expect(cond.headers.get("etag")).toBe(etag); | |
| 49 | }); | |
| 50 | ||
| 51 | test("a bearer token adds tools, stays private, and nothing of it reaches the public body", async () => { | |
| 52 | const anon = await get(); | |
| 53 | const anonBody = await anon.text(); | |
| 54 | const signed = await get({ authorization: `Bearer ${token}` }); | |
| 55 | const signedBody = await signed.text(); | |
| 56 | expect(signed.headers.get("cache-control")).toContain("private"); | |
| 57 | expect(signed.headers.get("etag")).not.toBe(anon.headers.get("etag")); | |
| 58 | expect(signedBody.length).toBeGreaterThan(anonBody.length); | |
| 59 | const anonLines = new Set(anonBody.split("\n")); | |
| 60 | const extra = signedBody.split("\n").filter((l) => !anonLines.has(l)); | |
| 61 | expect(extra.length).toBeGreaterThan(1); | |
| 62 | expect(anonBody).not.toContain(token); | |
| 63 | expect(anonBody).not.toContain("alice"); | |
| 64 | }); | |
| 65 | ||
| 66 | test("a changed tool list changes the anonymous ETag", async () => { | |
| 67 | const before = (await get()).headers.get("etag"); | |
| 68 | const open: Listed[] = [{ name: "zz_new_open_tool", description: "A tool added for the test.", writes: false }]; | |
| 69 | try { | |
| 70 | setMcpTools(TOOLS, open); | |
| 71 | const after = (await get()).headers.get("etag"); | |
| 72 | expect(after).not.toBe(before); | |
| 73 | } finally { | |
| 74 | setMcpTools(TOOLS, [REPORT_TOOL, START_TOOL]); | |
| 75 | } | |
| 76 | }); | |
| 77 | ||
| 78 | test("a read-only token lists fewer tools than an rw one, and never the writes it cannot make", async () => { | |
| 79 | const ro = await (await get({ authorization: `Bearer ${roToken}` })).text(); | |
| 80 | const rw = await (await get({ authorization: `Bearer ${token}` })).text(); | |
| 81 | const tools = (b: string) => b.split("\n").filter((l) => l.startsWith("- ") && l.includes(": ")); | |
| 82 | expect(tools(ro).length).toBeLessThan(tools(rw).length); | |
| 83 | const roSet = new Set(tools(ro)); | |
| 84 | expect(tools(rw).filter((l) => !roSet.has(l)).every((l) => l.includes("(writes)") || l.includes("(admin)"))).toBe(true); | |
| 85 | }); | |
| 86 | ||
| 87 | test("callableBy: a write needs rw unless it only plans; admin tools need admin scope and account", () => { | |
| 88 | const w: Listed = { name: "w", description: "d", writes: true }; | |
| 89 | expect(callableBy(w, { scope: "ro", isAdmin: false })).toBe(false); | |
| 90 | expect(callableBy(w, { scope: "rw", isAdmin: false })).toBe(true); | |
| 91 | expect(callableBy({ ...w, dryByDefault: true }, { scope: "ro", isAdmin: false })).toBe(true); | |
| 92 | const a: Listed = { name: "a", description: "d", writes: false, needsAdminScope: true }; | |
| 93 | expect(callableBy(a, { scope: "rw", isAdmin: true })).toBe(false); | |
| 94 | expect(callableBy(a, { scope: "admin", isAdmin: true })).toBe(true); | |
| 95 | expect(callableBy({ name: "o", description: "d", writes: false, adminOnly: true }, { scope: "admin", isAdmin: false })).toBe(false); | |
| 96 | }); |
test/llms-tools.test.ts +1 −1
| @@ -30,7 +30,7 @@ describe("/llms.txt MCP tools", () => { | ||
| 30 | 30 | adminIfFirst: true, |
| 31 | 31 | }); |
| 32 | 32 | if (!made.ok) throw new Error(made.error); |
| 33 | token = createToken(app.db, made.value.id, "rw", "rw").token; | |
| 33 | token = createToken(app.db, made.value.id, "rw", "admin").token; | |
| 34 | 34 | }); |
| 35 | 35 | afterAll(async () => { |
| 36 | 36 | await servers.stop(); |
test/standards.test.ts +2 −2
| @@ -317,8 +317,8 @@ describe("what the open web expects", () => { | ||
| 317 | 317 | expect(again.status).toBe(304); |
| 318 | 318 | expect(await again.text()).toBe(""); |
| 319 | 319 | expect(again.headers.get("etag")).toBe(etag); |
| 320 | expect(first.headers.get("cache-control")).toBe("private, no-cache"); | |
| 321 | expect(again.headers.get("cache-control")).toBe("private, no-cache"); | |
| 320 | expect(first.headers.get("cache-control")).toBe("public, max-age=300"); | |
| 321 | expect(again.headers.get("cache-control")).toBe("public, max-age=300"); | |
| 322 | 322 | expect(again.headers.get("vary")).toContain("Cookie"); |
| 323 | 323 | expect(again.headers.get("x-content-type-options")).toBe("nosniff"); |
| 324 | 324 | const made = await createRepo(app, alice, { name: "etagged", visibility: "public" }); |
new above
offline
A module-level counter makes the cache correct but the test must mutate global state to prove it; fine, but once the list is per scope the version should key each scope list too.