Public: anyone can read this; the crew can change it.

llms.txt is cacheable and role-aware: public by default, richer with a token, with an ETag that changes only when its words do #12334

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12322-llms-txt-is-cacheable-and-role-aware-pub → main 7 commits fork 2 threads · 2 unresolved

  • Merged. scout/12322-llms-txt-is-cacheable-and-role-aware-pub was merged into main as 1d9d508.

Review

Sign in to review this change.
7 files changed +155 −23
docs/MCP.md +1 −1
@@ -253,7 +253,7 @@ client's OAuth sign-in); the open tools are named in `/llms.txt`, the server
253253 card (`tokenFreeTools`) and `/api/v1/here`.
254254
255255 The complete list, with schemas, is MCP `tools/list` called with a token.
256 `/llms.txt`, under "## MCP tools", lists only the tools open without a token (one line each, from its `doc`) and counts the rest, pointing on to `tools/list`; signed in, it lists every tool.
256 `/llms.txt`, under "## MCP tools", lists only the tools open without a token (one line each, from its `doc`) and counts the rest, pointing on to `tools/list`; signed in, it lists the tools that token may call (a read-only token omits the writes it cannot make; admin tools need an admin account and an admin-scope token). It carries an ETag that changes only when its words do; a stranger's read is public for five minutes and answers `If-None-Match` with 304, a token's is private.
257257
258258 Sending a failed webhook delivery again (`redeliver`, #8075) is the terminal's `r` on a hook's Deliveries screen only; the `hooks` tool has no action for it yet.
259259
src/web/llms_tools.ts +37 −6
@@ -8,9 +8,10 @@
88 */
99 import type { Tool } from "../http/mcp/shared.ts";
1010 import { TOKEN_FREE_TOOLS } from "../core/mcp_card.ts";
11 import { scopeAllows, type TokenScope } from "../core/tokens.ts";
1112
1213 /** What a line reads: `report` is not a `Tool`, having no `run`, but is listed. */
13 export type Listed = Pick<Tool, "name" | "description" | "doc" | "writes" | "adminOnly" | "needsAdminScope">;
14 export type Listed = Pick<Tool, "name" | "description" | "doc" | "writes" | "adminOnly" | "needsAdminScope" | "dryByDefault" | "readsWithout" | "readsWhen">;
1415
1516 const oneLine = (s: string): string => s.replace(/\s+/g, " ").trim();
1617
@@ -43,19 +44,49 @@ export const anonymousToolsSection = (open: readonly Listed[], more: number, ori
4344 "",
4445 ];
4546
47 /** Who is reading: a stranger, or an account with the scope its token (or session) holds. */
48 export interface Reader { readonly scope: TokenScope; readonly isAdmin: boolean }
49
50 /**
51 * Whether this reader may call the tool: the rule the MCP dispatch applies
outdated

A module-level counter makes the cache correct but the test must mutate global state to prove it; fine, but once the list is per scope the version should key each scope list too.

52 * (http/mcp.ts) — admin-only tools need an admin account, admin-scope tools an
53 * admin-scope token, and a write needs `rw` unless a call can only plan or read.
54 */
55 export function callableBy(t: Listed, r: Reader): boolean {
56 if (t.adminOnly === true && !r.isAdmin) return false;
57 if (t.needsAdminScope === true && !scopeAllows(r.scope, "admin")) return false;
58 if (!t.writes || scopeAllows(r.scope, "rw")) return true;
59 return t.dryByDefault === true || t.readsWithout !== undefined || t.readsWhen !== undefined;
60 }
61
4662 // The index is handed in at boot by the server rather than imported here:
4763 // importing it from a page module closes an import cycle through the actions.
4864 let toolCount = 0;
4965 let open: readonly Listed[] = [];
50 let section: readonly string[] = [];
66 let every: readonly Listed[] = [];
67 let sections = new Map<string, readonly string[]>();
68 let version = 0;
69 /** Bumps whenever the list is set, so a kept /llms.txt body knows it is stale. */
70 export const mcpToolsVersion = (): number => version;
5171 export const setMcpTools = (all: readonly Listed[], tokenFree: readonly Listed[] = []): void => {
72 version++;
5273 // The open tools in TOOLS (explore, search) are listed with `report` and `start`,
5374 // not counted among the rest.
5475 const free = all.filter((t) => TOKEN_FREE_TOOLS.includes(t.name));
5576 toolCount = all.length - free.length;
5677 open = [...tokenFree, ...free];
57 section = toolsSection([...all, ...tokenFree]);
78 every = [...all, ...tokenFree];
79 sections = new Map(); // every kept per-reader list goes with the old index
5880 };
59 /** The signed-in reader's section is the whole index; a stranger's is folded. */
60 export const mcpToolsSection = (origin: string, signedIn: boolean): readonly string[] =>
61 signedIn ? section : anonymousToolsSection(open, toolCount, origin);
81
82 /** The kept-body key for a reader: tools differ by scope and by admin, nothing else. */
83 export const readerKey = (r: Reader | undefined): string => (r === undefined ? "anon" : `${r.scope}${r.isAdmin ? "+admin" : ""}`);
84
85 /** A signed-in reader's section is the tools they may call; a stranger's is folded. */
86 export function mcpToolsSection(origin: string, reader: Reader | undefined): readonly string[] {
87 if (reader === undefined) return anonymousToolsSection(open, toolCount, origin);
88 const k = readerKey(reader);
89 let s = sections.get(k);
90 if (s === undefined) sections.set(k, (s = toolsSection(every.filter((t) => callableBy(t, reader)))));
91 return s;
92 }
src/web/standards.ts +17 −12
@@ -47,7 +47,7 @@ import { offerTable } from "./offer.ts";
4747 import { getPlans } from "../settings/plans.ts";
4848 import { SITE_PATH, SITE_LINE } from "./site_line.ts";
4949 import { holds } from "./feeds.ts";
50 import { mcpToolsSection } from "./llms_tools.ts";
50 import { mcpToolsSection, mcpToolsVersion, readerKey, type Reader } from "./llms_tools.ts";
5151 import { threeDoors } from "./doors_words.ts";
5252 import { agentCard, toolsKey, apiCatalog, cardsLine, DISCOVERY, mcpCard, offeredPlugins, mcpPointers, offeredPluginRepo, pluginSource } from "../core/mcp_card.ts";
5353 import { forgeDocument } from "../core/standards_index.ts";
@@ -399,7 +399,7 @@ function runningSentence(): string {
399399 }
400400
401401 /** llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and the plugin source being public are the same. */
402 const llmsBodies = new Map<string, { signedIn: boolean; repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; body: string; etag: string }>();
402 const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; tools: number; body: string; etag: string }>();
403403
404404 /**
405405 * GET /llms.txt — the forge as an agent should meet it (llmstxt.org): what
outdated

The signed-in branch still serves the whole tool index keyed only by signedIn. Issue box 2 asks for the tools this token may call: filter by the token scope (key the kept body by scope, not just signedIn) and add a test that a read-only token sees fewer tools than an rw one. If no scope rule exists, extract one small module that owns it rather than leaving the box open.

@@ -409,8 +409,11 @@ const llmsBodies = new Map<string, { signedIn: boolean; repos: LlmsListing; runn
409409 */
410410 export function llmsTxt(ctx: Ctx): Response {
411411 const kept = llmsBody(ctx);
412 // The third line names the running commit: a client revalidates (one small 304) rather than keep an old build for an hour.
413 const cache = "private, no-cache";
412 // The third line names the running commit: a client revalidates (one small 304)
413 // rather than keep an old build for an hour.
414 // A stranger's text holds nothing of the reader's: it may be shared and kept five
415 // minutes. A reader with an account gets more and stays private.
416 const cache = ctx.auth.user === undefined ? "public, max-age=300" : "private, no-cache";
414417 const sent = ctx.request.headers.get("if-none-match");
415418 if (sent !== null && sent.split(",").some((t) => t.trim() === kept.etag))
416419 return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache, vary: "Cookie, Authorization" } });
@@ -428,15 +431,17 @@ export function llmsBody(ctx: Ctx): { body: string; etag: string } {
428431 const running = runningSentence();
429432 const plugins = offeredPluginRepo(ctx.app.db, ctx.app.config.forgeRepo, UPSTREAM_REPO);
430433 const repos = timePhase(ctx.request, "repos", () => llmsRepos(ctx.app.db));
431 // The tools section depends on who asks, so the signed-in and the anonymous body are kept apart.
432 const signedIn = ctx.auth.user !== undefined;
433 const key = `${o} ${signedIn}`;
434 // The tools section depends on who asks (scope and admin): each reader kind keeps its body.
435 // A session acts with full authority; a token with its stored scope.
436 const reader: Reader | undefined = ctx.auth.user === undefined ? undefined : { scope: ctx.auth.scope ?? "admin", isAdmin: ctx.auth.user.isAdmin };
437 const key = `${o} ${readerKey(reader)}`;
434438 const plans = JSON.stringify(getPlans(ctx.app.db));
439 const tools = mcpToolsVersion();
435440 let kept = llmsBodies.get(key);
436 if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell || kept.plugins !== plugins || kept.plans !== plans) {
437 const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos, signedIn, plugins));
441 if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell || kept.plugins !== plugins || kept.plans !== plans || kept.tools !== tools) {
442 const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos, reader, plugins));
438443 const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`);
439 kept = { signedIn, repos, running, tell, plugins, plans, body, etag };
444 kept = { repos, running, tell, plugins, plans, tools, body, etag };
440445 // Origins come from the Host header: keep a handful, not one per guess.
441446 if (llmsBodies.size >= 16) llmsBodies.clear();
442447 llmsBodies.set(key, kept);
@@ -454,7 +459,7 @@ function pluginLines(o: string, plugins: string | undefined): string[] {
454459 return plugins === undefined ? [...lines, "- Terminal agent plugin: not offered here (no public source)"] : lines;
455460 }
456461
457 function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing, signedIn: boolean, plugins: string | undefined): string {
462 function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing, reader: Reader | undefined, plugins: string | undefined): string {
458463 const mcp = mcpPointers(o);
459464 return (
460465 [
@@ -524,7 +529,7 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s
524529 `- API catalog (RFC 9727, one linkset of the above): ${o}${DISCOVERY.catalog}`,
525530 `- OpenAPI: GET ${o}${DISCOVERY.openapi}, or one slice with ?tag= (${TAGS.join(", ")})`,
526531 "",
527 ...mcpToolsSection(o, signedIn),
532 ...mcpToolsSection(o, reader),
528533 cardsLine(),
529534 "",
530535 "## Public repositories",
test/llms-anonymous.test.ts +1 −1
@@ -27,7 +27,7 @@ beforeAll(async () => {
2727 adminIfFirst: true,
2828 });
2929 if (!made.ok) throw new Error(made.error);
30 token = createToken(app.db, made.value.id, "rw", "rw").token;
30 token = createToken(app.db, made.value.id, "rw", "admin").token;
3131 }, 30_000);
3232
3333 afterAll(async () => {
test/llms-etag.test.ts added +96
@@ -0,0 +1,96 @@
1 /** /llms.txt: public and conditional for a stranger, private and richer with a token. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import { createToken } from "../src/core/tokens.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { callableBy, setMcpTools, type Listed } from "../src/web/llms_tools.ts";
11 import { TOOLS } from "../src/http/mcp/tools.ts";
12 import { REPORT_TOOL, START_TOOL } from "../src/http/mcp.ts";
13
14 let dataDir: string;
15 let app: ReturnType<typeof createApp>;
16 let servers: Servers;
17 let token: string;
18 let roToken: string;
19
20 beforeAll(async () => {
21 dataDir = await mkdtemp(join(tmpdir(), "forge-llms-etag-"));
22 app = createApp({ dataDir, port: 0 });
23 servers = await serve(app);
24 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
25 if (!made.ok) throw new Error(made.error);
26 token = createToken(app.db, made.value.id, "rw", "rw").token;
27 roToken = createToken(app.db, made.value.id, "ro", "ro").token;
28 }, 30_000);
29
30 afterAll(async () => {
31 servers.stop();
32 app.close();
33 await rm(dataDir, { recursive: true, force: true });
34 });
35
36 const get = (headers: Record<string, string> = {}) => fetch(`${servers.url}/llms.txt`, { headers });
37
38 test("an anonymous read is public for five minutes with a strong ETag that two reads share, and If-None-Match answers 304", async () => {
39 const a = await get();
40 const b = await get();
41 expect(a.headers.get("cache-control")).toBe("public, max-age=300");
42 const etag = a.headers.get("etag")!;
43 expect(etag).toMatch(/^"[0-9a-f]+"$/);
44 expect(b.headers.get("etag")).toBe(etag);
45 expect(await a.text()).toBe(await b.text());
46 const cond = await get({ "if-none-match": etag });
47 expect(cond.status).toBe(304);
48 expect(cond.headers.get("etag")).toBe(etag);
49 });
50
51 test("a bearer token adds tools, stays private, and nothing of it reaches the public body", async () => {
52 const anon = await get();
53 const anonBody = await anon.text();
54 const signed = await get({ authorization: `Bearer ${token}` });
55 const signedBody = await signed.text();
56 expect(signed.headers.get("cache-control")).toContain("private");
57 expect(signed.headers.get("etag")).not.toBe(anon.headers.get("etag"));
58 expect(signedBody.length).toBeGreaterThan(anonBody.length);
59 const anonLines = new Set(anonBody.split("\n"));
60 const extra = signedBody.split("\n").filter((l) => !anonLines.has(l));
61 expect(extra.length).toBeGreaterThan(1);
62 expect(anonBody).not.toContain(token);
63 expect(anonBody).not.toContain("alice");
64 });
65
66 test("a changed tool list changes the anonymous ETag", async () => {
67 const before = (await get()).headers.get("etag");
68 const open: Listed[] = [{ name: "zz_new_open_tool", description: "A tool added for the test.", writes: false }];
69 try {
70 setMcpTools(TOOLS, open);
71 const after = (await get()).headers.get("etag");
72 expect(after).not.toBe(before);
73 } finally {
74 setMcpTools(TOOLS, [REPORT_TOOL, START_TOOL]);
75 }
76 });
77
78 test("a read-only token lists fewer tools than an rw one, and never the writes it cannot make", async () => {
79 const ro = await (await get({ authorization: `Bearer ${roToken}` })).text();
80 const rw = await (await get({ authorization: `Bearer ${token}` })).text();
81 const tools = (b: string) => b.split("\n").filter((l) => l.startsWith("- ") && l.includes(": "));
82 expect(tools(ro).length).toBeLessThan(tools(rw).length);
83 const roSet = new Set(tools(ro));
84 expect(tools(rw).filter((l) => !roSet.has(l)).every((l) => l.includes("(writes)") || l.includes("(admin)"))).toBe(true);
85 });
86
87 test("callableBy: a write needs rw unless it only plans; admin tools need admin scope and account", () => {
88 const w: Listed = { name: "w", description: "d", writes: true };
89 expect(callableBy(w, { scope: "ro", isAdmin: false })).toBe(false);
90 expect(callableBy(w, { scope: "rw", isAdmin: false })).toBe(true);
91 expect(callableBy({ ...w, dryByDefault: true }, { scope: "ro", isAdmin: false })).toBe(true);
92 const a: Listed = { name: "a", description: "d", writes: false, needsAdminScope: true };
93 expect(callableBy(a, { scope: "rw", isAdmin: true })).toBe(false);
94 expect(callableBy(a, { scope: "admin", isAdmin: true })).toBe(true);
95 expect(callableBy({ name: "o", description: "d", writes: false, adminOnly: true }, { scope: "admin", isAdmin: false })).toBe(false);
96 });
test/llms-tools.test.ts +1 −1
@@ -30,7 +30,7 @@ describe("/llms.txt MCP tools", () => {
3030 adminIfFirst: true,
3131 });
3232 if (!made.ok) throw new Error(made.error);
33 token = createToken(app.db, made.value.id, "rw", "rw").token;
33 token = createToken(app.db, made.value.id, "rw", "admin").token;
3434 });
3535 afterAll(async () => {
3636 await servers.stop();
test/standards.test.ts +2 −2
@@ -317,8 +317,8 @@ describe("what the open web expects", () => {
317317 expect(again.status).toBe(304);
318318 expect(await again.text()).toBe("");
319319 expect(again.headers.get("etag")).toBe(etag);
320 expect(first.headers.get("cache-control")).toBe("private, no-cache");
321 expect(again.headers.get("cache-control")).toBe("private, no-cache");
320 expect(first.headers.get("cache-control")).toBe("public, max-age=300");
321 expect(again.headers.get("cache-control")).toBe("public, max-age=300");
322322 expect(again.headers.get("vary")).toContain("Cookie");
323323 expect(again.headers.get("x-content-type-options")).toBe("nosniff");
324324 const made = await createRepo(app, alice, { name: "etagged", visibility: "public" });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs