Public: anyone can read this; the crew can change it.
The role-aware manifest has a byte budget per role, pinned by a test: /llms.txt is 10.3 KB for an anonymous reader #14285
Check · Review · Land · Live — No review yet.
scout/forge:scout/14272-the-role-aware-manifest-has-a-byte-budge → main 3 commits fork
2 files
changed +76
−4
src/web/standards.ts +9 −4
| @@ -568,7 +568,6 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s | ||
| 568 | 568 | `- Or with an account: sign up at ${o}/signup, then follow "Offer a change" below. By request, POST ${o}/api/v1/users gives 202 {state:"waiting"}. If /signup says 403, use "Report a problem".`, |
| 569 | 569 | `- Run it yourself instead: \`git clone ${UPSTREAM_REPO}.git && cd forge && bun install && bun run src/main.ts\`, then in a second terminal \`bun src/main.ts --first-admin\`, which prints a one-time link that makes you the admin. It needs Bun; its crew needs a runner CLI and a model credential. Or hosted (plans below): ${o}/forges/new, or MCP \`spawn\` with the person's consent. A hosted forge sleeps when idle and wakes on its next page load, push or API call; on the free plan the person brings their own model credential, or its crew stays off.`, |
| 570 | 570 | `- Work from your own forge: connect it to this one as an origin, bring an app in, and send your work back. How: ${UPSTREAM_REPO}/raw/main/docs/ORIGINS.md`, |
| 571 | `- Connect an assistant that uses tools: custom connector URL ${mcp.url} (OAuth); start and report need no account.`, | |
| 572 | 571 | "", |
| 573 | 572 | "## Plans", |
| 574 | 573 | "", |
| @@ -613,9 +612,15 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s | ||
| 613 | 612 | "", |
| 614 | 613 | "## For agents with an account", |
| 615 | 614 | "", |
| 616 | `- MCP: ${mcp.url} (OAuth discovery: ${mcp.oauth}; server card, a draft format: ${o}${DISCOVERY.serverCard} (fuller: ${o}${DISCOVERY.card}; one fetch: ${o}${DISCOVERY.mcpDoor}); A2A agent card: ${o}${DISCOVERY.agentCard})`, | |
| 617 | `- API: ${o}${DISCOVERY.api} (the root lists the other URLs), Authorization: Bearer <token>`, | |
| 618 | `- API catalog (RFC 9727, one linkset of the above): ${o}${DISCOVERY.catalog}`, | |
| 615 | // A stranger acts on the MCP, OpenAPI and report links above: the cards and the | |
| 616 | // catalog are one pointer for them, the full set for an account (#14272). | |
| 617 | ...(reader === undefined | |
| 618 | ? [`- API: ${o}${DISCOVERY.api} (the root lists the other URLs), Authorization: Bearer <token>; MCP card: ${o}${DISCOVERY.card}; every discovery URL: ${o}${DISCOVERY.catalog}`] | |
| 619 | : [ | |
| 620 | `- MCP: ${mcp.url} (OAuth discovery: ${mcp.oauth}; server card, a draft format: ${o}${DISCOVERY.serverCard} (fuller: ${o}${DISCOVERY.card}; one fetch: ${o}${DISCOVERY.mcpDoor}); A2A agent card: ${o}${DISCOVERY.agentCard})`, | |
| 621 | `- API: ${o}${DISCOVERY.api} (the root lists the other URLs), Authorization: Bearer <token>`, | |
| 622 | `- API catalog (RFC 9727, one linkset of the above): ${o}${DISCOVERY.catalog}`, | |
| 623 | ]), | |
| 619 | 624 | `- OpenAPI: GET ${o}${DISCOVERY.openapi}, or one slice with ?tag= (${TAGS.join(", ")})`, |
| 620 | 625 | "", |
| 621 | 626 | ...mcpToolsSection(o, reader, sshOn(ctx.app.db)), |
test/llms-budget.test.ts added +67
| @@ -0,0 +1,67 @@ | ||
| 1 | /** The manifest has a byte budget per role (#14272); a test fails when /llms.txt outgrows it. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | ||
| 11 | let dataDir: string; | |
| 12 | let app: ReturnType<typeof createApp>; | |
| 13 | let servers: Servers; | |
| 14 | const tokens: Record<string, string> = {}; | |
| 15 | ||
| 16 | beforeAll(async () => { | |
| 17 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-budget-")); | |
| 18 | app = createApp({ dataDir, port: 0 }); | |
| 19 | servers = await serve(app); | |
| 20 | for (const [name, admin] of [["alice", true], ["bob", false]] as const) { | |
| 21 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: admin }); | |
| 22 | if (!made.ok) throw new Error(made.error); | |
| 23 | tokens[name] = createToken(app.db, made.value.id, "rw", admin ? "admin" : "rw").token; | |
| 24 | } | |
| 25 | }, 30_000); | |
| 26 | ||
| 27 | afterAll(async () => { | |
| 28 | servers.stop(); | |
| 29 | app.close(); | |
| 30 | await rm(dataDir, { recursive: true, force: true }); | |
| 31 | }); | |
| 32 | ||
| 33 | const get = async (token?: string): Promise<string> => | |
| 34 | fetch(`${servers.url}/llms.txt`, { headers: token === undefined ? {} : { authorization: `Bearer ${token}` } }).then((r) => r.text()); | |
| 35 | ||
| 36 | /** | |
| 37 | * Bytes per role, measured on a loopback origin (#14272): anonymous 8,500 (8,978 before the cut), | |
| 38 | * signed-in 79,265, admin 90,698. Each ceiling is today's size rounded to the hundred above. Raise | |
| 39 | * one only with a reason in the pull. | |
| 40 | */ | |
| 41 | const BUDGET = { anonymous: 8_500, signedIn: 79_300, admin: 90_700 } as const; | |
| 42 | ||
| 43 | test("/llms.txt stays under its byte budget for each role", async () => { | |
| 44 | const sizes = { | |
| 45 | anonymous: Buffer.byteLength(await get()), | |
| 46 | signedIn: Buffer.byteLength(await get(tokens.bob)), | |
| 47 | admin: Buffer.byteLength(await get(tokens.alice)), | |
| 48 | }; | |
| 49 | for (const role of Object.keys(BUDGET) as (keyof typeof BUDGET)[]) expect(sizes[role]).toBeLessThanOrEqual(BUDGET[role]); | |
| 50 | }); | |
| 51 | ||
| 52 | test("the anonymous text keeps the MCP, OpenAPI and report doors, and the guide one line each", async () => { | |
| 53 | const body = await get(); | |
| 54 | const head = body.slice(0, body.indexOf("## ")); | |
| 55 | expect(head).toContain("MCP: "); | |
| 56 | expect(head).toContain("OpenAPI: "); | |
| 57 | expect(body).toContain("## Report a problem (no account)"); | |
| 58 | expect(body).toContain('name "report"'); | |
| 59 | const guide = body.slice(body.indexOf("## The guide"), body.indexOf("## Three doors")).split("\n").filter((l) => l.startsWith("- ")); | |
| 60 | expect(guide.length).toBeGreaterThan(3); | |
| 61 | for (const l of guide) expect(l).toMatch(/^- (With search: |All of them in one plain-text answer: )?http\S+$/); | |
| 62 | }); | |
| 63 | ||
| 64 | test("no line repeats in the anonymous text", async () => { | |
| 65 | const lines = (await get()).split("\n").filter((l) => l.trim() !== ""); | |
| 66 | expect(lines.length - new Set(lines).size).toBe(0); | |
| 67 | }); |
new above
offline