Public: anyone can read this; the crew can change it.
/llms-full.txt is every guide in one plain-text answer, each under its own heading #13909
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13865-llms-full-txt-is-every-guide-in-one-plai → main 6 commits fork 4 threads · 4 unresolved
src/core/reserved.ts +1
| @@ -119,6 +119,7 @@ export const SYSTEM_NAMES: readonly string[] = [ | ||
| 119 | 119 | "apple-touch-icon.png", |
| 120 | 120 | "apple-touch-icon-precomposed.png", |
| 121 | 121 | "llms.txt", |
| 122 | "llms-full.txt", | |
| 122 | 123 | "openapi.json", |
| 123 | 124 | "runs", |
| 124 | 125 | "search", |
src/http/route_marks.ts +1
| @@ -138,6 +138,7 @@ export const HEADS: Readonly<Record<string, Mark | "mixed">> = { | ||
| 138 | 138 | "/hooks": "open", |
| 139 | 139 | "/icons": "open", |
| 140 | 140 | "/live": "mixed", |
| 141 | "/llms-full.txt": "open", | |
| 141 | 142 | "/llms.txt": "open", |
| 142 | 143 | "/login": "open", |
| 143 | 144 | "/logout": "open", |
src/http/server.ts +2
| @@ -204,6 +204,7 @@ import { queueCeiling, queuePage, queueSwitch } from "../web/pages/queue.ts"; | ||
| 204 | 204 | import { chapterAct, chapterPage, chaptersPage } from "../web/pages/chapters.ts"; |
| 205 | 205 | import { betAct, betPage, betPropose, betsPage } from "../web/pages/bets.ts"; |
| 206 | 206 | import { changelogJson, changelogPage } from "../web/pages/changelog.ts"; |
| 207 | import { llmsFull } from "../web/llms_full.ts"; | |
| 207 | 208 | import { docsFeature, docsFile, docsFolder } from "../web/pages/docs_link.ts"; |
| 208 | 209 | import { proveAgainSubmit, releaseHoldSubmit, releaseNowSubmit, liftSubmit, releasesPage, rollbackSubmit } from "../web/pages/releases.ts"; |
| 209 | 210 | import { |
| @@ -860,6 +861,7 @@ export function buildRouter(): Router<Handler> { | ||
| 860 | 861 | router.add("GET", "/.well-known/api-catalog", apiCatalogJson); |
| 861 | 862 | router.add("GET", "/.well-known/forge", forgeJson); |
| 862 | 863 | router.add("GET", "/llms.txt", llmsTxt); |
| 864 | router.add("GET", "/llms-full.txt", llmsFull); | |
| 863 | 865 | // The same words as a page, for an assistant whose reader takes only HTML. |
| 864 | 866 | router.add("GET", "/agents", agentsPage); |
| 865 | 867 | router.add("GET", "/.well-known/change-password", changePassword); |
src/web/llms_full.ts added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | /** | |
| 2 | * GET /llms-full.txt: every guide page of /llms.txt in one plain-text answer, in the same order, | |
| 3 | * each under `# <title>` and its own URL on this forge. The docs change only with the release, so | |
| 4 | * the answer carries an ETag and may be cached; the origin-free text is built once per process and | |
| 5 | * the origin is filled in per request. | |
| 6 | */ | |
| 7 | ||
| 8 | import type { Ctx } from "../http/context.ts"; | |
| 9 | import { holds } from "../http/etag.ts"; | |
| 10 | import { notFound } from "../http/respond.ts"; | |
| 11 | import { docEntry, docText } from "../view/docs.ts"; | |
| 12 | import { GUIDE } from "../view/guide.ts"; | |
| 13 | ||
| 14 | /** Stands for the origin in the template; NUL cannot occur in a guide. */ | |
| 15 | const ORIGIN = "\u0000origin\u0000"; | |
| 16 | ||
| 17 | let built: { template: string; digest: string } | undefined; | |
| 18 | ||
| 19 | /** The origin-free document, built once per process: the docs are constant per release. */ | |
| 20 | function template(): { template: string; digest: string } { | |
| 21 | if (built !== undefined) return built; | |
| 22 | const parts: string[] = []; | |
| 23 | for (const name of GUIDE) { | |
| 24 | const entry = docEntry(name); | |
| 25 | if (entry === undefined) continue; | |
| 26 | // The page's own first heading is the title already printed above its URL. | |
| 27 | const body = docText(entry).replace(/^\s*# .*\n/, "").trim(); | |
| 28 | parts.push(`# ${entry.title}\n\n${ORIGIN}/docs/${name}\n\n${body}\n`); | |
| 29 | } | |
outdated
The ETag is a SHA-256 of the whole concatenated guide set, recomputed on every request. Combined with the rebuild at line 26 this is the per-request work the issue ruled out. Fix: compute the template once, and derive the ETag from it plus the origin. |
||
| 30 | const text = parts.join("\n"); | |
| 31 | built = { template: text, digest: new Bun.CryptoHasher("sha256").update(text).digest("hex") }; | |
| 32 | return built; | |
| 33 | } | |
| 34 | ||
| 35 | export function llmsFull(ctx: Ctx): Response { | |
| 36 | const { template: text, digest } = template(); | |
| 37 | if (text === "") return notFound(); | |
| 38 | const origin = ctx.url.origin; | |
| 39 | // The template's digest is fixed; only the short origin is hashed per request. | |
| 40 | const etag = `"${new Bun.CryptoHasher("sha256").update(digest).update("\n").update(origin).digest("hex").slice(0, 32)}"`; | |
| 41 | const headers = { etag, "cache-control": "public, max-age=300" }; | |
| 42 | // Weakly: the edge hands the tag back as W/"…", and `*` holds any. | |
outdated
Compare weakly: |
||
| 43 | if (holds(ctx.request, etag)) return new Response(null, { status: 304, headers }); | |
| 44 | return new Response(text.split(ORIGIN).join(origin), { headers: { ...headers, "content-type": "text/plain; charset=utf-8" } }); | |
| 45 | } |
src/web/standards.ts +1
| @@ -532,6 +532,7 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s | ||
| 532 | 532 | "", |
| 533 | 533 | `- With search: ${o}/docs`, |
| 534 | 534 | ...GUIDE.map((page) => `- ${o}/docs/${page}`), |
| 535 | `- All of them in one plain-text answer: ${o}/llms-full.txt`, | |
| 535 | 536 | "", |
| 536 | 537 | ...threeDoors(o, mcp.url), |
| 537 | 538 | "## Start here", |
test/design-audit.test.ts +1 −1
| @@ -17,7 +17,7 @@ const doc = readFileSync("docs/design/audit.md", "utf8"); | ||
| 17 | 17 | |
| 18 | 18 | /** Routes that answer a file, a fragment, a redirect hop or a callback. */ |
| 19 | 19 | const NOT_PAGES = |
| 20 | /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|explore\.atom$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/; | |
| 20 | /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|explore\.atom$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/; | |
| 21 | 21 | |
| 22 | 22 | const getRoutes = [ |
| 23 | 23 | ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g), |
test/forge-looks.test.ts +1
| @@ -38,6 +38,7 @@ const NOT_A_PAGE: Record<string, string> = { | ||
| 38 | 38 | "/forges": "no picture: its slot went to the preview's /waves and /milestones; the page is covered by its own tests", |
| 39 | 39 | "/family": "no picture: its slot went to the preview's /waves and /milestones; the page is covered by its own tests", |
| 40 | 40 | "/llms.txt": "plain markdown for machines, not a page to picture", |
| 41 | "/llms-full.txt": "plain text for machines, not a page to picture", | |
| 41 | 42 | "/agents": "/llms.txt drawn as a page for an assistant; no picture: not in app.looks, and test/agents-page.test.ts covers it", |
| 42 | 43 | "/map": "no picture: its slot went to /security, the visitor's security page (#13208); covered by test/map-cache.test.ts and the other map tests", |
| 43 | 44 | "/packs": "no picture: not in app.looks; the page is covered by test/packs.test.ts", |
test/llms-full.test.ts added +74
| @@ -0,0 +1,74 @@ | ||
| 1 | /** /llms-full.txt: every guide in llms.txt's order, one build, a 304 on revalidation. */ | |
| 2 | import { afterAll, beforeAll, expect, spyOn, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import * as docs from "../src/view/docs.ts"; | |
| 9 | import { docEntry } from "../src/view/docs.ts"; | |
| 10 | import { GUIDE } from "../src/view/guide.ts"; | |
| 11 | ||
| 12 | let dataDir: string; | |
| 13 | let app: ReturnType<typeof createApp>; | |
| 14 | let servers: Servers; | |
| 15 | const docText = spyOn(docs, "docText"); | |
| 16 | ||
| 17 | beforeAll(async () => { | |
| 18 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-full-")); | |
| 19 | app = createApp({ dataDir, port: 0 }); | |
| 20 | servers = await serve(app); | |
| 21 | }, 30_000); | |
| 22 | ||
| 23 | afterAll(async () => { | |
| 24 | servers.stop(); | |
| 25 | app.close(); | |
| 26 | await rm(dataDir, { recursive: true, force: true }); | |
| 27 | }); | |
| 28 | ||
| 29 | test("every guide's title and URL are in it, in llms.txt's order", async () => { | |
| 30 | const res = await fetch(`${servers.url}/llms-full.txt`); | |
| 31 | expect(res.status).toBe(200); | |
| 32 | expect(res.headers.get("content-type")).toContain("text/plain"); | |
| 33 | const body = await res.text(); | |
| 34 | let at = -1; | |
| 35 | for (const name of GUIDE) { | |
| 36 | const title = docEntry(name)!.title; | |
| 37 | const heading = body.indexOf(`# ${title}\n\n${servers.url}/docs/${name}\n`); | |
| 38 | expect(heading).toBeGreaterThan(at); | |
| 39 | at = heading; | |
| 40 | } | |
| 41 | }); | |
outdated
This test checks the 304 but nothing checks that the body is built once. Add one that fails when the build runs per request. |
||
| 42 | ||
| 43 | test("a second request is a 304 on the same ETag", async () => { | |
| 44 | const a = await fetch(`${servers.url}/llms-full.txt`); | |
| 45 | const etag = a.headers.get("etag")!; | |
| 46 | expect(etag).toMatch(/^"[0-9a-f]+"$/); | |
| 47 | const b = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": etag } }); | |
| 48 | expect(b.status).toBe(304); | |
| 49 | expect(b.headers.get("etag")).toBe(etag); | |
| 50 | }); | |
| 51 | ||
| 52 | test("the guides are read once however many requests come", async () => { | |
| 53 | // Earlier tests in this file already requested it: one read per guide, never one per request. | |
| 54 | const reads = docText.mock.calls.length; | |
| 55 | expect(reads).toBeGreaterThan(0); | |
| 56 | expect(reads).toBeLessThanOrEqual(GUIDE.length); | |
| 57 | for (let i = 0; i < 3; i++) await (await fetch(`${servers.url}/llms-full.txt`)).text(); | |
| 58 | expect(docText.mock.calls.length).toBe(reads); | |
| 59 | }); | |
| 60 | ||
| 61 | test("the ETag the edge weakens, or *, still earns a 304", async () => { | |
| 62 | const etag = (await fetch(`${servers.url}/llms-full.txt`)).headers.get("etag")!; | |
| 63 | for (const held of [`W/${etag}`, `"other", W/${etag}`, "*"]) { | |
| 64 | const res = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": held } }); | |
| 65 | expect(res.status).toBe(304); | |
| 66 | } | |
| 67 | const other = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": 'W/"other"' } }); | |
| 68 | expect(other.status).toBe(200); | |
| 69 | }); | |
| 70 | ||
| 71 | test("/llms.txt names it", async () => { | |
| 72 | const body = await (await fetch(`${servers.url}/llms.txt`)).text(); | |
| 73 | expect(body).toContain(`${servers.url}/llms-full.txt`); | |
| 74 | }); |
Box 2 left: renderLlmsFull(ctx.url.origin) reads and strips every doc per request, which is both a latency and a DoS cost on a public, unauthenticated route. Cache the origin-free template at module level (it is constant per release) and replace an origin token per request.