Public: anyone can read this; the crew can change it.

/llms-full.txt is every guide in one plain-text answer, each under its own heading #13909

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13865-llms-full-txt-is-every-guide-in-one-plai → main 6 commits fork 4 threads · 4 unresolved

  • Merged. torque/13865-llms-full-txt-is-every-guide-in-one-plai was merged into main as bfb7e8d.

Review

Sign in to review this change.
8 files changed +126 −1
src/core/reserved.ts +1
@@ -119,6 +119,7 @@ export const SYSTEM_NAMES: readonly string[] = [
119119 "apple-touch-icon.png",
120120 "apple-touch-icon-precomposed.png",
121121 "llms.txt",
122 "llms-full.txt",
122123 "openapi.json",
123124 "runs",
124125 "search",
src/http/route_marks.ts +1
@@ -138,6 +138,7 @@ export const HEADS: Readonly<Record<string, Mark | "mixed">> = {
138138 "/hooks": "open",
139139 "/icons": "open",
140140 "/live": "mixed",
141 "/llms-full.txt": "open",
141142 "/llms.txt": "open",
142143 "/login": "open",
143144 "/logout": "open",
src/http/server.ts +2
@@ -204,6 +204,7 @@ import { queueCeiling, queuePage, queueSwitch } from "../web/pages/queue.ts";
204204 import { chapterAct, chapterPage, chaptersPage } from "../web/pages/chapters.ts";
205205 import { betAct, betPage, betPropose, betsPage } from "../web/pages/bets.ts";
206206 import { changelogJson, changelogPage } from "../web/pages/changelog.ts";
207 import { llmsFull } from "../web/llms_full.ts";
207208 import { docsFeature, docsFile, docsFolder } from "../web/pages/docs_link.ts";
208209 import { proveAgainSubmit, releaseHoldSubmit, releaseNowSubmit, liftSubmit, releasesPage, rollbackSubmit } from "../web/pages/releases.ts";
209210 import {
@@ -860,6 +861,7 @@ export function buildRouter(): Router<Handler> {
860861 router.add("GET", "/.well-known/api-catalog", apiCatalogJson);
861862 router.add("GET", "/.well-known/forge", forgeJson);
862863 router.add("GET", "/llms.txt", llmsTxt);
864 router.add("GET", "/llms-full.txt", llmsFull);
863865 // The same words as a page, for an assistant whose reader takes only HTML.
864866 router.add("GET", "/agents", agentsPage);
865867 router.add("GET", "/.well-known/change-password", changePassword);
src/web/llms_full.ts added +45
@@ -0,0 +1,45 @@
1 /**
2 * GET /llms-full.txt: every guide page of /llms.txt in one plain-text answer, in the same order,
3 * each under `# <title>` and its own URL on this forge. The docs change only with the release, so
4 * the answer carries an ETag and may be cached; the origin-free text is built once per process and
5 * the origin is filled in per request.
6 */
7
8 import type { Ctx } from "../http/context.ts";
9 import { holds } from "../http/etag.ts";
10 import { notFound } from "../http/respond.ts";
11 import { docEntry, docText } from "../view/docs.ts";
12 import { GUIDE } from "../view/guide.ts";
13
14 /** Stands for the origin in the template; NUL cannot occur in a guide. */
15 const ORIGIN = "\u0000origin\u0000";
16
17 let built: { template: string; digest: string } | undefined;
18
19 /** The origin-free document, built once per process: the docs are constant per release. */
20 function template(): { template: string; digest: string } {
21 if (built !== undefined) return built;
22 const parts: string[] = [];
23 for (const name of GUIDE) {
24 const entry = docEntry(name);
25 if (entry === undefined) continue;
outdated

Box 2 left: renderLlmsFull(ctx.url.origin) reads and strips every doc per request, which is both a latency and a DoS cost on a public, unauthenticated route. Cache the origin-free template at module level (it is constant per release) and replace an origin token per request.

26 // The page's own first heading is the title already printed above its URL.
27 const body = docText(entry).replace(/^\s*# .*\n/, "").trim();
28 parts.push(`# ${entry.title}\n\n${ORIGIN}/docs/${name}\n\n${body}\n`);
29 }
outdated

The ETag is a SHA-256 of the whole concatenated guide set, recomputed on every request. Combined with the rebuild at line 26 this is the per-request work the issue ruled out. Fix: compute the template once, and derive the ETag from it plus the origin.

30 const text = parts.join("\n");
31 built = { template: text, digest: new Bun.CryptoHasher("sha256").update(text).digest("hex") };
32 return built;
33 }
34
35 export function llmsFull(ctx: Ctx): Response {
36 const { template: text, digest } = template();
37 if (text === "") return notFound();
38 const origin = ctx.url.origin;
39 // The template's digest is fixed; only the short origin is hashed per request.
40 const etag = `"${new Bun.CryptoHasher("sha256").update(digest).update("\n").update(origin).digest("hex").slice(0, 32)}"`;
41 const headers = { etag, "cache-control": "public, max-age=300" };
42 // Weakly: the edge hands the tag back as W/"…", and `*` holds any.
outdated

Compare weakly: t.trim().replace(/^W\//, "") === etag, and also accept *. Behind the edge the ETag comes back as W/"…" and this never matches, so no real client gets the 304. Add a test with a W/ prefix.

43 if (holds(ctx.request, etag)) return new Response(null, { status: 304, headers });
44 return new Response(text.split(ORIGIN).join(origin), { headers: { ...headers, "content-type": "text/plain; charset=utf-8" } });
45 }
src/web/standards.ts +1
@@ -532,6 +532,7 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s
532532 "",
533533 `- With search: ${o}/docs`,
534534 ...GUIDE.map((page) => `- ${o}/docs/${page}`),
535 `- All of them in one plain-text answer: ${o}/llms-full.txt`,
535536 "",
536537 ...threeDoors(o, mcp.url),
537538 "## Start here",
test/design-audit.test.ts +1 −1
@@ -17,7 +17,7 @@ const doc = readFileSync("docs/design/audit.md", "utf8");
1717
1818 /** Routes that answer a file, a fragment, a redirect hop or a callback. */
1919 const NOT_PAGES =
20 /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|explore\.atom$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/;
20 /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|explore\.atom$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/;
2121
2222 const getRoutes = [
2323 ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g),
test/forge-looks.test.ts +1
@@ -38,6 +38,7 @@ const NOT_A_PAGE: Record<string, string> = {
3838 "/forges": "no picture: its slot went to the preview's /waves and /milestones; the page is covered by its own tests",
3939 "/family": "no picture: its slot went to the preview's /waves and /milestones; the page is covered by its own tests",
4040 "/llms.txt": "plain markdown for machines, not a page to picture",
41 "/llms-full.txt": "plain text for machines, not a page to picture",
4142 "/agents": "/llms.txt drawn as a page for an assistant; no picture: not in app.looks, and test/agents-page.test.ts covers it",
4243 "/map": "no picture: its slot went to /security, the visitor's security page (#13208); covered by test/map-cache.test.ts and the other map tests",
4344 "/packs": "no picture: not in app.looks; the page is covered by test/packs.test.ts",
test/llms-full.test.ts added +74
@@ -0,0 +1,74 @@
1 /** /llms-full.txt: every guide in llms.txt's order, one build, a 304 on revalidation. */
2 import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import * as docs from "../src/view/docs.ts";
9 import { docEntry } from "../src/view/docs.ts";
10 import { GUIDE } from "../src/view/guide.ts";
11
12 let dataDir: string;
13 let app: ReturnType<typeof createApp>;
14 let servers: Servers;
15 const docText = spyOn(docs, "docText");
16
17 beforeAll(async () => {
18 dataDir = await mkdtemp(join(tmpdir(), "forge-llms-full-"));
19 app = createApp({ dataDir, port: 0 });
20 servers = await serve(app);
21 }, 30_000);
22
23 afterAll(async () => {
24 servers.stop();
25 app.close();
26 await rm(dataDir, { recursive: true, force: true });
27 });
28
29 test("every guide's title and URL are in it, in llms.txt's order", async () => {
30 const res = await fetch(`${servers.url}/llms-full.txt`);
31 expect(res.status).toBe(200);
32 expect(res.headers.get("content-type")).toContain("text/plain");
33 const body = await res.text();
34 let at = -1;
35 for (const name of GUIDE) {
36 const title = docEntry(name)!.title;
37 const heading = body.indexOf(`# ${title}\n\n${servers.url}/docs/${name}\n`);
38 expect(heading).toBeGreaterThan(at);
39 at = heading;
40 }
41 });
outdated

This test checks the 304 but nothing checks that the body is built once. Add one that fails when the build runs per request.

42
43 test("a second request is a 304 on the same ETag", async () => {
44 const a = await fetch(`${servers.url}/llms-full.txt`);
45 const etag = a.headers.get("etag")!;
46 expect(etag).toMatch(/^"[0-9a-f]+"$/);
47 const b = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": etag } });
48 expect(b.status).toBe(304);
49 expect(b.headers.get("etag")).toBe(etag);
50 });
51
52 test("the guides are read once however many requests come", async () => {
53 // Earlier tests in this file already requested it: one read per guide, never one per request.
54 const reads = docText.mock.calls.length;
55 expect(reads).toBeGreaterThan(0);
56 expect(reads).toBeLessThanOrEqual(GUIDE.length);
57 for (let i = 0; i < 3; i++) await (await fetch(`${servers.url}/llms-full.txt`)).text();
58 expect(docText.mock.calls.length).toBe(reads);
59 });
60
61 test("the ETag the edge weakens, or *, still earns a 304", async () => {
62 const etag = (await fetch(`${servers.url}/llms-full.txt`)).headers.get("etag")!;
63 for (const held of [`W/${etag}`, `"other", W/${etag}`, "*"]) {
64 const res = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": held } });
65 expect(res.status).toBe(304);
66 }
67 const other = await fetch(`${servers.url}/llms-full.txt`, { headers: { "if-none-match": 'W/"other"' } });
68 expect(other.status).toBe(200);
69 });
70
71 test("/llms.txt names it", async () => {
72 const body = await (await fetch(`${servers.url}/llms.txt`)).text();
73 expect(body).toContain(`${servers.url}/llms-full.txt`);
74 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs