Public: anyone can read this; the crew can change it.
/llms.txt answers a strong ETag and a 304 on If-None-Match, as /sitemap.xml does #8338
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8329-llms-txt-answers-a-strong-etag-and-a-304 → main 1 commit fork
2 files
changed +49
−7
src/web/standards.ts +30 −7
| @@ -30,7 +30,7 @@ import { metaThemeColor } from "../design/themes.ts"; | ||
| 30 | 30 | import { BODY_MAX, TITLE_MAX } from "../core/reports.ts"; |
| 31 | 31 | import { REPORT_LINK_TITLE } from "../actions/report_words.ts"; |
| 32 | 32 | import { sshTellCommand } from "../ssh/command.ts"; |
| 33 | import { llmsRepos, sitemapRepos } from "../core/listings.ts"; | |
| 33 | import { llmsRepos, sitemapRepos, type LlmsListing } from "../core/listings.ts"; | |
| 34 | 34 | import { canRead, getRepo } from "../core/repos.ts"; |
| 35 | 35 | import type { Ctx } from "../http/context.ts"; |
| 36 | 36 | import { notFound } from "../http/respond.ts"; |
| @@ -326,6 +326,9 @@ function runningSentence(): string { | ||
| 326 | 326 | : `Running ${r.version} at ${r.commit.slice(0, 7)}`; |
| 327 | 327 | } |
| 328 | 328 | |
| 329 | /** llms.txt and its ETag per origin, valid while the listing read, the build and the tell command are the same. */ | |
| 330 | const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; body: string; etag: string }>(); | |
| 331 | ||
| 329 | 332 | /** |
| 330 | 333 | * GET /llms.txt — the forge as an agent should meet it (llmstxt.org): what |
| 331 | 334 | * this is, how to report a problem with no account, how to offer a change, |
| @@ -334,14 +337,35 @@ function runningSentence(): string { | ||
| 334 | 337 | */ |
| 335 | 338 | export function llmsTxt(ctx: Ctx): Response { |
| 336 | 339 | const o = ctx.url.origin; |
| 337 | const mcp = mcpPointers(o); | |
| 338 | 340 | const tell = sshTellCommand(ctx.app); |
| 341 | const running = runningSentence(); | |
| 339 | 342 | const repos = timePhase(ctx.request, "repos", () => llmsRepos(ctx.app.db)); |
| 340 | return timePhase(ctx.request, "render", () => text( | |
| 343 | let kept = llmsBodies.get(o); | |
| 344 | if (kept === undefined || kept.repos !== repos || kept.running !== running || kept.tell !== tell) { | |
| 345 | const body = timePhase(ctx.request, "render", () => renderLlmsTxt(ctx, o, tell, running, repos)); | |
| 346 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); | |
| 347 | kept = { repos, running, tell, body, etag }; | |
| 348 | // Origins come from the Host header: keep a handful, not one per guess. | |
| 349 | if (llmsBodies.size >= 16) llmsBodies.clear(); | |
| 350 | llmsBodies.set(o, kept); | |
| 351 | } | |
| 352 | const cache = "public, max-age=3600"; | |
| 353 | const sent = ctx.request.headers.get("if-none-match"); | |
| 354 | if (sent !== null && sent.split(",").some((t) => t.trim() === kept.etag)) | |
| 355 | return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache } }); | |
| 356 | const res = text(kept.body, "text/markdown; charset=utf-8"); | |
| 357 | res.headers.set("etag", kept.etag); | |
| 358 | return res; | |
| 359 | } | |
| 360 | ||
| 361 | /** The llms.txt document for one origin. */ | |
| 362 | function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: string, repos: LlmsListing): string { | |
| 363 | const mcp = mcpPointers(o); | |
| 364 | return ( | |
| 341 | 365 | [ |
| 342 | 366 | `# forge at ${ctx.url.host}`, |
| 343 | 367 | "", |
| 344 | runningSentence(), | |
| 368 | running, | |
| 345 | 369 | "", |
| 346 | 370 | "> Git repositories, their issues and pulls, and the apps they run. Anyone may read a public repository, and report a problem in it without an account.", |
| 347 | 371 | "", |
| @@ -409,9 +433,8 @@ export function llmsTxt(ctx: Ctx): Response { | ||
| 409 | 433 | ] |
| 410 | 434 | : ["None yet."]), |
| 411 | 435 | "", |
| 412 | ].join("\n"), | |
| 413 | "text/markdown; charset=utf-8", | |
| 414 | )); | |
| 436 | ].join("\n") | |
| 437 | ); | |
| 415 | 438 | } |
| 416 | 439 | |
| 417 | 440 | export interface PageLink { |
test/standards.test.ts +19
| @@ -204,6 +204,25 @@ describe("what the open web expects", () => { | ||
| 204 | 204 | } |
| 205 | 205 | }); |
| 206 | 206 | |
| 207 | test("llms.txt answers a strong ETag and a 304, and a new public repository changes the ETag", async () => { | |
| 208 | const first = await fetch(`${base}/llms.txt`); | |
| 209 | const etag = first.headers.get("etag")!; | |
| 210 | expect(etag).toMatch(/^"[0-9a-f]{32}"$/); | |
| 211 | await first.text(); | |
| 212 | const again = await fetch(`${base}/llms.txt`, { headers: { "if-none-match": etag } }); | |
| 213 | expect(again.status).toBe(304); | |
| 214 | expect(await again.text()).toBe(""); | |
| 215 | expect(again.headers.get("etag")).toBe(etag); | |
| 216 | const made = await createRepo(app, alice, { name: "etagged", visibility: "public" }); | |
| 217 | if (!made.ok) throw new Error(String(made.error)); | |
| 218 | // A new repository reaches the listing when its minute is up. | |
| 219 | forgetListings(app.db); | |
| 220 | const after = await fetch(`${base}/llms.txt`, { headers: { "if-none-match": etag } }); | |
| 221 | expect(after.status).toBe(200); | |
| 222 | expect(after.headers.get("etag")).not.toBe(etag); | |
| 223 | expect(await after.text()).toContain("alice/etagged"); | |
| 224 | }); | |
| 225 | ||
| 207 | 226 | test("llms.txt opens with the version, and the short commit only when one is known", async () => { |
| 208 | 227 | const saved = { |
| 209 | 228 | v: process.env["FORGE_VERSION"], |
new above
offline