Public: anyone can read this; the crew can change it.
The guide URLs that llms.txt lists as .md answer markdown with an ETag, not the HTML page #13892
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13864-the-guide-urls-that-llms-txt-lists-as-md → main 5 commits fork
4 files
changed +58
−10
src/web/layout.ts +6
| @@ -134,6 +134,8 @@ export interface PageOptions { | ||
| 134 | 134 | moreFeeds?: { href: string; title: string }[]; |
| 135 | 135 | /** This page's JSON twin, advertised in the head for agents. */ |
| 136 | 136 | json?: { href: string; title: string }; |
| 137 | /** This page's markdown source, advertised in the head for agents. */ | |
| 138 | markdown?: { href: string; title: string }; | |
| 137 | 139 | /** Structured data (JSON-LD) for the head, already escaped by `safeJson`. */ |
| 138 | 140 | jsonLd?: string; |
| 139 | 141 | /** Breadcrumb segments rendered in the topbar, e.g. owner / repo. */ |
| @@ -900,6 +902,10 @@ export function page(options: PageOptions, status = 200): Response { | ||
| 900 | 902 | options.json, |
| 901 | 903 | (j) => html`<link rel="alternate" type="application/json" title="${j.title}" href="${j.href}" />`, |
| 902 | 904 | )} |
| 905 | ${when( | |
| 906 | options.markdown, | |
| 907 | (m) => html`<link rel="alternate" type="text/markdown" title="${m.title}" href="${m.href}" />`, | |
| 908 | )} | |
| 903 | 909 | ${when(options.jsonLd, (d) => html`<script type="application/ld+json">${raw(d)}</script>`)} |
| 904 | 910 | <meta name="color-scheme" content="light dark" /> |
| 905 | 911 | ${theme |
src/web/pages/docs_link.ts +31 −5
| @@ -17,10 +17,21 @@ import { anyDocEntry, docEntries, docSpine, docText, featureEntries, searchDocs | ||
| 17 | 17 | import { html, raw, type Html } from "../html.ts"; |
| 18 | 18 | import { renderMarkdown } from "../markdown.ts"; |
| 19 | 19 | import { page } from "../layout.ts"; |
| 20 | import { GUIDE } from "../../view/guide.ts"; | |
| 21 | import { holds } from "../../http/etag.ts"; | |
| 20 | 22 | import { row } from "../ui.ts"; |
| 21 | 23 | |
| 22 | 24 | const rendered = new Map<string, string>(); |
| 23 | 25 | |
| 26 | const guides: ReadonlySet<string> = new Set(GUIDE); | |
| 27 | ||
| 28 | /** | |
| 29 | * Where a doc's page is read: a guide's page is `/docs/guide-x`, its `.md` is the markdown source. | |
| 30 | */ | |
| 31 | export function docHref(name: string): string { | |
| 32 | return `/docs/${guides.has(name) ? name.slice(0, -3) : name}`; | |
| 33 | } | |
| 34 | ||
| 24 | 35 | /** |
| 25 | 36 | * `OTHER.md` and `./OTHER.md` (with an anchor) become `/docs/OTHER.md` when OTHER.md is in the |
| 26 | 37 | * listing; any other link stays as written. |
| @@ -28,7 +39,7 @@ const rendered = new Map<string, string>(); | ||
| 28 | 39 | export function relinkDocs(markup: string, known: ReadonlySet<string> = new Set([...docEntries(), ...featureEntries()].map((e) => e.name))): string { |
| 29 | 40 | return markup.replace( |
| 30 | 41 | /href="(?:\.\/)?((?:features\/)?[A-Za-z0-9][A-Za-z0-9._-]{0,80}\.md)(#[^"]*)?"/g, |
| 31 | (m, name: string, tail: string | undefined) => (known.has(name) ? `href="/docs/${name}${tail ?? ""}"` : m), | |
| 42 | (m, name: string, tail: string | undefined) => (known.has(name) ? `href="${docHref(name)}${tail ?? ""}"` : m), | |
| 32 | 43 | ); |
| 33 | 44 | } |
| 34 | 45 | |
| @@ -56,7 +67,7 @@ export function anchorHeadings(markup: string): string { | ||
| 56 | 67 | * so a bare link in a list, styled by .docs-index; the closing </li> is optional HTML and left off. |
| 57 | 68 | */ |
| 58 | 69 | function indexRow(e: { name: string; title: string }): Html { |
| 59 | return html`<li><a href="/docs/${e.name}">${e.title}</a>`; | |
| 70 | return html`<li><a href="${docHref(e.name)}">${e.title}</a>`; | |
| 60 | 71 | } |
| 61 | 72 | |
| 62 | 73 | export function docsFolder(ctx: Ctx): Response { |
| @@ -74,7 +85,7 @@ export function docsFolder(ctx: Ctx): Response { | ||
| 74 | 85 | body = html`<h1>Documentation</h1>${form}${trimmed} |
| 75 | 86 | ${hits.length === 0 |
| 76 | 87 | ? html`<p>Nothing matches “${echoed(q)}”. Try one word.</p>` |
| 77 | : html`<ul class="list">${hits.map((h) => row({ href: `/docs/${h.name}`, title: h.title, meta: html`${h.snippet}` }))}</ul>`}`; | |
| 88 | : html`<ul class="list">${hits.map((h) => row({ href: docHref(h.name), title: h.title, meta: html`${h.snippet}` }))}</ul>`}`; | |
| 78 | 89 | } else { |
| 79 | 90 | const { guide: spine, rest } = docSpine(); |
| 80 | 91 | body = html`<h1>Documentation</h1>${form}${trimmed} |
| @@ -97,7 +108,21 @@ export function docsFolder(ctx: Ctx): Response { | ||
| 97 | 108 | export function docsFile(ctx: Ctx): Response { |
| 98 | 109 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 99 | 110 | const began = performance.now(); |
| 100 | return docPage(ctx, ctx.params.file ?? "", began); | |
| 111 | const file = ctx.params.file ?? ""; | |
| 112 | if (guides.has(file)) return guideMarkdown(ctx, file); | |
| 113 | // The guide's page has no extension: `/docs/guide-x` is HTML, `/docs/guide-x.md` its source. | |
| 114 | return docPage(ctx, guides.has(`${file}.md`) ? `${file}.md` : file, began); | |
| 115 | } | |
| 116 | ||
| 117 | /** A guide's markdown source, tagged by its bytes so a repeat read is a 304. */ | |
| 118 | function guideMarkdown(ctx: Ctx, name: string): Response { | |
| 119 | const entry = anyDocEntry(name); | |
| 120 | if (entry === undefined) return notFound(); | |
| 121 | const text = docText(entry); | |
| 122 | const etag = `"${Bun.hash(text).toString(36)}"`; | |
| 123 | const headers = { "content-type": "text/markdown; charset=utf-8", etag, "cache-control": "public, no-cache" }; | |
| 124 | if (holds(ctx.request, etag)) return new Response(null, { status: 304, headers }); | |
| 125 | return new Response(ctx.request.method === "HEAD" ? null : text, { headers }); | |
| 101 | 126 | } |
| 102 | 127 | |
| 103 | 128 | /** /docs/features/NAME.md: the reference notes the guide pages link to. */ |
| @@ -124,8 +149,9 @@ function docPage(ctx: Ctx, name: string, began: number): Response { | ||
| 124 | 149 | title: entry.title, |
| 125 | 150 | crumbs: [ |
| 126 | 151 | { href: "/docs", label: "Docs" }, |
| 127 | { href: `/docs/${name}`, label: name.replace(/^features\//, "") }, | |
| 152 | { href: docHref(name), label: name.replace(/^features\//, "") }, | |
| 128 | 153 | ], |
| 154 | ...(guides.has(name) ? { markdown: { href: `/docs/${name}`, title: `${entry.title} (markdown)` } } : {}), | |
| 129 | 155 | body, |
| 130 | 156 | }); |
| 131 | 157 | notePhase(ctx.request, "render", performance.now() - began); |
test/docs-link.test.ts +2 −2
| @@ -8,7 +8,7 @@ import { tmpdir } from "node:os"; | ||
| 8 | 8 | import { join } from "node:path"; |
| 9 | 9 | import { createApp } from "../src/app.ts"; |
| 10 | 10 | import { serve, type Servers } from "../src/http/server.ts"; |
| 11 | import { relinkDocs } from "../src/web/pages/docs_link.ts"; | |
| 11 | import { docHref, relinkDocs } from "../src/web/pages/docs_link.ts"; | |
| 12 | 12 | import { GUIDE } from "../src/view/guide.ts"; |
| 13 | 13 | import { docEntries, titleOf } from "../src/view/docs.ts"; |
| 14 | 14 | import { expectWork, requestStatements } from "./helpers/work.ts"; |
| @@ -112,7 +112,7 @@ test("a link is rewritten only when the name is in the listing; titles come from | ||
| 112 | 112 | |
| 113 | 113 | test("the /docs index stays small: every document linked, under 15 KB", async () => { |
| 114 | 114 | const text = await (await get("/docs")).text(); |
| 115 | for (const e of docEntries()) expect(text).toContain(`href="/docs/${e.name}"`); | |
| 115 | for (const e of docEntries()) expect(text).toContain(`href="${docHref(e.name)}"`); | |
| 116 | 116 | // Production chrome is larger than the test app's; |
| 117 | 117 | // 14.5 KB here leaves it room under the issue's 15 KB (the footer gained Status and Privacy). |
| 118 | 118 | expect(new TextEncoder().encode(text).length).toBeLessThan(14.5 * 1024); |
test/guide.test.ts +19 −3
| @@ -131,7 +131,7 @@ describe("the pages", () => { | ||
| 131 | 131 | describe("/docs", () => { |
| 132 | 132 | test("the index puts the guide first, in order, before the reference", async () => { |
| 133 | 133 | const text = await (await get("/docs")).text(); |
| 134 | const at = GUIDE.map((n) => text.indexOf(`href="/docs/${n}"`)); | |
| 134 | const at = GUIDE.map((n) => text.indexOf(`href="/docs/${n.slice(0, -3)}"`)); | |
| 135 | 135 | expect(at.every((i) => i > 0)).toBe(true); |
| 136 | 136 | expect([...at].sort((a, b) => a - b)).toEqual(at); |
| 137 | 137 | expect(text.indexOf("href=\"/docs/VISION.md\"")).toBeGreaterThan(at[at.length - 1] ?? 0); |
| @@ -141,7 +141,7 @@ describe("/docs", () => { | ||
| 141 | 141 | test("search finds the guide first, escapes what was typed and shrugs at nothing", async () => { |
| 142 | 142 | expect(searchDocs("tell the crew")[0]?.name).toBe("guide-tell-the-crew.md"); |
| 143 | 143 | const res = await get("/docs?q=" + encodeURIComponent("tell crew")); |
| 144 | expect(await res.text()).toContain('href="/docs/guide-tell-the-crew.md"'); | |
| 144 | expect(await res.text()).toContain('href="/docs/guide-tell-the-crew"'); | |
| 145 | 145 | const xss = await (await get("/docs?q=" + encodeURIComponent("<script>alert(1)</script>"))).text(); |
| 146 | 146 | expect(xss).not.toContain("<script>alert(1)"); |
| 147 | 147 | expect(await (await get("/docs?q=zzzqqqxx")).text()).toContain("Nothing matches"); |
| @@ -149,13 +149,29 @@ describe("/docs", () => { | ||
| 149 | 149 | }); |
| 150 | 150 | |
| 151 | 151 | test("a guide page and a feature note render, and a link to a note resolves in /docs", async () => { |
| 152 | const page = await get("/docs/guide-your-apps.md"); | |
| 152 | const page = await get("/docs/guide-your-apps"); | |
| 153 | 153 | expect(page.status).toBe(200); |
| 154 | 154 | expect(await page.text()).toContain('href="/docs/features/app-own-domain.md"'); |
| 155 | 155 | expect((await get("/docs/features/plans.md")).status).toBe(200); |
| 156 | 156 | for (const bad of ["..%2FVISION.md", "nope.md", "%2e%2e%2f%2e%2e%2fpackage.json"]) expect((await get(`/docs/features/${bad}`)).status).toBe(404); |
| 157 | 157 | }); |
| 158 | 158 | |
| 159 | test("a guide's .md is its markdown with an ETag and a 304; the page links it; an unknown guide is a 404", async () => { | |
| 160 | const res = await get("/docs/guide-first-app.md"); | |
| 161 | expect(res.status).toBe(200); | |
| 162 | expect(res.headers.get("content-type")).toBe("text/markdown; charset=utf-8"); | |
| 163 | expect(await res.text()).toStartWith("# "); | |
| 164 | const etag = res.headers.get("etag")!; | |
| 165 | expect(etag).toBeTruthy(); | |
| 166 | const again = await fetch(`${servers.url}/docs/guide-first-app.md`, { headers: { "if-none-match": etag } }); | |
| 167 | expect(again.status).toBe(304); | |
| 168 | const html = await (await get("/docs/guide-first-app")).text(); | |
| 169 | expect(html).toContain('rel="alternate" type="text/markdown"'); | |
| 170 | expect(html).toContain('href="/docs/guide-first-app.md"'); | |
| 171 | expect((await get("/docs/guide-nope.md")).status).toBe(404); | |
| 172 | expect((await get("/docs/guide-nope")).status).toBe(404); | |
| 173 | }); | |
| 174 | ||
| 159 | 175 | test("llms.txt points an agent at the guide", async () => { |
| 160 | 176 | const text = await (await get("/llms.txt")).text(); |
| 161 | 177 | expect(text).toContain("/docs/guide-first-app.md"); |
new above
offline