Public: anyone can read this; the crew can change it.

The guide URLs that llms.txt lists as .md answer markdown with an ETag, not the HTML page #13892

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13864-the-guide-urls-that-llms-txt-lists-as-md → main 5 commits fork

  • Merged. scout/13864-the-guide-urls-that-llms-txt-lists-as-md was merged into main as 56fe7d8.

Review

Sign in to review this change.
4 files changed +58 −10
src/web/layout.ts +6
@@ -134,6 +134,8 @@ export interface PageOptions {
134134 moreFeeds?: { href: string; title: string }[];
135135 /** This page's JSON twin, advertised in the head for agents. */
136136 json?: { href: string; title: string };
137 /** This page's markdown source, advertised in the head for agents. */
138 markdown?: { href: string; title: string };
137139 /** Structured data (JSON-LD) for the head, already escaped by `safeJson`. */
138140 jsonLd?: string;
139141 /** Breadcrumb segments rendered in the topbar, e.g. owner / repo. */
@@ -900,6 +902,10 @@ export function page(options: PageOptions, status = 200): Response {
900902 options.json,
901903 (j) => html`<link rel="alternate" type="application/json" title="${j.title}" href="${j.href}" />`,
902904 )}
905 ${when(
906 options.markdown,
907 (m) => html`<link rel="alternate" type="text/markdown" title="${m.title}" href="${m.href}" />`,
908 )}
903909 ${when(options.jsonLd, (d) => html`<script type="application/ld+json">${raw(d)}</script>`)}
904910 <meta name="color-scheme" content="light dark" />
905911 ${theme
test/guide.test.ts +19 −3
@@ -131,7 +131,7 @@ describe("the pages", () => {
131131 describe("/docs", () => {
132132 test("the index puts the guide first, in order, before the reference", async () => {
133133 const text = await (await get("/docs")).text();
134 const at = GUIDE.map((n) => text.indexOf(`href="/docs/${n}"`));
134 const at = GUIDE.map((n) => text.indexOf(`href="/docs/${n.slice(0, -3)}"`));
135135 expect(at.every((i) => i > 0)).toBe(true);
136136 expect([...at].sort((a, b) => a - b)).toEqual(at);
137137 expect(text.indexOf("href=\"/docs/VISION.md\"")).toBeGreaterThan(at[at.length - 1] ?? 0);
@@ -141,7 +141,7 @@ describe("/docs", () => {
141141 test("search finds the guide first, escapes what was typed and shrugs at nothing", async () => {
142142 expect(searchDocs("tell the crew")[0]?.name).toBe("guide-tell-the-crew.md");
143143 const res = await get("/docs?q=" + encodeURIComponent("tell crew"));
144 expect(await res.text()).toContain('href="/docs/guide-tell-the-crew.md"');
144 expect(await res.text()).toContain('href="/docs/guide-tell-the-crew"');
145145 const xss = await (await get("/docs?q=" + encodeURIComponent("<script>alert(1)</script>"))).text();
146146 expect(xss).not.toContain("<script>alert(1)");
147147 expect(await (await get("/docs?q=zzzqqqxx")).text()).toContain("Nothing matches");
@@ -149,13 +149,29 @@ describe("/docs", () => {
149149 });
150150
151151 test("a guide page and a feature note render, and a link to a note resolves in /docs", async () => {
152 const page = await get("/docs/guide-your-apps.md");
152 const page = await get("/docs/guide-your-apps");
153153 expect(page.status).toBe(200);
154154 expect(await page.text()).toContain('href="/docs/features/app-own-domain.md"');
155155 expect((await get("/docs/features/plans.md")).status).toBe(200);
156156 for (const bad of ["..%2FVISION.md", "nope.md", "%2e%2e%2f%2e%2e%2fpackage.json"]) expect((await get(`/docs/features/${bad}`)).status).toBe(404);
157157 });
158158
159 test("a guide's .md is its markdown with an ETag and a 304; the page links it; an unknown guide is a 404", async () => {
160 const res = await get("/docs/guide-first-app.md");
161 expect(res.status).toBe(200);
162 expect(res.headers.get("content-type")).toBe("text/markdown; charset=utf-8");
163 expect(await res.text()).toStartWith("# ");
164 const etag = res.headers.get("etag")!;
165 expect(etag).toBeTruthy();
166 const again = await fetch(`${servers.url}/docs/guide-first-app.md`, { headers: { "if-none-match": etag } });
167 expect(again.status).toBe(304);
168 const html = await (await get("/docs/guide-first-app")).text();
169 expect(html).toContain('rel="alternate" type="text/markdown"');
170 expect(html).toContain('href="/docs/guide-first-app.md"');
171 expect((await get("/docs/guide-nope.md")).status).toBe(404);
172 expect((await get("/docs/guide-nope")).status).toBe(404);
173 });
174
159175 test("llms.txt points an agent at the guide", async () => {
160176 const text = await (await get("/llms.txt")).text();
161177 expect(text).toContain("/docs/guide-first-app.md");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs