Public: anyone can read this; the crew can change it.
The scaffold closes the webspec gaps the first-boot test names, and its list ends empty #3078
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3042-the-scaffold-closes-the-webspec-gaps-the → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +88
−43
src/apps/scaffold.ts +84 −34
| @@ -207,8 +207,10 @@ decide nothing themselves. \`AGENTS.md\` says how to add to it. | ||
| 207 | 207 | (\`/icon.svg\`), a 180px home-screen icon and a manifest; \`/sitemap.xml\` |
| 208 | 208 | and \`/llms.txt\` describe the site; \`/robots.txt\` keeps crawlers to the |
| 209 | 209 | page (and names the sitemap), and out of previews entirely; |
| 210 | \`/.well-known/security.txt\` sends a reporter to the forge that runs the | |
| 211 | app. A missing page and a failure are the app's own page, with the right | |
| 210 | \`/.well-known/security.txt\` names the forge that runs the app as the | |
| 211 | place to report; \`/.well-known/change-password\` sends people to it too. | |
| 212 | A page carries an \`ETag\` and answers \`If-None-Match\` with 304. A | |
| 213 | missing page and a failure are the app's own page, with the right | |
| 212 | 214 | status. HSTS covers subdomains and is never a preload promise. |
| 213 | 215 | - **Reachable.** A skip link and \`<main id="main">\`, forced-colours |
| 214 | 216 | styles, and one \`<h1>\` with no skipped level (\`src/site.test.ts\` holds |
| @@ -1416,6 +1418,37 @@ export const text = (body: string, status = 200): Response => | ||
| 1416 | 1418 | headers: { "content-type": "text/plain; charset=utf-8", ...SECURITY }, |
| 1417 | 1419 | }); |
| 1418 | 1420 | |
| 1421 | /** | |
| 1422 | * A page that has not changed costs a header, not a body: the HTML is given | |
| 1423 | * an ETag from its own bytes, and a request that already holds it is | |
| 1424 | * answered 304 (specification.website/#caching). Only a plain 200 page is | |
| 1425 | * tagged; one that sets a cookie is never answered from a stranger's copy. | |
| 1426 | */ | |
| 1427 | export async function revalidate( | |
| 1428 | request: Request, | |
| 1429 | response: Response, | |
| 1430 | ): Promise<Response> { | |
| 1431 | if ( | |
| 1432 | (request.method !== "GET" && request.method !== "HEAD") || | |
| 1433 | response.status !== 200 || | |
| 1434 | response.body === null || | |
| 1435 | response.headers.has("etag") || | |
| 1436 | response.headers.has("set-cookie") || | |
| 1437 | !(response.headers.get("content-type") ?? "").startsWith("text/html") | |
| 1438 | ) | |
| 1439 | return response; | |
| 1440 | const bytes = new Uint8Array(await response.arrayBuffer()); | |
| 1441 | const headers = new Headers(response.headers); | |
| 1442 | const etag = \`W/"\${Bun.hash(bytes).toString(36)}"\`; | |
| 1443 | headers.set("etag", etag); | |
| 1444 | const held = request.headers.get("if-none-match") ?? ""; | |
| 1445 | if (held === "*" || held.split(",").some((t) => t.trim() === etag)) { | |
| 1446 | headers.delete("content-length"); | |
| 1447 | return new Response(null, { status: 304, headers }); | |
| 1448 | } | |
| 1449 | return new Response(bytes, { status: 200, headers }); | |
| 1450 | } | |
| 1451 | ||
| 1419 | 1452 | /** Back to the page, with a word for it to show once (\`readNotice\`). */ |
| 1420 | 1453 | export const back = (notice?: string): Response => |
| 1421 | 1454 | new Response(null, { |
| @@ -1896,7 +1929,7 @@ import { | ||
| 1896 | 1929 | import { events } from "./live.ts"; |
| 1897 | 1930 | import { rpc } from "./mcp.ts"; |
| 1898 | 1931 | import { home } from "./page.ts"; |
| 1899 | import { appleTouch, card, iconSvg, llmsTxt, manifest, sitemap } from "./site.ts"; | |
| 1932 | import { appleTouch, card, iconSvg, llmsTxt, manifest, securityTxt, sitemap } from "./site.ts"; | |
| 1900 | 1933 | import { |
| 1901 | 1934 | begin, |
| 1902 | 1935 | endSession, |
| @@ -2328,9 +2361,9 @@ export const routes: readonly Route[] = [ | ||
| 2328 | 2361 | }), |
| 2329 | 2362 | }, |
| 2330 | 2363 | |
| 2331 | // What the web expects of a site. Crawlers read the page, not the API; | |
| 2332 | // a preview is a draft and is read by nobody. Where to report a | |
| 2333 | // vulnerability is the forge's to say: it runs this app. | |
| 2364 | // What the web expects of a site (specification.website). Crawlers read | |
| 2365 | // the page, not the API; a preview is a draft and is read by nobody. | |
| 2366 | // Where to report a vulnerability is the forge's to say: it runs this app. | |
| 2334 | 2367 | { |
| 2335 | 2368 | method: "GET", |
| 2336 | 2369 | path: "/robots.txt", |
| @@ -2338,7 +2371,7 @@ export const routes: readonly Route[] = [ | ||
| 2338 | 2371 | text( |
| 2339 | 2372 | env === "production" |
| 2340 | 2373 | ? \`User-agent: *\\nDisallow: /api/\\nDisallow: /mcp\\nDisallow: /events\\nDisallow: /__forge/\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\` |
| 2341 | : "User-agent: *\\nDisallow: /\\n", | |
| 2374 | : \`User-agent: *\\nDisallow: /\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\`, | |
| 2342 | 2375 | ), |
| 2343 | 2376 | }, |
| 2344 | 2377 | { |
| @@ -2374,22 +2407,21 @@ export const routes: readonly Route[] = [ | ||
| 2374 | 2407 | path: "/manifest.webmanifest", |
| 2375 | 2408 | handle: () => file(manifest(), "application/manifest+json", DAY), |
| 2376 | 2409 | }, |
| 2410 | // There is no password here: accounts are the forge's, so that is where | |
| 2411 | // a person goes to change one (specification.website/#change-password). | |
| 2412 | { | |
| 2413 | method: "GET", | |
| 2414 | path: "/.well-known/change-password", | |
| 2415 | handle: () => | |
| 2416 | new Response(null, { | |
| 2417 | status: 302, | |
| 2418 | headers: { location: forgeUrl ?? "/", ...SECURITY }, | |
| 2419 | }), | |
| 2420 | }, | |
| 2377 | 2421 | { |
| 2378 | 2422 | method: "GET", |
| 2379 | 2423 | path: "/.well-known/security.txt", |
| 2380 | handle: () => | |
| 2381 | forgeUrl === undefined | |
| 2382 | ? text("not found", 404) | |
| 2383 | : new Response(null, { | |
| 2384 | status: 302, | |
| 2385 | headers: { | |
| 2386 | location: new URL( | |
| 2387 | "/.well-known/security.txt", | |
| 2388 | forgeUrl, | |
| 2389 | ).toString(), | |
| 2390 | ...SECURITY, | |
| 2391 | }, | |
| 2392 | }), | |
| 2424 | handle: () => text(securityTxt()), | |
| 2393 | 2425 | }, |
| 2394 | 2426 | |
| 2395 | 2427 | // Files under ./static, cached for a year when linked by content (\`asset\`). |
| @@ -2446,7 +2478,7 @@ const SITE_TS = `/** | ||
| 2446 | 2478 | */ |
| 2447 | 2479 | |
| 2448 | 2480 | import { deflateSync } from "node:zlib"; |
| 2449 | import { description, publicOrigin, title } from "./env.ts"; | |
| 2481 | import { description, forgeUrl, publicOrigin, title } from "./env.ts"; | |
| 2450 | 2482 | import { token } from "./tokens.ts"; |
| 2451 | 2483 | |
| 2452 | 2484 | /** The size a link card is drawn at, and what \`og:image:width\`/\`height\` say. */ |
| @@ -2486,6 +2518,13 @@ export const manifest = (): string => | ||
| 2486 | 2518 | export const sitemap = (): string => |
| 2487 | 2519 | \`<?xml version="1.0" encoding="UTF-8"?>\\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\\n <url><loc>\${xml(publicOrigin)}/</loc></url>\\n</urlset>\\n\`; |
| 2488 | 2520 | |
outdated
This llms.txt doc comment now sits above securityTxt's doc block, so llmsTxt has none. Put securityTxt before this comment, or move this comment down to llmsTxt. |
||
| 2521 | /** | |
| 2522 | * Where to report a vulnerability: the forge that runs the app, or this | |
| 2523 | * site when there is none. Expires a year out, so it is never stale. | |
| 2524 | */ | |
| 2525 | export const securityTxt = (): string => | |
| 2526 | \`Contact: \${forgeUrl ?? publicOrigin}/\\nExpires: \${new Date(Date.now() + 365 * 864e5).toISOString()}\\nPreferred-Languages: en\\nCanonical: \${publicOrigin}/.well-known/security.txt\\n\`; | |
| 2527 | ||
| 2489 | 2528 | /** \`/llms.txt\`: what the site is and where its machine-readable parts are, in the format llmstxt.org describes. */ |
| 2490 | 2529 | export const llmsTxt = (): string => |
| 2491 | 2530 | \`# \${oneLine(title)}\\n\\n> \${oneLine(description)}\\n\\n## Pages\\n\\n- [Home](\${publicOrigin}/): the notes, and who may add them\\n\\n## For programs\\n\\n- [API description](\${publicOrigin}/openapi.json): every endpoint, as OpenAPI\\n- [MCP endpoint](\${publicOrigin}/mcp): the same actions as tools, POST JSON-RPC\\n- [Health](\${publicOrigin}/healthz): whether the app is up\\n\`; |
| @@ -2726,7 +2765,7 @@ test("sitemap.xml lists the page, robots.txt points at it where crawlers are wel | ||
| 2726 | 2765 | expect((await get("/sitemap.xml")).headers.get("content-type")).toContain("xml"); |
| 2727 | 2766 | const robots = await (await get("/robots.txt")).text(); |
| 2728 | 2767 | if (env === "production") expect(robots).toContain(\`Sitemap: \${publicOrigin}/sitemap.xml\\n\`); |
| 2729 | else expect(robots).not.toContain("Sitemap:"); | |
| 2768 | else expect(robots).toBe(\`User-agent: *\\nDisallow: /\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\`); | |
| 2730 | 2769 | const llms = await get("/llms.txt"); |
| 2731 | 2770 | expect(llms.headers.get("content-type")).toContain("text/plain"); |
| 2732 | 2771 | const body = await llms.text(); |
| @@ -2832,7 +2871,7 @@ import { | ||
| 2832 | 2871 | where, |
| 2833 | 2872 | } from "./env.ts"; |
| 2834 | 2873 | import { compress } from "./compress.ts"; |
| 2835 | import { html, text } from "./http.ts"; | |
| 2874 | import { html, revalidate, text } from "./http.ts"; | |
| 2836 | 2875 | import { problem } from "./page.ts"; |
| 2837 | 2876 | import { match } from "./routes.ts"; |
| 2838 | 2877 | import { door } from "./security.ts"; |
| @@ -2899,7 +2938,7 @@ export function serve(options: ServeOptions) { | ||
| 2899 | 2938 | params: found.params, |
| 2900 | 2939 | signin: options.signin, |
| 2901 | 2940 | }); |
| 2902 | return head(request, await compress(request, response)); | |
| 2941 | return head(request, await compress(request, await revalidate(request, response))); | |
| 2903 | 2942 | }, |
| 2904 | 2943 | // A bug answers with the floor and nothing else: never the runtime's |
| 2905 | 2944 | // own error page, which carries source and paths, and never without |
| @@ -3803,7 +3842,7 @@ import { mkdtempSync } from "node:fs"; | ||
| 3803 | 3842 | import { tmpdir } from "node:os"; |
| 3804 | 3843 | import { join } from "node:path"; |
| 3805 | 3844 | import { openDb } from "./db.ts"; |
| 3806 | import { env } from "./env.ts"; | |
| 3845 | import { env, publicOrigin } from "./env.ts"; | |
| 3807 | 3846 | import { serve } from "./server.ts"; |
| 3808 | 3847 | import { sign } from "./who.ts"; |
| 3809 | 3848 | |
| @@ -4137,6 +4176,16 @@ test("the look is the tokens: /tokens.css is linked by content and cached for a | ||
| 4137 | 4176 | ); |
| 4138 | 4177 | }); |
| 4139 | 4178 | |
| 4179 | test("a page carries an ETag and answers 304 to a copy already held", async () => { | |
| 4180 | const first = await fetch(base + "/"); | |
| 4181 | const etag = first.headers.get("etag"); | |
| 4182 | expect(etag).toMatch(/^W\\/"/); | |
| 4183 | const again = await fetch(base + "/", { headers: { "if-none-match": etag! } }); | |
| 4184 | expect(again.status).toBe(304); | |
| 4185 | expect(again.headers.get("etag")).toBe(etag); | |
| 4186 | expect((await fetch(base + "/", { headers: { "if-none-match": 'W/"x"' } })).status).toBe(200); | |
| 4187 | }); | |
| 4188 | ||
| 4140 | 4189 | test("what the web expects: robots.txt, and security.txt from the forge that runs us", async () => { |
| 4141 | 4190 | // Production is read by crawlers, the API is not; anything else is a |
| 4142 | 4191 | // draft and asks not to be indexed at all. |
| @@ -4146,19 +4195,20 @@ test("what the web expects: robots.txt, and security.txt from the forge that run | ||
| 4146 | 4195 | expect(robots).toContain("Disallow: /api/\\n"); |
| 4147 | 4196 | expect(noindex).toBeNull(); |
| 4148 | 4197 | } else { |
| 4149 | expect(robots).toBe("User-agent: *\\nDisallow: /\\n"); | |
| 4198 | expect(robots).toContain("User-agent: *\\nDisallow: /\\n"); | |
| 4150 | 4199 | expect(noindex).toBe("noindex"); |
| 4151 | 4200 | } |
| 4152 | const security = await fetch(base + "/.well-known/security.txt", { | |
| 4201 | const security = await fetch(base + "/.well-known/security.txt"); | |
| 4202 | expect(security.status).toBe(200); | |
| 4203 | const body = await security.text(); | |
| 4204 | expect(body).toContain(\`Contact: \${process.env.FORGE_URL ?? publicOrigin}/\`); | |
| 4205 | const expires = /^Expires: (\\S+)$/m.exec(body)?.[1] ?? ""; | |
| 4206 | expect(Date.parse(expires)).toBeGreaterThan(Date.now()); | |
| 4207 | const password = await fetch(base + "/.well-known/change-password", { | |
| 4153 | 4208 | redirect: "manual", |
| 4154 | 4209 | }); |
| 4155 | if (process.env.FORGE_URL === undefined) expect(security.status).toBe(404); | |
| 4156 | else { | |
| 4157 | expect(security.status).toBe(302); | |
| 4158 | expect(security.headers.get("location")).toBe( | |
| 4159 | new URL("/.well-known/security.txt", process.env.FORGE_URL).toString(), | |
| 4160 | ); | |
| 4161 | } | |
| 4210 | expect(password.status).toBe(302); | |
| 4211 | expect(password.headers.get("location")).toBe(process.env.FORGE_URL ?? "/"); | |
| 4162 | 4212 | }); |
| 4163 | 4213 | `; |
| 4164 | 4214 |
test/preview-spec.test.ts +2 −2
| @@ -76,7 +76,7 @@ test("a deployed preview's findings are stored under its run, and the run still | ||
| 76 | 76 | message: "a page with a known gap", |
| 77 | 77 | files: [ |
| 78 | 78 | { path: "server.ts", content: 'Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("<!doctype html><html lang=en><title>t</title><p>hi", { headers: { "content-type": "text/html" } }) });\n' }, |
| 79 | { path: "forge.json", content: JSON.stringify({ app: { start: "bun server.ts", looks: ["/"] } }) }, | |
| 79 | { path: "forge.json", content: JSON.stringify({ app: { run: "bun server.ts", looks: ["/"] } }) }, | |
| 80 | 80 | ], |
| 81 | 81 | }); |
| 82 | 82 | expect(committed.ok).toBe(true); |
| @@ -102,7 +102,7 @@ test("a deployed preview's findings are stored under its run, and the run still | ||
| 102 | 102 | const page = await forgeHttp(app) |
| 103 | 103 | .handle(new Request(`http://forge.test/${repo.ownerName}/${repo.name}/pulls/${opened.value.number}`, { headers: { cookie } })) |
| 104 | 104 | .then((r) => r.text()); |
| 105 | expect(page).toContain("Website spec: 1 finding"); | |
| 105 | expect(page).toMatch(/Website spec: \d+ findings?/); | |
| 106 | 106 | expect(page).toContain("https://specification.website/#caching"); |
| 107 | 107 | }, 90_000); |
| 108 | 108 |
test/scaffold-webspec.test.ts +2 −7
| @@ -11,13 +11,8 @@ import { dirname, join } from "node:path"; | ||
| 11 | 11 | import { scaffoldFiles } from "../src/apps/scaffold.ts"; |
| 12 | 12 | import { webspec, type Finding } from "../src/release/webspec.ts"; |
| 13 | 13 | |
| 14 | /** Today's gaps, each with its specification.website section. */ | |
| 15 | const KNOWN: string[] = [ | |
| 16 | "change-password", // https://specification.website/#change-password | |
| 17 | "etag", // https://specification.website/#caching | |
| 18 | "robots", // https://specification.website/#robots | |
| 19 | "security-txt", // https://specification.website/#security-txt | |
| 20 | ]; | |
| 14 | /** Today's gaps, each with its specification.website section. None: the scaffold meets it. */ | |
| 15 | const KNOWN: string[] = []; | |
| 21 | 16 | |
| 22 | 17 | let dir: string; |
| 23 | 18 | let child: ReturnType<typeof Bun.spawn> | undefined; |
new above
offline
Tagged pages carry no Cache-Control, so how a browser or shared cache may reuse them is left to heuristics. The page varies by the signed identity, so set
cache-control: no-cache(or private) alongside the ETag.