Public: anyone can read this; the crew can change it.

The scaffold closes the webspec gaps the first-boot test names, and its list ends empty #3078

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/3042-the-scaffold-closes-the-webspec-gaps-the → main 2 commits fork 2 threads · 2 unresolved

  • Merged. scout/3042-the-scaffold-closes-the-webspec-gaps-the was merged into main as 6a97f19.

Review

Sign in to review this change.
3 files changed +88 −43
src/apps/scaffold.ts +84 −34
@@ -207,8 +207,10 @@ decide nothing themselves. \`AGENTS.md\` says how to add to it.
207207 (\`/icon.svg\`), a 180px home-screen icon and a manifest; \`/sitemap.xml\`
208208 and \`/llms.txt\` describe the site; \`/robots.txt\` keeps crawlers to the
209209 page (and names the sitemap), and out of previews entirely;
210 \`/.well-known/security.txt\` sends a reporter to the forge that runs the
211 app. A missing page and a failure are the app's own page, with the right
210 \`/.well-known/security.txt\` names the forge that runs the app as the
211 place to report; \`/.well-known/change-password\` sends people to it too.
212 A page carries an \`ETag\` and answers \`If-None-Match\` with 304. A
213 missing page and a failure are the app's own page, with the right
212214 status. HSTS covers subdomains and is never a preload promise.
213215 - **Reachable.** A skip link and \`<main id="main">\`, forced-colours
214216 styles, and one \`<h1>\` with no skipped level (\`src/site.test.ts\` holds
@@ -1416,6 +1418,37 @@ export const text = (body: string, status = 200): Response =>
14161418 headers: { "content-type": "text/plain; charset=utf-8", ...SECURITY },
14171419 });
14181420
1421 /**
1422 * A page that has not changed costs a header, not a body: the HTML is given
1423 * an ETag from its own bytes, and a request that already holds it is
1424 * answered 304 (specification.website/#caching). Only a plain 200 page is
1425 * tagged; one that sets a cookie is never answered from a stranger's copy.
1426 */
1427 export async function revalidate(
1428 request: Request,
1429 response: Response,
1430 ): Promise<Response> {
1431 if (
1432 (request.method !== "GET" && request.method !== "HEAD") ||
1433 response.status !== 200 ||
1434 response.body === null ||

Tagged pages carry no Cache-Control, so how a browser or shared cache may reuse them is left to heuristics. The page varies by the signed identity, so set cache-control: no-cache (or private) alongside the ETag.

1435 response.headers.has("etag") ||
1436 response.headers.has("set-cookie") ||
1437 !(response.headers.get("content-type") ?? "").startsWith("text/html")
1438 )
1439 return response;
1440 const bytes = new Uint8Array(await response.arrayBuffer());
1441 const headers = new Headers(response.headers);
1442 const etag = \`W/"\${Bun.hash(bytes).toString(36)}"\`;
1443 headers.set("etag", etag);
1444 const held = request.headers.get("if-none-match") ?? "";
1445 if (held === "*" || held.split(",").some((t) => t.trim() === etag)) {
1446 headers.delete("content-length");
1447 return new Response(null, { status: 304, headers });
1448 }
1449 return new Response(bytes, { status: 200, headers });
1450 }
1451
14191452 /** Back to the page, with a word for it to show once (\`readNotice\`). */
14201453 export const back = (notice?: string): Response =>
14211454 new Response(null, {
@@ -1896,7 +1929,7 @@ import {
18961929 import { events } from "./live.ts";
18971930 import { rpc } from "./mcp.ts";
18981931 import { home } from "./page.ts";
1899 import { appleTouch, card, iconSvg, llmsTxt, manifest, sitemap } from "./site.ts";
1932 import { appleTouch, card, iconSvg, llmsTxt, manifest, securityTxt, sitemap } from "./site.ts";
19001933 import {
19011934 begin,
19021935 endSession,
@@ -2328,9 +2361,9 @@ export const routes: readonly Route[] = [
23282361 }),
23292362 },
23302363
2331 // What the web expects of a site. Crawlers read the page, not the API;
2332 // a preview is a draft and is read by nobody. Where to report a
2333 // vulnerability is the forge's to say: it runs this app.
2364 // What the web expects of a site (specification.website). Crawlers read
2365 // the page, not the API; a preview is a draft and is read by nobody.
2366 // Where to report a vulnerability is the forge's to say: it runs this app.
23342367 {
23352368 method: "GET",
23362369 path: "/robots.txt",
@@ -2338,7 +2371,7 @@ export const routes: readonly Route[] = [
23382371 text(
23392372 env === "production"
23402373 ? \`User-agent: *\\nDisallow: /api/\\nDisallow: /mcp\\nDisallow: /events\\nDisallow: /__forge/\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\`
2341 : "User-agent: *\\nDisallow: /\\n",
2374 : \`User-agent: *\\nDisallow: /\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\`,
23422375 ),
23432376 },
23442377 {
@@ -2374,22 +2407,21 @@ export const routes: readonly Route[] = [
23742407 path: "/manifest.webmanifest",
23752408 handle: () => file(manifest(), "application/manifest+json", DAY),
23762409 },
2410 // There is no password here: accounts are the forge's, so that is where
2411 // a person goes to change one (specification.website/#change-password).
2412 {
2413 method: "GET",
2414 path: "/.well-known/change-password",
2415 handle: () =>
2416 new Response(null, {
2417 status: 302,
2418 headers: { location: forgeUrl ?? "/", ...SECURITY },
2419 }),
2420 },
23772421 {
23782422 method: "GET",
23792423 path: "/.well-known/security.txt",
2380 handle: () =>
2381 forgeUrl === undefined
2382 ? text("not found", 404)
2383 : new Response(null, {
2384 status: 302,
2385 headers: {
2386 location: new URL(
2387 "/.well-known/security.txt",
2388 forgeUrl,
2389 ).toString(),
2390 ...SECURITY,
2391 },
2392 }),
2424 handle: () => text(securityTxt()),
23932425 },
23942426
23952427 // Files under ./static, cached for a year when linked by content (\`asset\`).
@@ -2446,7 +2478,7 @@ const SITE_TS = `/**
24462478 */
24472479
24482480 import { deflateSync } from "node:zlib";
2449 import { description, publicOrigin, title } from "./env.ts";
2481 import { description, forgeUrl, publicOrigin, title } from "./env.ts";
24502482 import { token } from "./tokens.ts";
24512483
24522484 /** The size a link card is drawn at, and what \`og:image:width\`/\`height\` say. */
@@ -2486,6 +2518,13 @@ export const manifest = (): string =>
24862518 export const sitemap = (): string =>
24872519 \`<?xml version="1.0" encoding="UTF-8"?>\\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\\n <url><loc>\${xml(publicOrigin)}/</loc></url>\\n</urlset>\\n\`;
24882520
outdated

This llms.txt doc comment now sits above securityTxt's doc block, so llmsTxt has none. Put securityTxt before this comment, or move this comment down to llmsTxt.

2521 /**
2522 * Where to report a vulnerability: the forge that runs the app, or this
2523 * site when there is none. Expires a year out, so it is never stale.
2524 */
2525 export const securityTxt = (): string =>
2526 \`Contact: \${forgeUrl ?? publicOrigin}/\\nExpires: \${new Date(Date.now() + 365 * 864e5).toISOString()}\\nPreferred-Languages: en\\nCanonical: \${publicOrigin}/.well-known/security.txt\\n\`;
2527
24892528 /** \`/llms.txt\`: what the site is and where its machine-readable parts are, in the format llmstxt.org describes. */
24902529 export const llmsTxt = (): string =>
24912530 \`# \${oneLine(title)}\\n\\n> \${oneLine(description)}\\n\\n## Pages\\n\\n- [Home](\${publicOrigin}/): the notes, and who may add them\\n\\n## For programs\\n\\n- [API description](\${publicOrigin}/openapi.json): every endpoint, as OpenAPI\\n- [MCP endpoint](\${publicOrigin}/mcp): the same actions as tools, POST JSON-RPC\\n- [Health](\${publicOrigin}/healthz): whether the app is up\\n\`;
@@ -2726,7 +2765,7 @@ test("sitemap.xml lists the page, robots.txt points at it where crawlers are wel
27262765 expect((await get("/sitemap.xml")).headers.get("content-type")).toContain("xml");
27272766 const robots = await (await get("/robots.txt")).text();
27282767 if (env === "production") expect(robots).toContain(\`Sitemap: \${publicOrigin}/sitemap.xml\\n\`);
2729 else expect(robots).not.toContain("Sitemap:");
2768 else expect(robots).toBe(\`User-agent: *\\nDisallow: /\\n\\nSitemap: \${publicOrigin}/sitemap.xml\\n\`);
27302769 const llms = await get("/llms.txt");
27312770 expect(llms.headers.get("content-type")).toContain("text/plain");
27322771 const body = await llms.text();
@@ -2832,7 +2871,7 @@ import {
28322871 where,
28332872 } from "./env.ts";
28342873 import { compress } from "./compress.ts";
2835 import { html, text } from "./http.ts";
2874 import { html, revalidate, text } from "./http.ts";
28362875 import { problem } from "./page.ts";
28372876 import { match } from "./routes.ts";
28382877 import { door } from "./security.ts";
@@ -2899,7 +2938,7 @@ export function serve(options: ServeOptions) {
28992938 params: found.params,
29002939 signin: options.signin,
29012940 });
2902 return head(request, await compress(request, response));
2941 return head(request, await compress(request, await revalidate(request, response)));
29032942 },
29042943 // A bug answers with the floor and nothing else: never the runtime's
29052944 // own error page, which carries source and paths, and never without
@@ -3803,7 +3842,7 @@ import { mkdtempSync } from "node:fs";
38033842 import { tmpdir } from "node:os";
38043843 import { join } from "node:path";
38053844 import { openDb } from "./db.ts";
3806 import { env } from "./env.ts";
3845 import { env, publicOrigin } from "./env.ts";
38073846 import { serve } from "./server.ts";
38083847 import { sign } from "./who.ts";
38093848
@@ -4137,6 +4176,16 @@ test("the look is the tokens: /tokens.css is linked by content and cached for a
41374176 );
41384177 });
41394178
4179 test("a page carries an ETag and answers 304 to a copy already held", async () => {
4180 const first = await fetch(base + "/");
4181 const etag = first.headers.get("etag");
4182 expect(etag).toMatch(/^W\\/"/);
4183 const again = await fetch(base + "/", { headers: { "if-none-match": etag! } });
4184 expect(again.status).toBe(304);
4185 expect(again.headers.get("etag")).toBe(etag);
4186 expect((await fetch(base + "/", { headers: { "if-none-match": 'W/"x"' } })).status).toBe(200);
4187 });
4188
41404189 test("what the web expects: robots.txt, and security.txt from the forge that runs us", async () => {
41414190 // Production is read by crawlers, the API is not; anything else is a
41424191 // draft and asks not to be indexed at all.
@@ -4146,19 +4195,20 @@ test("what the web expects: robots.txt, and security.txt from the forge that run
41464195 expect(robots).toContain("Disallow: /api/\\n");
41474196 expect(noindex).toBeNull();
41484197 } else {
4149 expect(robots).toBe("User-agent: *\\nDisallow: /\\n");
4198 expect(robots).toContain("User-agent: *\\nDisallow: /\\n");
41504199 expect(noindex).toBe("noindex");
41514200 }
4152 const security = await fetch(base + "/.well-known/security.txt", {
4201 const security = await fetch(base + "/.well-known/security.txt");
4202 expect(security.status).toBe(200);
4203 const body = await security.text();
4204 expect(body).toContain(\`Contact: \${process.env.FORGE_URL ?? publicOrigin}/\`);
4205 const expires = /^Expires: (\\S+)$/m.exec(body)?.[1] ?? "";
4206 expect(Date.parse(expires)).toBeGreaterThan(Date.now());
4207 const password = await fetch(base + "/.well-known/change-password", {
41534208 redirect: "manual",
41544209 });
4155 if (process.env.FORGE_URL === undefined) expect(security.status).toBe(404);
4156 else {
4157 expect(security.status).toBe(302);
4158 expect(security.headers.get("location")).toBe(
4159 new URL("/.well-known/security.txt", process.env.FORGE_URL).toString(),
4160 );
4161 }
4210 expect(password.status).toBe(302);
4211 expect(password.headers.get("location")).toBe(process.env.FORGE_URL ?? "/");
41624212 });
41634213 `;
41644214
test/preview-spec.test.ts +2 −2
@@ -76,7 +76,7 @@ test("a deployed preview's findings are stored under its run, and the run still
7676 message: "a page with a known gap",
7777 files: [
7878 { path: "server.ts", content: 'Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("<!doctype html><html lang=en><title>t</title><p>hi", { headers: { "content-type": "text/html" } }) });\n' },
79 { path: "forge.json", content: JSON.stringify({ app: { start: "bun server.ts", looks: ["/"] } }) },
79 { path: "forge.json", content: JSON.stringify({ app: { run: "bun server.ts", looks: ["/"] } }) },
8080 ],
8181 });
8282 expect(committed.ok).toBe(true);
@@ -102,7 +102,7 @@ test("a deployed preview's findings are stored under its run, and the run still
102102 const page = await forgeHttp(app)
103103 .handle(new Request(`http://forge.test/${repo.ownerName}/${repo.name}/pulls/${opened.value.number}`, { headers: { cookie } }))
104104 .then((r) => r.text());
105 expect(page).toContain("Website spec: 1 finding");
105 expect(page).toMatch(/Website spec: \d+ findings?/);
106106 expect(page).toContain("https://specification.website/#caching");
107107 }, 90_000);
108108
test/scaffold-webspec.test.ts +2 −7
@@ -11,13 +11,8 @@ import { dirname, join } from "node:path";
1111 import { scaffoldFiles } from "../src/apps/scaffold.ts";
1212 import { webspec, type Finding } from "../src/release/webspec.ts";
1313
14 /** Today's gaps, each with its specification.website section. */
15 const KNOWN: string[] = [
16 "change-password", // https://specification.website/#change-password
17 "etag", // https://specification.website/#caching
18 "robots", // https://specification.website/#robots
19 "security-txt", // https://specification.website/#security-txt
20 ];
14 /** Today's gaps, each with its specification.website section. None: the scaffold meets it. */
15 const KNOWN: string[] = [];
2116
2217 let dir: string;
2318 let child: ReturnType<typeof Bun.spawn> | undefined;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs