trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns11g rEventsFlows

Website spec checks 3-4: webspec() reports well-known files and error-page findings #2973

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/2954-website-spec-checks-3-4-webspec-reports → main 3 commits fork

  • Merged. scout/2954-website-spec-checks-3-4-webspec-reports was merged into main as dceb8f8.

Review

Sign in to review this change.
5 files changed +301 −10
README.md +13 −5
@@ -315,11 +315,19 @@ pull adds the proof but does not reopen signup — the gitops gate stays for a
315315 person.
316316
317317 `bun scripts/smoke.ts <url> --standards` prints the website-specification
318 report (`src/release/webspec.ts`): it fetches each path in forge.json's
319 `app.looks` from `<url>` and prints one line per head or header finding with
320 the specification.website section it cites. It only reports and always exits
321 0. `(edge)` marks a finding the edge in front of the app may own (HSTS), so
322 fix it there rather than in the app. Each fetch gives up after 10 seconds.
318 report (`src/release/webspec.ts`, `webspec-site.ts`): it fetches each path in
319 forge.json's `app.looks` from `<url>` and prints one line per finding with the
320 specification.website section it cites. Head and header findings come per
321 path. Once per site it also checks the well-known files (robots.txt and the
322 sitemap it names, with each listed same-origin URL, at most 50, fetched
323 anonymously; `/.well-known/security.txt` with a future `Expires:`; the web
324 manifest with a name and icons; `/llms.txt`; `/.well-known/change-password`
325 redirecting) and the error pages (a random path must answer 404 with the app's
326 own HTML page). `--error-path <path>` names a path that is meant to answer 500
327 with the custom page; the 500 check runs only when it is named, because the
328 report never causes an error on its own. It only reports and always exits 0.
329 `(edge)` marks a finding the edge in front of the app may own (HSTS), so fix it
330 there rather than in the app. Each fetch gives up after 10 seconds.
323331
324332 What you asked for: `Ctrl+K`, then **Asked**, is `/asked` in the terminal — your
325333 thoughts, then the issues you can write to that are not shipped, oldest first
scripts/smoke.ts +5 −3
@@ -12,7 +12,8 @@
1212 *
1313 * `--standards` runs the website-specification report (`release/webspec.ts`)
1414 * over forge.json's `app.looks` paths, prints one line per finding with its
15 * section, and exits 0: it reports, it does not gate.
15 * section (head, headers, well-known files, error pages; `--error-path`
16 * names a path that must answer 500), and exits 0: it reports, it does not gate.
1617 *
1718 * `--outsider` also signs up a fresh account and proves each thing a
1819 * stranger must not do is refused over HTTP and MCP (`release/outsider.ts`,
@@ -37,12 +38,13 @@ if (import.meta.main) {
3738 if (withOutsider) args.splice(args.indexOf("--outsider"), 1);
3839 const standards = args.includes("--standards");
3940 if (standards) args.splice(args.indexOf("--standards"), 1);
41 const errorPath = flag("--error-path");
4042 const version = flag("--version");
4143 const wait = flag("--wait");
4244 const base = args[0];
4345 const seconds = wait === undefined ? 120 : Number(wait);
4446 if (!base || !Number.isFinite(seconds) || seconds < 0 || args.length > 1) {
45 console.error("usage: bun scripts/smoke.ts <url> [--version <v>] [--wait <seconds>] [--outsider] [--standards] (--version needs FORGE_PROBE, the forge's probe)");
47 console.error("usage: bun scripts/smoke.ts <url> [--version <v>] [--wait <seconds>] [--outsider] [--standards [--error-path <path that answers 500>]] (--version needs FORGE_PROBE, the forge's probe)");
4648 process.exit(2);
4749 }
4850 if (withOutsider && !process.env["FORGE_TOKEN"]) {
@@ -52,7 +54,7 @@ if (import.meta.main) {
5254 if (standards) {
5355 const looks = (await Bun.file(new URL("../forge.json", import.meta.url)).json())?.app?.looks;
5456 const paths: string[] = Array.isArray(looks) ? looks.filter((p): p is string => typeof p === "string") : ["/"];
55 const findings = await webspec(base, paths);
57 const findings = await webspec(base, paths, fetch, { site: true, ...(errorPath ? { errorPath } : {}) });
5658 for (const f of findings) console.log(`${f.path} ${f.check}${f.edge ? " (edge)" : ""}: ${f.message} <${f.section}>`);
5759 console.log(`standards: ${findings.length} finding(s) over ${paths.length} path(s); report only`);
5860 process.exit(0);
src/release/webspec-site.ts added +157
@@ -0,0 +1,157 @@
1 /**
2 * The website specification as a report (#2915, checks 3 and 4): the
3 * well-known files a site answers once per base, and its error pages. Same
4 * contract as `webspec.ts`: findings with a section, nothing thrown.
5 */
6
7 import type { Finding } from "./webspec.ts";
8
9 type Fetcher = (input: string, init?: RequestInit) => Promise<Response>;
10 type Seen = { status: number; headers: Headers; text: string };
11
12 const TIMEOUT_MS = 10000;
13 /** The most sitemap URLs fetched, so a huge sitemap cannot make the report slow. */
14 export const SITEMAP_CAP = 50;
15 const s = (name: string) => `https://specification.website/#${name}`;
16 const SIGN_IN_SEGMENT = /^(sign-?in|log-?in|auth|sessions?)$/i;
17
18 /** Whether a redirect target is a sign-in page, judged by whole path segments so /authors is not one. */
19 function isSignIn(where: string, root: URL): boolean {
20 try {
21 return new URL(where, root).pathname.split("/").some((seg) => SIGN_IN_SEGMENT.test(seg));
22 } catch {
23 return false; // an unparsable Location is not a sign-in page
24 }
25 }
26
27 export interface SiteOptions {
28 /** A path that is meant to answer 500 with the custom page; 500 is checked only when named. */
29 errorPath?: string;
30 }
31
32 async function see(fetcher: Fetcher, url: string, accept = "*/*", body = true): Promise<Seen | undefined> {
33 try {
34 const r = await fetcher(url, { headers: { accept }, redirect: "manual", signal: AbortSignal.timeout(TIMEOUT_MS) });
35 if (!body) {
36 await r.body?.cancel();
37 return { status: r.status, headers: r.headers, text: "" };
38 }
39 return { status: r.status, headers: r.headers, text: await r.text() };
40 } catch {
41 return undefined;
42 }
43 }
44
45 /** Whether a body is the app's own page: HTML with a <title> and a stylesheet. */
46 export function isOwnPage(r: Seen): boolean {
47 const type = r.headers.get("content-type") ?? "";
48 return /html/i.test(type) && /<title\b/i.test(r.text) && (/<link\b[^>]*\brel\s*=\s*["']?stylesheet/i.test(r.text) || /<style\b/i.test(r.text));
49 }
50
51 export async function siteFindings(base: string, fetcher: Fetcher, opts: SiteOptions = {}): Promise<Finding[]> {
52 const out: Finding[] = [];
53 const add = (path: string, check: string, message: string, section: string) => out.push({ path, check, message, section: s(section) });
54 let root: URL;
55 try {
56 root = new URL(base);
57 } catch {
58 return [{ path: "/", check: "fetch", message: `${base} is not a URL`, section: "https://specification.website/" }];
59 }
60 const at = (p: string) => new URL(p, root).href;
61 const ok = (r?: Seen) => !!r && r.status >= 200 && r.status < 300;
62
63 // robots.txt and the sitemap it names
64 const robots = await see(fetcher, at("/robots.txt"));
65 if (!ok(robots)) add("/robots.txt", "robots", `/robots.txt answered ${robots?.status ?? "nothing"}`, "robots");
66 else {
67 const line = /^sitemap:[ \t]*(\S+)/im.exec(robots!.text);
68 if (!line) add("/robots.txt", "robots", "robots.txt has no Sitemap: line", "robots");
69 else {
70 let map: URL | undefined;
71 try {
72 map = new URL(line[1]!, root);
73 } catch {
74 map = undefined; // reported below as not on this site
75 }
76 if (!map || map.origin !== root.origin) add("/robots.txt", "sitemap", `Sitemap: ${line[1]} is not on this site`, "sitemap");
77 else {
78 const sm = await see(fetcher, map.href);
79 if (!ok(sm)) add(map.pathname, "sitemap", `${map.href} answered ${sm?.status ?? "nothing"}`, "sitemap");
80 else {
81 const locs = [...sm!.text.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1]!.replace(/&amp;/g, "&"));
82 for (const loc of locs.slice(0, SITEMAP_CAP)) {
83 let u: URL;
84 try {
85 u = new URL(loc, root);
86 } catch {
87 continue;
88 }
89 if (u.origin !== root.origin) continue;
90 const r = await see(fetcher, u.href, "text/html", false);
91 if (!r) continue;
92 const where = r.headers.get("location") ?? "";
93 if (r.status === 401 || r.status === 403 || (r.status >= 300 && r.status < 400 && isSignIn(where, root)))
94 add(map.pathname, "sitemap", `${u.href} is listed but answers ${r.status}${where ? ` to ${where}` : ""} for an anonymous fetch`, "sitemap");
95 }
96 }
97 }
98 }
99 }
100
101 // security.txt
102 const sec = await see(fetcher, at("/.well-known/security.txt"));
103 if (!ok(sec)) add("/.well-known/security.txt", "security-txt", `security.txt answered ${sec?.status ?? "nothing"}`, "security-txt");
104 else {
105 const exp = /^expires:[ \t]*(\S+)/im.exec(sec!.text);
106 const when = exp ? Date.parse(exp[1]!) : NaN;
107 if (!exp) add("/.well-known/security.txt", "security-txt", "security.txt has no Expires:", "security-txt");
108 else if (!(when > Date.now())) add("/.well-known/security.txt", "security-txt", `security.txt Expires: ${exp[1]} is not in the future`, "security-txt");
109 }
110
111 // manifest: the one the home page links, else the conventional name
112 const home = await see(fetcher, at("/"), "text/html");
113 const link = home && /<link\b[^>]*\brel\s*=\s*["']?manifest\b[^>]*>/i.exec(home.text)?.[0];
114 const href = link && /\bhref\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(link);
115 const manifestUrl = at(href ? (href[1] ?? href[2] ?? "/manifest.webmanifest") : "/manifest.webmanifest");
116 const man = await see(fetcher, manifestUrl);
117 if (!ok(man)) add("/manifest", "manifest", `${manifestUrl} answered ${man?.status ?? "nothing"}`, "manifest");
118 else {
119 let j: unknown;
120 try {
121 j = JSON.parse(man!.text);
122 } catch {
123 j = undefined; // reported below as not a JSON object
124 }
125 const o = j && typeof j === "object" ? (j as Record<string, unknown>) : undefined;
126 if (!o) add("/manifest", "manifest", "the manifest is not a JSON object", "manifest");
127 else {
128 if (typeof o["name"] !== "string" || !o["name"].trim()) add("/manifest", "manifest", "the manifest has no name", "manifest");
129 if (!Array.isArray(o["icons"]) || o["icons"].length === 0) add("/manifest", "manifest", "the manifest has no icons", "manifest");
130 }
131 }
132
133 // llms.txt and change-password
134 const llms = await see(fetcher, at("/llms.txt"));
135 if (!ok(llms)) add("/llms.txt", "llms-txt", `/llms.txt answered ${llms?.status ?? "nothing"}`, "llms-txt");
136 const pw = await see(fetcher, at("/.well-known/change-password"));
137 if (!pw || pw.status < 300 || pw.status >= 400 || !pw.headers.get("location"))
138 add("/.well-known/change-password", "change-password", `change-password did not redirect (${pw?.status ?? "nothing"})`, "change-password");
139
140 // error pages
141 const missing = `/__webspec-${crypto.randomUUID().replace(/-/g, "")}`;
142 const nf = await see(fetcher, at(missing), "text/html");
143 if (!nf) add(missing, "error-404", "a random path did not answer", "error-pages");
144 else {
145 if (nf.status !== 404) add(missing, "error-404", `a random path answered ${nf.status}, not 404`, "error-pages");
146 if (!isOwnPage(nf)) add(missing, "error-404", "the 404 body is not the app's own HTML page (title and stylesheet)", "error-pages");
147 }
148 if (opts.errorPath) {
149 const e = await see(fetcher, at(opts.errorPath), "text/html");
150 if (!e) add(opts.errorPath, "error-500", "the error path did not answer", "error-pages");
151 else {
152 if (e.status !== 500) add(opts.errorPath, "error-500", `the error path answered ${e.status}, not 500`, "error-pages");
153 if (!isOwnPage(e)) add(opts.errorPath, "error-500", "the 500 body is not the app's own HTML page (title and stylesheet)", "error-pages");
154 }
155 }
156 return out;
157 }
src/release/webspec.ts +9 −2
@@ -6,6 +6,8 @@
66 * The HTML is read with small regexes, one function per check.
77 */
88
9 import { siteFindings } from "./webspec-site.ts";
10
911 export interface Finding {
1012 path: string;
1113 check: string;
@@ -132,8 +134,12 @@ export function encodingProblems(h: Headers, bodyBytes: number): Problem[] {
132134 return out;
133135 }
134136
135 /** Fetches `paths` of `base` and reports head and header findings. */
136 export async function webspec(base: string, paths: string[], fetcher: Fetcher = fetch): Promise<Finding[]> {
137 /**
138 * Fetches `paths` of `base` and reports head and header findings. With
139 * `opts.site` it also reports the well-known files and error pages, once for
140 * the base (checks 3 and 4); `opts.errorPath` names a path that must answer 500.
141 */
142 export async function webspec(base: string, paths: string[], fetcher: Fetcher = fetch, opts: { site?: boolean; errorPath?: string } = {}): Promise<Finding[]> {
137143 const findings: Finding[] = [];
138144 for (const path of paths) {
139145 const report = (ps: Problem[]) => {
@@ -190,5 +196,6 @@ export async function webspec(base: string, paths: string[], fetcher: Fetcher =
190196 if (again !== 304) report([{ check: "etag", message: `If-None-Match answered ${again || "nothing"}, not 304`, section: s("caching") }]);
191197 }
192198 }
199 if (opts.site) findings.push(...(await siteFindings(base, fetcher, opts.errorPath ? { errorPath: opts.errorPath } : {})));
193200 return findings;
194201 }
test/webspec-site.test.ts added +117
@@ -0,0 +1,117 @@
1 import { expect, test } from "bun:test";
2 import { webspec } from "../src/release/webspec.ts";
3 import { SITEMAP_CAP } from "../src/release/webspec-site.ts";
4
5 const PAGE = `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Not found</title><link rel="stylesheet" href="/a.css"><link rel="manifest" href="/m.json"></head><body>x</body></html>`;
6 const html = { "content-type": "text/html" };
7 const future = new Date(Date.now() + 86400e3 * 100).toISOString();
8
9 type Route = Response;
10 function good(): Record<string, Route> {
11 return {
12 "/": new Response(PAGE, { headers: html }),
13 "/robots.txt": new Response("User-agent: *\nSitemap: https://x.test/sitemap.xml\n"),
14 "/sitemap.xml": new Response("<urlset><url><loc>https://x.test/</loc></url><url><loc>https://x.test/a</loc></url></urlset>"),
15 "/a": new Response("ok"),
16 "/.well-known/security.txt": new Response(`Contact: mailto:[email protected]\nExpires: ${future}\n`),
17 "/m.json": new Response(JSON.stringify({ name: "X", icons: [{ src: "/i.png" }] })),
18 "/llms.txt": new Response("# X"),
19 "/.well-known/change-password": new Response(null, { status: 302, headers: { location: "/account" } }),
20 };
21 }
22
23 const run = async (over: Record<string, Route | null> = {}, errorPath?: string) => {
24 const routes: Record<string, Route | null> = { ...good(), ...over };
25 const f = async (input: string): Promise<Response> => {
26 const p = new URL(input).pathname;
27 const r = p in routes ? routes[p] : p.startsWith("/__webspec-") ? new Response(PAGE, { status: 404, headers: html }) : undefined;
28 if (r === undefined || r === null) return new Response("nope", { status: 404 });
29 return r.clone() as Response;
30 };
31 return (await webspec("https://x.test", [], f, { site: true, ...(errorPath ? { errorPath } : {}) })).map((x) => x.check);
32 };
33
34 test("a site meeting checks 3 and 4 has no findings, and every finding cites a section", async () => {
35 expect(await run()).toEqual([]);
36 const bad = await webspec("https://x.test", [], async () => new Response("no", { status: 500 }), { site: true });
37 expect(bad.length).toBeGreaterThan(0);
38 expect(bad.every((f) => f.section.startsWith("https://specification.website/#"))).toBe(true);
39 });
40
41 test("without opts.site nothing extra is fetched", async () => {
42 let n = 0;
43 await webspec("https://x.test", [], async () => (n++, new Response("")));
44 expect(n).toBe(0);
45 });
46
47 test("robots.txt must answer and name a sitemap", async () => {
48 expect(await run({ "/robots.txt": null })).toEqual(["robots"]);
49 expect(await run({ "/robots.txt": new Response("User-agent: *\n") })).toEqual(["robots"]);
50 });
51
52 test("the sitemap must answer and list no URL a stranger is refused", async () => {
53 const map = (loc: string) => new Response(`<urlset><url><loc>${loc}</loc></url></urlset>`);
54 expect(await run({ "/sitemap.xml": null })).toEqual(["sitemap"]);
55 expect(await run({ "/sitemap.xml": map("https://x.test/a"), "/a": new Response("", { status: 403 }) })).toEqual(["sitemap"]);
56 expect(await run({ "/sitemap.xml": map("https://x.test/a"), "/a": new Response("", { status: 401 }) })).toEqual(["sitemap"]);
57 expect(await run({ "/sitemap.xml": map("https://x.test/a"), "/a": new Response("", { status: 302, headers: { location: "/sign-in?next=/a" } }) })).toEqual(["sitemap"]);
58 expect(await run({ "/sitemap.xml": map("https://x.test/a"), "/a": new Response("", { status: 301, headers: { location: "/b" } }) })).toEqual([]);
59 expect(await run({ "/sitemap.xml": map("https://x.test/a"), "/a": new Response("", { status: 301, headers: { location: "/authors" } }) })).toEqual([]);
60 });
61
62 test("the sitemap walk is capped and stays on the site", async () => {
63 const locs = Array.from({ length: SITEMAP_CAP + 30 }, (_, i) => `<url><loc>https://x.test/p${i}</loc></url>`).join("") + "<url><loc>https://evil.test/z</loc></url>";
64 const seen: string[] = [];
65 const f = async (input: string) => {
66 const u = new URL(input);
67 seen.push(u.href);
68 if (u.pathname === "/robots.txt") return new Response("Sitemap: https://x.test/sitemap.xml");
69 if (u.pathname === "/sitemap.xml") return new Response(`<urlset>${locs}</urlset>`);
70 return new Response("", { status: 404 });
71 };
72 await webspec("https://x.test", [], f, { site: true });
73 expect(seen.filter((u) => /\/p\d+$/.test(u)).length).toBe(SITEMAP_CAP);
74 expect(seen.some((u) => u.includes("evil.test"))).toBe(false);
75 });
76
77 test("security.txt must answer with an Expires in the future", async () => {
78 expect(await run({ "/.well-known/security.txt": null })).toEqual(["security-txt"]);
79 expect(await run({ "/.well-known/security.txt": new Response("Contact: a") })).toEqual(["security-txt"]);
80 expect(await run({ "/.well-known/security.txt": new Response("Expires: 2001-01-01T00:00:00Z") })).toEqual(["security-txt"]);
81 });
82
83 test("the manifest must parse and carry name and icons", async () => {
84 expect(await run({ "/m.json": null })).toEqual(["manifest"]);
85 expect(await run({ "/m.json": new Response("{nope") })).toEqual(["manifest"]);
86 expect(await run({ "/m.json": new Response(JSON.stringify({ icons: [{}] })) })).toEqual(["manifest"]);
87 expect(await run({ "/m.json": new Response(JSON.stringify({ name: "X", icons: [] })) })).toEqual(["manifest"]);
88 });
89
90 test("llms.txt must answer and change-password must redirect", async () => {
91 expect(await run({ "/llms.txt": null })).toEqual(["llms-txt"]);
92 expect(await run({ "/.well-known/change-password": new Response("hi") })).toEqual(["change-password"]);
93 expect(await run({ "/.well-known/change-password": null })).toEqual(["change-password"]);
94 });
95
96 test("the random path must answer 404 with the app's own page", async () => {
97 const withRandom = async (res: Response) => {
98 const g: Record<string, Route> = good();
99 const f = async (input: string): Promise<Response> => {
100 const p = new URL(input).pathname;
101 if (p.startsWith("/__webspec-")) return res.clone() as Response;
102 return g[p] ? (g[p]!.clone() as Response) : new Response("", { status: 404 });
103 };
104 return (await webspec("https://x.test", [], f, { site: true })).map((x) => x.check);
105 };
106 expect(await withRandom(new Response("Not found", { status: 404 }))).toEqual(["error-404"]);
107 expect(await withRandom(new Response(PAGE, { status: 200, headers: html }))).toEqual(["error-404"]);
108 expect(await withRandom(new Response(PAGE.replace(/<title>.*<\/title>/, ""), { status: 404, headers: html }))).toEqual(["error-404"]);
109 });
110
111 test("a 500 is checked only when a path is named, and must be the custom page", async () => {
112 const over = (r: Response) => ({ "/boom": r });
113 expect(await run(over(new Response("x", { status: 200 })))).toEqual([]);
114 expect(await run(over(new Response(PAGE, { status: 500, headers: html })), "/boom")).toEqual([]);
115 expect(await run(over(new Response(PAGE, { status: 200, headers: html })), "/boom")).toEqual(["error-500"]);
116 expect(await run(over(new Response("Internal", { status: 500 })), "/boom")).toEqual(["error-500"]);
117 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs