Public: anyone can read this; the crew can change it.

The looks run reports axe-core violations for each look in light and dark, and proves / renders whole without JavaScript, report-only #2974

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2970-the-looks-run-reports-axe-core-violation → main 4 commits fork 3 threads · 3 unresolved

  • Merged. scout/2970-the-looks-run-reports-axe-core-violation was merged into main as b652797.

Review

Sign in to review this change.
8 files changed +268 −5
README.md +7
@@ -332,6 +332,13 @@ report never causes an error on its own. It only reports and always exits 0.
332332 `(edge)` marks a finding the edge in front of the app may own (HSTS), so fix it
333333 there rather than in the app. Each fetch gives up after 10 seconds.
334334
335 `--standards` also runs axe-core (`src/release/axe.ts`) over each path in light
336 and dark and loads each with JavaScript off, reporting an empty `<main>` or a
337 missing `<h1>`; it needs `chromium` on `PATH` or `FORGE_CHROMIUM` (without one
338 it falls back to a regex read of the served HTML). axe-core is a devDependency
339 of this check only, never loaded by the server: its rules need a rendered page
340 and an accessibility tree, which nothing dependency-free gives.
341
335342 What you asked for: `Ctrl+K`, then **Asked**, is `/asked` in the terminal — your
336343 thoughts, then the issues you can write to that are not shipped, oldest first
337344 (`j`/`k` move, `enter` gives the issue's web address, `r` refreshes). It is read
bun.lock +3
@@ -7,6 +7,7 @@
77 "devDependencies": {
88 "@anthropic-ai/sandbox-runtime": "^0.0.79",
99 "@types/bun": "^1.2.0",
10 "axe-core": "^4.14.0",
1011 "typescript": "^7.0.2",
1112 "typescript-api": "npm:typescript@^5.9.0",
1213 },
@@ -61,6 +62,8 @@
6162
6263 "@typescript/typescript-win32-x64": ["@typescript/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="],
6364
65 "axe-core": ["[email protected]", "", {}, "sha512-9WTZxEjsZ7b13TH8JPmbV2z8CHbl80/2hm3XPEG4JgNdQLK81IBRXmSxHfMAOkSqQeRxT/0dwNDz2GOm3zzpcQ=="],
66
6467 "bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
6568
6669 "commander": ["[email protected]", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
package.json +1
@@ -14,6 +14,7 @@
1414 "devDependencies": {
1515 "@anthropic-ai/sandbox-runtime": "^0.0.79",
1616 "@types/bun": "^1.2.0",
17 "axe-core": "^4.14.0",
1718 "typescript": "^7.0.2",
1819 "typescript-api": "npm:typescript@^5.9.0"
1920 }
scripts/smoke.ts +11 −4
@@ -11,9 +11,10 @@
1111 * never answers with `Response.error()`.
1212 *
1313 * `--standards` runs the website-specification report (`release/webspec.ts`)
14 * over forge.json's `app.looks` paths, prints one line per finding with its
15 * section (head, headers, well-known files, error pages; `--error-path`
16 * names a path that must answer 500), and exits 0: it reports, it does not gate.
14 * over forge.json's `app.looks` paths (head, headers, well-known files, error pages,
15 * a no-JavaScript render and axe-core in light and dark, `release/axe.ts`), prints one
16 * line per finding with its section (`--error-path` names a path that must answer
17 * 500), and exits 0: it reports, it does not gate.
1718 *
1819 * `--outsider` also signs up a fresh account and proves each thing a
1920 * stranger must not do is refused over HTTP and MCP (`release/outsider.ts`,
@@ -22,6 +23,8 @@
2223 */
2324
2425 import { smoke } from "../src/release/smoke.ts";
26 import { axe, noScript } from "../src/release/axe.ts";
27 import { findChromium } from "../src/looks/capture.ts";
2528 import { webspec } from "../src/release/webspec.ts";
2629 import { outsider } from "../src/release/outsider.ts";
2730 import { PROBE_HEADER } from "../src/kernel/probe.ts";
@@ -54,7 +57,11 @@ if (import.meta.main) {
5457 if (standards) {
5558 const looks = (await Bun.file(new URL("../forge.json", import.meta.url)).json())?.app?.looks;
5659 const paths: string[] = Array.isArray(looks) ? looks.filter((p): p is string => typeof p === "string") : ["/"];
57 const findings = await webspec(base, paths, fetch, { site: true, ...(errorPath ? { errorPath } : {}) });
60 const findings = [
61 ...(await webspec(base, paths, fetch, { site: true, ...(errorPath ? { errorPath } : {}) })),
62 ...(await axe(base, paths)),
63 ...(findChromium() === undefined ? await noScript(base, paths) : []),
64 ];
5865 for (const f of findings) console.log(`${f.path} ${f.check}${f.edge ? " (edge)" : ""}: ${f.message} <${f.section}>`);
5966 console.log(`standards: ${findings.length} finding(s) over ${paths.length} path(s); report only`);
6067 process.exit(0);
src/release/axe.ts added +201
@@ -0,0 +1,201 @@
1 /**
2 * The website specification's accessibility checks (#2915 check 8), report
3 * only. `axe()` opens each path in headless Chromium once per colour scheme,
4 * runs axe-core in the page and returns its violations as `Finding`s;

The doc comment reads oddly: "also loads each path…" has no subject. Reword so it says axe() does both, and that the browser render is authoritative and noScript() is the regex fallback.

5 * also loads each path with script execution disabled in the same session and
6 * reports an empty <main> or a missing main heading in the rendered DOM.
7 * `noScript()` is the fallback where there is no browser: it reads the served
8 * HTML with regexes, which is only a proxy for the render (it cannot see what
9 * CSS hides). Neither throws or fails a run.
10 *
11 * axe-core is a devDependency of the check runner and is read here as a file
12 * to inject into the page; nothing the server loads imports this module.
13 * There is no zero-dependency equivalent: axe's rules (contrast, names,
14 * roles) need a rendered page and a computed accessibility tree.
15 */
16
17 import { mkdtempSync, rmSync } from "node:fs";
18 import { tmpdir } from "node:os";
19 import { join } from "node:path";
20 import { findChromium } from "../looks/capture.ts";
21 import { s, type Finding } from "./webspec.ts";
22
23 const SCHEMES = ["light", "dark"] as const;
24 const STEP_MS = 15000;
25
26 export interface AxeViolation {
27 id: string;
28 impact?: string | null;
29 help: string;
30 nodes: { target: unknown[] }[];
31 }
32
33 /** One axe violation as a finding; the scheme it appeared in is in the message. */
34 export function axeFinding(path: string, scheme: string, v: AxeViolation): Finding {
35 const first = String(v.nodes[0]?.target?.[0] ?? "");
36 return {
37 path,
38 check: `axe:${v.id}`,
39 message: `${v.help} (${v.impact ?? "unrated"}, ${scheme} scheme): ${v.nodes.length} element(s)${first ? `, first ${first.slice(0, 120)}` : ""}`,
40 section: s("accessibility"),
41 };
42 }
43
44 /** What a visitor without JavaScript is served: <main> must hold content and a main heading. */
45 export function noScriptProblems(html: string): string[] {
46 const body = html.replace(/<!--[\s\S]*?-->/g, "").replace(/<(script|style|template)\b[\s\S]*?<\/\1>/gi, "");
47 const main = /<main\b[^>]*>([\s\S]*?)<\/main>/i.exec(body);
48 const out: string[] = [];
49 if (!main) out.push("the page has no <main>");
50 else if (main[1]!.replace(/<[^>]*>/g, "").trim() === "" && !/<(img|svg|input|video|canvas|iframe)\b/i.test(main[1]!))
51 out.push("<main> is empty without JavaScript");
52 if (!/<h1\b[^>]*>[\s\S]*?\S[\s\S]*?<\/h1>/i.test(main ? main[1]! : body)) out.push("no main heading (<h1>) without JavaScript");
53 return out;
54 }
55
56 /** Fetches each path as a script-less client would see it and reports empty pages. */
57 export async function noScript(base: string, paths: string[], fetcher: typeof fetch = fetch): Promise<Finding[]> {
58 const findings: Finding[] = [];
59 for (const path of paths) {
outdated

This regex proxy only reads the served HTML. Disabling script execution in the DevTools session would prove the page renders whole without JavaScript. Failing that, document the limit here.

60 try {
61 const res = await fetcher(new URL(path, base).href, { headers: { accept: "text/html" }, redirect: "manual", signal: AbortSignal.timeout(10000) });
62 if (!res.ok) continue; // webspec() already reports the status
63 for (const message of noScriptProblems(await res.text()))
64 findings.push({ path, check: "no-js", message, section: s("progressive-enhancement") });
65 } catch {
66 // unreachable pages are webspec()'s finding
67 }
68 }
69 return findings;
70 }
71
72 /** A DevTools connection to one page of a private headless Chromium. */
73 async function withPage<T>(binary: string, use: (send: (method: string, params?: object) => Promise<any>, once: (event: string) => Promise<void>) => Promise<T>): Promise<T> {
74 const profile = mkdtempSync(join(tmpdir(), "forge-axe-"));
75 const proc = Bun.spawn(
76 [binary, "--headless=new", "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-background-networking", "--no-first-run", `--user-data-dir=${profile}`, "--remote-debugging-port=0", "about:blank"],
77 { stdin: "ignore", stdout: "ignore", stderr: "pipe" },
78 );
79 try {
80 const reader = proc.stderr.getReader();
81 const decoder = new TextDecoder();
82 let said = "";
83 let endpoint: string | undefined;
84 const deadline = Date.now() + STEP_MS;
85 while (!endpoint && Date.now() < deadline) {
86 const { done, value } = await Promise.race([reader.read(), Bun.sleep(STEP_MS).then(() => ({ done: true, value: undefined }))]);
87 if (done) break;
88 said += decoder.decode(value, { stream: true });
89 endpoint = /DevTools listening on (ws:\/\/\S+)/.exec(said)?.[1];
90 }
91 if (!endpoint) throw new Error("chromium did not offer a DevTools endpoint");
92 // The rest of what it says is read and dropped, so the pipe never fills.
93 (async () => {
94 while (!(await reader.read()).done);
95 })().catch(() => {});
96 const ws = new WebSocket(endpoint);
97 await new Promise<void>((resolve, reject) => {
98 ws.onopen = () => resolve();
99 ws.onerror = () => reject(new Error("could not reach DevTools"));
100 });
101 let id = 0;
102 let session: string | undefined;
103 const pending = new Map<number, { ok: (v: any) => void; no: (e: Error) => void }>();
104 const waiting = new Map<string, (() => void)[]>();
105 ws.onmessage = (m) => {
106 const msg = JSON.parse(String(m.data));
107 if (msg.id !== undefined) {
108 const p = pending.get(msg.id);
109 pending.delete(msg.id);
110 if (msg.error) p?.no(new Error(msg.error.message));
111 else p?.ok(msg.result);
112 } else if (msg.method) for (const f of [...(waiting.get(msg.method) ?? [])]) f();
113 };
114 const send = (method: string, params: object = {}) =>
115 new Promise<any>((ok, no) => {
116 const n = ++id;
117 const timer = setTimeout(() => {
118 if (pending.delete(n)) no(new Error(`${method} timed out`));
119 }, STEP_MS);
120 pending.set(n, {
121 ok: (v) => (clearTimeout(timer), ok(v)),
122 no: (e) => (clearTimeout(timer), no(e)),
123 });
124 ws.send(JSON.stringify({ id: n, method, params, ...(session ? { sessionId: session } : {}) }));
125 });
126 const once = (event: string) =>
127 new Promise<void>((ok, no) => {
128 const done = () => {
129 clearTimeout(timer);
130 waiting.set(event, (waiting.get(event) ?? []).filter((f) => f !== done));
131 ok();
132 };
133 const timer = setTimeout(() => {
134 waiting.set(event, (waiting.get(event) ?? []).filter((f) => f !== done));
135 no(new Error(`${event} timed out`));
136 }, STEP_MS);
137 waiting.set(event, [...(waiting.get(event) ?? []), done]);
138 });
139 const { targetId } = await send("Target.createTarget", { url: "about:blank" });
140 session = (await send("Target.attachToTarget", { targetId, flatten: true })).sessionId;
141 try {
142 return await use(send, once);
143 } finally {
144 ws.close();
145 }
146 } finally {
147 proc.kill("SIGKILL");
148 await proc.exited.catch(() => {});
149 rmSync(profile, { recursive: true, force: true });
150 }
151 }
152
153 /** Runs axe-core over each path in each colour scheme, once each, and reports its violations. */
154 export async function axe(base: string, paths: string[]): Promise<Finding[]> {
155 const binary = findChromium();
156 if (binary === undefined) return [{ path: "/", check: "axe", message: "no chromium here: axe-core did not run", section: s("accessibility") }];
157 const source = await Bun.file(require.resolve("axe-core/axe.min.js")).text();
158 const findings: Finding[] = [];
159 try {
160 await withPage(binary, async (send, once) => {
161 await send("Page.enable");
162 const load = async (path: string) => {
163 const loaded = once("Page.loadEventFired");
164 loaded.catch(() => {});
165 await send("Page.navigate", { url: new URL(path, base).href });
166 await loaded;
167 };
168 for (const path of paths) {
169 try {
170 await send("Emulation.setScriptExecutionDisabled", { value: true });
171 await load(path);
172 const dom = await send("Runtime.evaluate", { expression: "document.documentElement.outerHTML", returnByValue: true });
173 for (const message of noScriptProblems(`<!doctype html>${dom.result.value}`))
174 findings.push({ path, check: "no-js", message, section: s("progressive-enhancement") });
175 } catch (e) {
176 findings.push({ path, check: "axe", message: `the no-JavaScript render did not run: ${e instanceof Error ? e.message : String(e)}`, section: s("progressive-enhancement") });
177 } finally {
178 await send("Emulation.setScriptExecutionDisabled", { value: false }).catch(() => {});
179 }
180 for (const scheme of SCHEMES) {
181 try {
182 await send("Emulation.setEmulatedMedia", { features: [{ name: "prefers-color-scheme", value: scheme }] });
183 await load(path);
184 const res = await send("Runtime.evaluate", {
185 expression: `${source};axe.run(document,{resultTypes:["violations"]}).then(r=>JSON.stringify(r.violations))`,
186 awaitPromise: true,
187 returnByValue: true,
188 });
189 if (res.exceptionDetails) throw new Error(res.exceptionDetails.exception?.description ?? "axe failed");
190 for (const v of JSON.parse(res.result.value) as AxeViolation[]) findings.push(axeFinding(path, scheme, v));
191 } catch (e) {
192 findings.push({ path, check: "axe", message: `axe-core did not run (${scheme}): ${e instanceof Error ? e.message : String(e)}`, section: s("accessibility") });
193 }
194 }
195 }
196 });
197 } catch (e) {
198 findings.push({ path: "/", check: "axe", message: `axe-core did not run: ${e instanceof Error ? e.message : String(e)}`, section: s("accessibility") });
199 }
200 return findings;
201 }
src/release/webspec.ts +1 −1
@@ -23,7 +23,7 @@ type Problem = { check: string; message: string; section: string; edge?: true };
2323
2424 const TIMEOUT_MS = 10000;
2525 const SPEC = "https://specification.website/";
26 const s = (name: string) => `${SPEC}#${name}`;
26 export const s = (name: string) => `${SPEC}#${name}`;
2727
2828 const tags = (html: string, name: string): string[] => html.match(new RegExp(`<${name}\\b[^>]*>`, "gi")) ?? [];
2929
test/axe.test.ts added +43
@@ -0,0 +1,43 @@
1 import { afterAll, expect, test } from "bun:test";
2 import { axe, noScript, noScriptProblems } from "../src/release/axe.ts";
3 import { findChromium } from "../src/looks/capture.ts";
4
5 const PAGES: Record<string, string> = {
6 "/": `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>t</title></head><body><main><h1>Hi</h1><input type="text"></main></body></html>`,
7 "/script": `<!doctype html><html lang="en"><head><title>t</title></head><body><main></main><script>document.querySelector("main").innerHTML="<h1>Hi</h1>"</script></body></html>`,
8 "/good": `<!doctype html><html lang="en"><head><title>t</title></head><body><main><h1>Hi</h1><p>text</p></main></body></html>`,
9 };
10 const server = Bun.serve({
11 port: 0,
12 hostname: "127.0.0.1",
13 fetch: (req) => {
14 const html = PAGES[new URL(req.url).pathname];
15 return html ? new Response(html, { headers: { "content-type": "text/html" } }) : new Response("no", { status: 404 });
16 },
17 });
18 afterAll(() => server.stop(true));
19 const base = `http://127.0.0.1:${server.port}`;
20
21 test("a page that fills <main> from a script is a no-js finding; a rendered one is not", async () => {
22 const found = await noScript(base, ["/script", "/good"]);
23 expect(found.map((f) => [f.path, f.check])).toEqual([["/script", "no-js"], ["/script", "no-js"]]);
24 expect(found[0]!.section).toContain("specification.website");
25 });
26
27 test("noScriptProblems names a missing main and heading", () => {
28 expect(noScriptProblems("<body>x</body>")).toEqual(["the page has no <main>", "no main heading (<h1>) without JavaScript"]);
29 });
30
31 const noChromium = findChromium() === undefined;
32
33 test.skipIf(noChromium)("in a real browser: an unlabelled input yields one axe:label finding in each scheme, and a script-filled <main> a no-js finding", async () => {
outdated

This early return makes the test vacuous. Fail on any axe check finding (expect(all.filter(f=>f.check==="axe")).toEqual([])) so a broken browser connection turns the check red.

34 const all = await axe(base, ["/", "/script"]);
35 // A browser that fails to start, a step that times out or axe throwing is a red test, not a skip.
36 expect(all.filter((f) => f.check === "axe")).toEqual([]);
37 const found = all.filter((f) => f.check === "axe:label" && f.path === "/");
38 expect(found.map((f) => /light|dark/.exec(f.message)![0]).sort()).toEqual(["dark", "light"]);
39 expect(found[0]!.section).toContain("#accessibility");
40 expect(all.filter((f) => f.check === "no-js").map((f) => f.path)).toEqual(["/script", "/script"]);
41 }, 90000);
42
43 test.skipIf(!noChromium)("no chromium on this machine: the browser test (axe:label per scheme, no-js render) did not run", () => {});
test/spawn-pipes.test.ts +1
@@ -28,6 +28,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
2828 "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout and stderr into the channel" },
2929 "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
3030 "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
31 "src/release/axe.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" },
3132 "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" },
3233 "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" },
3334 "src/cli/config.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs