Public: anyone can read this; the crew can change it.
Web standards complete and tested: robots, sitemap, manifest, security.txt, llms.txt, favicons, theme colors, structured data #9618
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9579-web-standards-complete-and-tested-robots → main 5 commits fork 1 thread · 1 unresolved
6 files
changed +96
−2
docs/web.md +1 −1
| @@ -8,7 +8,7 @@ Anything that is the same bytes for everyone and changes only when the code does | ||
| 8 | 8 | |
| 9 | 9 | ## What is HTML |
| 10 | 10 | |
| 11 | Markup only, and small. It names its assets; it never carries them. Budget: **home under 40 KB raw**, no inline `<script>` (a `type="application/json"` data block is data), no inline `<symbol>`. The two inline `<style>` blocks (the boot theme and a page's own) are admitted by hash in the policy, not by a nonce, so a page is the same bytes for every visitor and can be cached. | |
| 11 | Markup only, and small. It names its assets; it never carries them. Budget: **home under 40 KB raw**, no inline `<script>` (a `type="application/json"` or `application/ld+json` data block is data), no inline `<symbol>`. The two inline `<style>` blocks (the boot theme and a page's own) are admitted by hash in the policy, not by a nonce, so a page is the same bytes for every visitor and can be cached. | |
| 12 | 12 | |
| 13 | 13 | ## How each is cached |
| 14 | 14 |
src/web/layout.ts +3
| @@ -108,6 +108,8 @@ export interface PageOptions { | ||
| 108 | 108 | feed?: { href: string; title: string }; |
| 109 | 109 | /** This page's JSON twin, advertised in the head for agents. */ |
| 110 | 110 | json?: { href: string; title: string }; |
| 111 | /** Structured data (JSON-LD) for the head, already escaped by `safeJson`. */ | |
| 112 | jsonLd?: string; | |
| 111 | 113 | /** Breadcrumb segments rendered in the topbar, e.g. owner / repo. */ |
| 112 | 114 | crumbs?: { label: string; href: string }[]; |
| 113 | 115 | body: Html; |
| @@ -813,6 +815,7 @@ export function page(options: PageOptions, status = 200): Response { | ||
| 813 | 815 | options.json, |
| 814 | 816 | (j) => html`<link rel="alternate" type="application/json" title="${j.title}" href="${j.href}" />`, |
| 815 | 817 | )} |
| 818 | ${when(options.jsonLd, (d) => html`<script type="application/ld+json">${raw(d)}</script>`)} | |
| 816 | 819 | <meta name="color-scheme" content="light dark" /> |
| 817 | 820 | ${theme |
| 818 | 821 | ? html`<meta name="theme-color" content="${metaThemeColor(theme)}" />` |
src/web/structured_data.ts added +43
| @@ -0,0 +1,43 @@ | ||
| 1 | /** | |
| 2 | * The front page's structured data (schema.org JSON-LD): the site as a | |
| 3 | * WebSite and the forge as a SoftwareApplication. Plain data in, a string | |
| 4 | * out; the page puts it in a `<script type="application/ld+json">`. | |
| 5 | */ | |
| 6 | import { SITE_LINE } from "./site_line.ts"; | |
| 7 | ||
| 8 | /** JSON that cannot close its own script tag: `<`, `>` and `&` are written as \u escapes. */ | |
| 9 | export function safeJson(value: unknown): string { | |
| 10 | return JSON.stringify(value) | |
| 11 | .replace(/</g, "\\u003c") | |
| 12 | .replace(/>/g, "\\u003e") | |
| 13 | .replace(/&/g, "\\u0026") | |
| 14 | .replace(/\u2028/g, "\\u2028") | |
| 15 | .replace(/\u2029/g, "\\u2029"); | |
| 16 | } | |
| 17 | ||
| 18 | export function siteStructuredData(origin: string, version: string): string { | |
| 19 | return safeJson({ | |
| 20 | "@context": "https://schema.org", | |
| 21 | "@graph": [ | |
| 22 | { | |
| 23 | "@type": "WebSite", | |
| 24 | "@id": `${origin}/#site`, | |
| 25 | name: "forge", | |
| 26 | url: `${origin}/`, | |
| 27 | description: SITE_LINE, | |
| 28 | }, | |
| 29 | { | |
| 30 | "@type": "SoftwareApplication", | |
| 31 | "@id": `${origin}/#software`, | |
| 32 | name: "forge", | |
| 33 | url: `${origin}/`, | |
| 34 | description: SITE_LINE, | |
| 35 | applicationCategory: "DeveloperApplication", | |
| 36 | operatingSystem: "Web", | |
| 37 | softwareVersion: version, | |
| 38 | isAccessibleForFree: true, | |
| 39 | offers: { "@type": "Offer", price: "0", priceCurrency: "USD" }, | |
| 40 | }, | |
| 41 | ], | |
| 42 | }); | |
| 43 | } |
src/web/studio/home.ts +3
| @@ -34,6 +34,8 @@ import { field, readForm, type Form } from "../form.ts"; | ||
| 34 | 34 | import { liveRegionsJs } from "../client.ts"; |
| 35 | 35 | import { html, when, type Html } from "../html.ts"; |
| 36 | 36 | import { fragment, page, redirect } from "../layout.ts"; |
| 37 | import { siteStructuredData } from "../structured_data.ts"; | |
| 38 | import { running } from "../../kernel/version.ts"; | |
| 37 | 39 | import { liveClientJs } from "../live.ts"; |
| 38 | 40 | import { |
| 39 | 41 | btn, |
| @@ -872,6 +874,7 @@ bun run src/main.ts</code></pre> | ||
| 872 | 874 | title: "forge: software for what you're building, yours to keep", |
| 873 | 875 | description: "Your own forge, with a private AI crew that builds your apps, games and business tools. Open source, and yours to keep.", |
| 874 | 876 | body, |
| 877 | jsonLd: siteStructuredData(origin, running().version), | |
| 875 | 878 | ...(visitor |
| 876 | 879 | ? { |
| 877 | 880 | tools: html`<a class="small door-wide" href="#made">Made here</a> |
test/standards.test.ts +45
| @@ -514,4 +514,49 @@ describe("what the open web expects", () => { | ||
| 514 | 514 | expect(html).toContain(`<meta property="og:description" content="${said}"`); |
| 515 | 515 | expect(html).toContain(SITE_LINE); |
| 516 | 516 | }); |
| 517 | ||
| 518 | test("each web standard is served with its type and shape; the home page carries JSON-LD", async () => { | |
| 519 | const get = (path: string) => fetch(`${base}${path}`); | |
| 520 | const robots = await get("/robots.txt"); | |
| 521 | expect(robots.headers.get("content-type")).toContain("text/plain"); | |
| 522 | expect(await robots.text()).toContain(`Sitemap: ${base}/sitemap.xml`); | |
| 523 | const sitemap = await get("/sitemap.xml"); | |
| 524 | expect(sitemap.headers.get("content-type")).toContain("xml"); | |
| 525 | expect(await sitemap.text()).toContain("<urlset"); | |
| 526 | const manifest = await get("/manifest.webmanifest"); | |
| 527 | expect(manifest.headers.get("content-type")).toMatch(/manifest\+json|json/); | |
| 528 | expect(((await manifest.json()) as { icons: unknown[] }).icons.length).toBeGreaterThan(0); | |
| 529 | const sec = await get("/.well-known/security.txt"); | |
| 530 | expect(sec.headers.get("content-type")).toContain("text/plain"); | |
| 531 | const secBody = await sec.text(); | |
| 532 | expect(secBody).toMatch(/^Contact: \S+/m); | |
| 533 | const expires = /^Expires: (\S+)/m.exec(secBody)?.[1] ?? ""; | |
| 534 | expect(Date.parse(expires)).toBeGreaterThan(Date.now()); | |
| 535 | const llms = await get("/llms.txt"); | |
| 536 | expect(llms.headers.get("content-type")).toContain("text/plain"); | |
| 537 | expect((await llms.text()).startsWith("# ")).toBe(true); | |
| 538 | expect((await get("/favicon.ico")).headers.get("content-type")).toContain("image/png"); | |
| 539 | expect((await get("/icons/forge-192.png")).headers.get("content-type")).toBe("image/png"); | |
| 540 | const home = await (await get("/")).text(); | |
| 541 | expect(home).toMatch(/name="theme-color"\s+media="\(prefers-color-scheme: light\)"/); | |
| 542 | expect(home).toMatch(/name="theme-color"\s+media="\(prefers-color-scheme: dark\)"/); | |
| 543 | const ld = /<script type="application\/ld\+json">([^<]*)<\/script>/.exec(home)?.[1]; | |
| 544 | expect(ld).toBeDefined(); | |
| 545 | const graph = (JSON.parse(ld ?? "{}") as { "@graph": { "@type": string; url: string }[] })["@graph"]; | |
| 546 | expect(graph.map((n) => n["@type"])).toEqual(["WebSite", "SoftwareApplication"]); | |
| 547 | expect(graph[0]?.url).toBe(`${base}/`); | |
| 548 | }); | |
| 549 | ||
| 550 | test("the JSON-LD origin is the configured canonical URL, not the request's Host", async () => { | |
| 551 | app.config.url = "https://canon.example"; | |
| 552 | try { | |
| 553 | const home = await fetch(`${base}/`, { headers: { host: "evil.example" } }).then((r) => r.text()); | |
| 554 | const ld = /<script type="application\/ld\+json">([^<]*)<\/script>/.exec(home)?.[1] ?? ""; | |
| 555 | expect(ld).toContain("https://canon.example/"); | |
| 556 | expect(ld).not.toContain("evil.example"); | |
| 557 | expect(ld).not.toContain(base); | |
| 558 | } finally { | |
| 559 | delete app.config.url; | |
| 560 | } | |
| 561 | }); | |
| 517 | 562 | }); |
test/web-budget.test.ts +1 −1
| @@ -54,7 +54,7 @@ describe("web budgets (docs/web.md)", () => { | ||
| 54 | 54 | const res = await fetch(base + path); |
| 55 | 55 | const text = await res.text(); |
| 56 | 56 | expect(new TextEncoder().encode(text).length).toBeLessThan(BUDGET.html); |
| 57 | expect(/<script(?![^>]*\s(src=|type="application\/json"))/.test(text)).toBe(BUDGET.inlineScripts > 0); | |
| 57 | expect(/<script(?![^>]*\s(src=|type="application\/(ld\+)?json"))/.test(text)).toBe(BUDGET.inlineScripts > 0); | |
| 58 | 58 | expect(text.includes("<symbol")).toBe(BUDGET.inlineSvg > 0); |
| 59 | 59 | expect(text).toContain("/assets/i-"); |
| 60 | 60 | }); |
Use the canonical origin const from line 743 (config.url ?? ctx.url.origin), not ctx.url.origin. Add a test for it.