Public: anyone can read this; the crew can change it.
/sitemap.xml also lists the guides, /templates, /security and /status, each with its last change #13905
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13884-sitemap-xml-also-lists-the-guides-templa → main 5 commits fork 4 threads · 4 unresolved
src/web/sitemap_pages.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * The guide pages of the sitemap (#13884): every document `/docs` links, from | |
| 3 | * the same index the page reads (`view/docs.ts`), so a new guide appears with | |
| 4 | * no second edit, plus the three public pages that have no file. Each file's | |
| 5 | * `lastmod` is the day of its last commit, read once with one fixed-argument | |
| 6 | * `git log` over `docs/` in the release's checkout. Where there is no | |
| 7 | * history (a release shipped without `.git`) or the page has no file, the | |
| 8 | * caller puts the newest release's day. | |
| 9 | */ | |
| 10 | ||
| 11 | import { join } from "node:path"; | |
| 12 | import { runGit } from "../git/env.ts"; | |
| 13 | import { docEntries } from "../view/docs.ts"; | |
| 14 | ||
| 15 | export interface SitemapPage { | |
| 16 | readonly path: string; | |
| 17 | /** `YYYY-MM-DD` of the file's last commit; null when there is no file or no history. */ | |
| 18 | readonly lastmod: string | null; | |
| 19 | } | |
| 20 | ||
| 21 | /** Public pages that are code, not a file. */ | |
| 22 | const FILELESS = ["/templates", "/security", "/status"] as const; | |
| 23 | const ROOT = join(import.meta.dir, "..", ".."); | |
| 24 | const DAY = /^\d{4}-\d{2}-\d{2}$/; | |
| 25 | ||
| 26 | /** `git log --name-only` output as file → newest day (the log is newest first). */ | |
| 27 | export function parseLastChanges(out: string): Map<string, string> { | |
| 28 | const days = new Map<string, string>(); | |
| 29 | let day: string | undefined; | |
| 30 | for (const line of out.split("\n")) { | |
| 31 | if (line.startsWith("@")) day = DAY.test(line.slice(1)) ? line.slice(1) : undefined; | |
| 32 | else if (line !== "" && day !== undefined && !days.has(line)) days.set(line, day); | |
| 33 | } | |
| 34 | return days; | |
| 35 | } | |
| 36 | ||
| 37 | async function lastChanges(): Promise<Map<string, string> | undefined> { | |
| 38 | const read = await runGit(["log", "--format=@%cs", "--name-only", "--", "docs"], { cwd: ROOT }); | |
| 39 | return read.ok ? parseLastChanges(read.value) : undefined; | |
| 40 | } | |
| 41 | ||
| 42 | /** The day this process started (UTC): the last resort when no release carries a time. */ | |
| 43 | const STARTED = new Date().toISOString().slice(0, 10); | |
| 44 | ||
| 45 | /** | |
| 46 | * The day a page's `lastmod` shows: its file's last commit, else the newest release's day, else the | |
| 47 | * day this process started, so a page always has one. | |
| 48 | */ | |
| 49 | export function pageDay(page: SitemapPage, released: string | null, started: string = STARTED): string { | |
| 50 | return page.lastmod ?? released ?? started; | |
| 51 | } | |
| 52 | ||
| 53 | /** A failed read is kept this long, so a checkout without git is not asked on every request. */ | |
| 54 | const RETRY_MS = 60_000; | |
| 55 | let kept: { pages: readonly SitemapPage[]; failedAt: number | null } | undefined; | |
| 56 | let reading: Promise<readonly SitemapPage[]> | undefined; | |
| 57 | ||
| 58 | function build(days: Map<string, string> | undefined): readonly SitemapPage[] { | |
| 59 | return [ | |
| 60 | ...docEntries().map((e) => ({ path: `/docs/${e.name}`, lastmod: days?.get(`docs/${e.name}`) ?? null })), | |
| 61 | ...FILELESS.map((path) => ({ path, lastmod: null })), | |
| 62 | ]; | |
| 63 | } | |
| 64 | ||
| 65 | /** | |
| 66 | * The guide pages, in the order /docs lists them, then the fileless ones. A good read is kept for | |
| 67 | * the life of the process; a failed one only for a minute. | |
| 68 | */ | |
| 69 | export function sitemapPages(now: number = Date.now()): Promise<readonly SitemapPage[]> { | |
| 70 | if (kept !== undefined && (kept.failedAt === null || now - kept.failedAt < RETRY_MS)) return Promise.resolve(kept.pages); | |
| 71 | reading ??= lastChanges() | |
| 72 | .then((days) => { | |
| 73 | kept = { pages: build(days), failedAt: days === undefined ? now : null }; | |
| 74 | return kept.pages; | |
| 75 | }) | |
| 76 | .finally(() => { | |
| 77 | reading = undefined; | |
| 78 | }); | |
| 79 | return reading; | |
| 80 | } |
src/web/standards.ts +13 −4
| @@ -33,6 +33,7 @@ import { CRAWL_DELAY_S, DEEP_SEGMENTS } from "../http/crawl.ts"; | ||
| 33 | 33 | import { markPng, type MarkShape } from "../design/mark.ts"; |
| 34 | 34 | import { metaThemeColor } from "../design/themes.ts"; |
| 35 | 35 | import { BODY_MAX, TITLE_MAX, reportsClosed } from "../core/reports.ts"; |
| 36 | import { pageDay, sitemapPages } from "./sitemap_pages.ts"; | |
| 36 | 37 | import { dayOf } from "../release/days.ts"; |
| 37 | 38 | import { readLandings } from "../release/landings.ts"; |
| 38 | 39 | import { takesOutside } from "../actions/contribute_token.ts"; |
| @@ -296,9 +297,9 @@ function xml(s: string): string { | ||
| 296 | 297 | } |
| 297 | 298 | |
| 298 | 299 | /** The sitemap and its ETag per origin, valid while the listing read is the same one. */ |
| 299 | const sitemapBodies = new Map<string, { rows: unknown; changed: string | null; body: string; etag: string }>(); | |
| 300 | const sitemapBodies = new Map<string, { rows: unknown; pages: unknown; released: string | null; changed: string | null; body: string; etag: string }>(); | |
| 300 | 301 | |
| 301 | /** GET /sitemap.xml — the front page, the changelog and every public repository. */ | |
| 302 | /** GET /sitemap.xml — the front page, the changelog, the guides and every public repository. */ | |
| 302 | 303 | export async function sitemapXml(ctx: Ctx): Promise<Response> { |
| 303 | 304 | // The rows are kept without an origin, and the origin is put on per answer, |
| 304 | 305 | // so a caller who varies `Host` cannot make the forge read the rows again. |
| @@ -309,8 +310,13 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> { | ||
| 309 | 310 | newestLanding(ctx.app.db, async () => (await readLandings(ctx.app))?.live[0]?.at), |
| 310 | 311 | ); |
| 311 | 312 | const changed = newest === undefined ? null : dayOf(newest, "UTC"); |
| 313 | const pages = await sitemapPages(); | |
| 314 | // The guides' lastmod where git has none: the newest release's day (frozen releases carry no | |
| 315 | // time), then the start day. | |
outdated
On the preview this is null, so every guide and the three fileless pages are emitted with no lastmod. The fallback must give a day (the newest release, or the process start or build day) or box 1 stays unmet. Add a test with an empty |
||
| 316 | const at = shippedReleases().find((r) => r.at !== undefined)?.at; | |
| 317 | const released = at === undefined ? null : dayOf(new Date(at).toISOString(), "UTC"); | |
| 312 | 318 | let kept = sitemapBodies.get(origin); |
| 313 | if (kept === undefined || kept.rows !== repos || kept.changed !== changed) { | |
| 319 | if (kept === undefined || kept.rows !== repos || kept.pages !== pages || kept.released !== released || kept.changed !== changed) { | |
| 314 | 320 | // Escaping is per character, so the escaped origin and the escaped path |
| 315 | 321 | // join into the escaped URL. |
| 316 | 322 | const body = timePhase(ctx.request, "render", () => { |
| @@ -324,6 +330,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> { | ||
| 324 | 330 | `<url><loc>${o}/privacy</loc></url>`, |
| 325 | 331 | `<url><loc>${o}/terms</loc></url>`, |
| 326 | 332 | `<url><loc>${o}/docs</loc></url>`, |
| 333 | ...pages.map((p) => { | |
| 334 | return `<url><loc>${o}${xml(p.path)}</loc><lastmod>${pageDay(p, released)}</lastmod></url>`; | |
| 335 | }), | |
| 327 | 336 | ...repos.map((r) => { |
| 328 | 337 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; |
| 329 | 338 | return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`; |
| @@ -332,7 +341,7 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> { | ||
| 332 | 341 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; |
| 333 | 342 | }); |
| 334 | 343 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); |
| 335 | kept = { rows: repos, changed, body, etag }; | |
| 344 | kept = { rows: repos, pages, released, changed, body, etag }; | |
| 336 | 345 | // Origins come from the Host header: keep a handful, not one per guess. |
| 337 | 346 | if (sitemapBodies.size >= 16) sitemapBodies.clear(); |
| 338 | 347 | sitemapBodies.set(origin, kept); |
test/sitemap-guides.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** /sitemap.xml lists every guide /docs links, plus /templates, /security and /status (#13884). */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { pageDay, parseLastChanges, sitemapPages } from "../src/web/sitemap_pages.ts"; | |
| 9 | import { docEntries } from "../src/view/docs.ts"; | |
| 10 | ||
| 11 | let dataDir: string; | |
| 12 | let app: App; | |
| 13 | let servers: Servers; | |
| 14 | let body: string; | |
| 15 | ||
| 16 | beforeAll(async () => { | |
| 17 | dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-guides-")); | |
| 18 | app = createApp({ dataDir, port: 0 }); | |
| 19 | servers = await serve(app); | |
| 20 | body = await (await fetch(`${servers.url}/sitemap.xml`)).text(); | |
| 21 | }); | |
| 22 | afterAll(async () => { | |
| 23 | servers.stop(); | |
| 24 | app.close(); | |
| 25 | await rm(dataDir, { recursive: true, force: true }); | |
| 26 | }); | |
| 27 | ||
| 28 | test("every doc the index lists is in the sitemap, and so are /templates, /security and /status", () => { | |
| 29 | const entries = docEntries(); | |
| 30 | expect(entries.length).toBeGreaterThan(10); | |
| 31 | for (const e of entries) expect(body).toContain(`<loc>${servers.url}/docs/${e.name}</loc>`); | |
| 32 | for (const p of ["templates", "security", "status"]) expect(body).toContain(`<loc>${servers.url}/${p}</loc>`); | |
| 33 | }); | |
| 34 | ||
| 35 | test("every lastmod is a real date, the file is well-formed and under 50,000 URLs", () => { | |
This only checks the lastmod values that exist; it never asserts that each guide and the three pages have one, which is what box 1 requires. Assert that every page URL has a lastmod, and test the no-git path. |
||
| 36 | const mods = [...body.matchAll(/<lastmod>([^<]*)<\/lastmod>/g)].map((m) => m[1]!); | |
| 37 | for (const m of mods) { | |
| 38 | expect(m).toMatch(/^\d{4}-\d{2}-\d{2}$/); | |
| 39 | expect(Number.isNaN(Date.parse(m))).toBe(false); | |
| 40 | } | |
| 41 | const locs = body.match(/<url>/g)!.length; | |
| 42 | expect(locs).toBe(body.match(/<\/url>/g)!.length); | |
| 43 | expect(locs).toBeLessThan(50_000); | |
| 44 | expect(body.startsWith('<?xml version="1.0" encoding="UTF-8"?>\n<urlset')).toBe(true); | |
| 45 | expect(body.trimEnd().endsWith("</urlset>")).toBe(true); | |
| 46 | }); | |
| 47 | ||
| 48 | test("the last commit's day is the first one the log names for a file", () => { | |
| 49 | const days = parseLastChanges("@2026-11-02\ndocs/A.md\ndocs/B.md\n\n@2026-10-01\ndocs/A.md\ndocs/C.md\n"); | |
| 50 | expect(days.get("docs/A.md")).toBe("2026-11-02"); | |
| 51 | expect(days.get("docs/C.md")).toBe("2026-10-01"); | |
| 52 | expect(parseLastChanges("@garbage\ndocs/D.md\n").has("docs/D.md")).toBe(false); | |
| 53 | }); | |
| 54 | ||
| 55 | test("every guide and the three pages carry a lastmod", () => { | |
| 56 | const paths = [...docEntries().map((e) => `/docs/${e.name}`), "/templates", "/security", "/status"]; | |
| 57 | for (const p of paths) expect(body).toMatch(new RegExp(`<loc>${servers.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}${p}</loc><lastmod>\\d{4}-\\d{2}-\\d{2}</lastmod>`)); | |
| 58 | }); | |
| 59 | ||
| 60 | test("with no git history the day is the release's, else the day the process started", () => { | |
| 61 | const none = { path: "/docs/x.md", lastmod: null }; | |
| 62 | expect(pageDay({ path: "/docs/x.md", lastmod: "2026-01-02" }, "2026-05-05", "2026-06-06")).toBe("2026-01-02"); | |
| 63 | expect(pageDay(none, "2026-05-05", "2026-06-06")).toBe("2026-05-05"); | |
| 64 | expect(pageDay(none, null, "2026-06-06")).toBe("2026-06-06"); | |
| 65 | }); | |
| 66 | ||
| 67 | test("a good git read is kept for the life of the process (a failed one is retried after a minute)", async () => { | |
| 68 | const first = await sitemapPages(); | |
| 69 | expect(await sitemapPages()).toBe(first); | |
| 70 | }); |
test/standards-sitemap.test.ts +9 −1
| @@ -58,7 +58,15 @@ test("the sitemap lists / , /explore and the public repository, never the privat | ||
| 58 | 58 | test("a push gives the repository a lastmod and changes the ETag; a repeat with the ETag is a bare 304", async () => { |
| 59 | 59 | const first = await get("/sitemap.xml"); |
| 60 | 60 | const before = first.headers.get("etag")!; |
| 61 | expect(await first.text()).not.toContain("<lastmod>"); | |
| 61 | // Before a push the repositories have no lastmod; the guide pages and /changelog are the only | |
outdated
This weakens the old assertion to a regex on one repo, so it would no longer catch a stray lastmod on other rows. Keep it targeted, and make the 'no lastmod' meaning explicit for repos only. |
||
| 62 | // rows that do. | |
| 63 | const rows = [...(await first.text()).matchAll(/<url><loc>([^<]*)<\/loc>(<lastmod>[^<]*<\/lastmod>)?<\/url>/g)]; | |
| 64 | expect(rows.length).toBeGreaterThan(8); | |
| 65 | for (const [, loc, mod] of rows) { | |
| 66 | const path = new URL(loc!).pathname; | |
| 67 | const dated = path === "/changelog" || path.startsWith("/docs/") || ["/templates", "/security", "/status"].includes(path); | |
| 68 | if (!dated) expect(mod).toBeUndefined(); | |
| 69 | } | |
| 62 | 70 | const again = await get("/sitemap.xml", { "if-none-match": before }); |
| 63 | 71 | expect(again.status).toBe(304); |
| 64 | 72 | expect(await again.text()).toBe(""); |
/templates,/securityand/statusalways get null here, so they depend entirely on the release fallback, which is unproven (see standards.ts). Alsokeptcaches a failed git read (an empty Map) for the life of the process; don't cache a failure.