Public: anyone can read this; the crew can change it.

/sitemap.xml also lists the guides, /templates, /security and /status, each with its last change #13905

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13884-sitemap-xml-also-lists-the-guides-templa → main 5 commits fork 4 threads · 4 unresolved

  • Merged. torque/13884-sitemap-xml-also-lists-the-guides-templa was merged into main as 9838ca5.

Review

Sign in to review this change.
4 files changed +172 −5
src/web/sitemap_pages.ts added +80
@@ -0,0 +1,80 @@
1 /**
2 * The guide pages of the sitemap (#13884): every document `/docs` links, from
3 * the same index the page reads (`view/docs.ts`), so a new guide appears with
4 * no second edit, plus the three public pages that have no file. Each file's
5 * `lastmod` is the day of its last commit, read once with one fixed-argument
6 * `git log` over `docs/` in the release's checkout. Where there is no
7 * history (a release shipped without `.git`) or the page has no file, the
8 * caller puts the newest release's day.
9 */
10
11 import { join } from "node:path";
12 import { runGit } from "../git/env.ts";
13 import { docEntries } from "../view/docs.ts";
14
15 export interface SitemapPage {
16 readonly path: string;
17 /** `YYYY-MM-DD` of the file's last commit; null when there is no file or no history. */
18 readonly lastmod: string | null;
19 }
20
21 /** Public pages that are code, not a file. */
22 const FILELESS = ["/templates", "/security", "/status"] as const;
23 const ROOT = join(import.meta.dir, "..", "..");
24 const DAY = /^\d{4}-\d{2}-\d{2}$/;
25
26 /** `git log --name-only` output as file → newest day (the log is newest first). */
27 export function parseLastChanges(out: string): Map<string, string> {
28 const days = new Map<string, string>();
29 let day: string | undefined;
30 for (const line of out.split("\n")) {
31 if (line.startsWith("@")) day = DAY.test(line.slice(1)) ? line.slice(1) : undefined;
32 else if (line !== "" && day !== undefined && !days.has(line)) days.set(line, day);
33 }
34 return days;
35 }
36
37 async function lastChanges(): Promise<Map<string, string> | undefined> {
38 const read = await runGit(["log", "--format=@%cs", "--name-only", "--", "docs"], { cwd: ROOT });
39 return read.ok ? parseLastChanges(read.value) : undefined;
40 }
41
42 /** The day this process started (UTC): the last resort when no release carries a time. */
43 const STARTED = new Date().toISOString().slice(0, 10);
44
45 /**
outdated

/templates, /security and /status always get null here, so they depend entirely on the release fallback, which is unproven (see standards.ts). Also kept caches a failed git read (an empty Map) for the life of the process; don't cache a failure.

46 * The day a page's `lastmod` shows: its file's last commit, else the newest release's day, else the
47 * day this process started, so a page always has one.
48 */
49 export function pageDay(page: SitemapPage, released: string | null, started: string = STARTED): string {
50 return page.lastmod ?? released ?? started;
51 }
52
53 /** A failed read is kept this long, so a checkout without git is not asked on every request. */
54 const RETRY_MS = 60_000;
55 let kept: { pages: readonly SitemapPage[]; failedAt: number | null } | undefined;
56 let reading: Promise<readonly SitemapPage[]> | undefined;
57
58 function build(days: Map<string, string> | undefined): readonly SitemapPage[] {
59 return [
60 ...docEntries().map((e) => ({ path: `/docs/${e.name}`, lastmod: days?.get(`docs/${e.name}`) ?? null })),
61 ...FILELESS.map((path) => ({ path, lastmod: null })),
62 ];
63 }
64
65 /**
66 * The guide pages, in the order /docs lists them, then the fileless ones. A good read is kept for
67 * the life of the process; a failed one only for a minute.
68 */
69 export function sitemapPages(now: number = Date.now()): Promise<readonly SitemapPage[]> {
70 if (kept !== undefined && (kept.failedAt === null || now - kept.failedAt < RETRY_MS)) return Promise.resolve(kept.pages);
71 reading ??= lastChanges()
72 .then((days) => {
73 kept = { pages: build(days), failedAt: days === undefined ? now : null };
74 return kept.pages;
75 })
76 .finally(() => {
77 reading = undefined;
78 });
79 return reading;
80 }
src/web/standards.ts +13 −4
@@ -33,6 +33,7 @@ import { CRAWL_DELAY_S, DEEP_SEGMENTS } from "../http/crawl.ts";
3333 import { markPng, type MarkShape } from "../design/mark.ts";
3434 import { metaThemeColor } from "../design/themes.ts";
3535 import { BODY_MAX, TITLE_MAX, reportsClosed } from "../core/reports.ts";
36 import { pageDay, sitemapPages } from "./sitemap_pages.ts";
3637 import { dayOf } from "../release/days.ts";
3738 import { readLandings } from "../release/landings.ts";
3839 import { takesOutside } from "../actions/contribute_token.ts";
@@ -296,9 +297,9 @@ function xml(s: string): string {
296297 }
297298
298299 /** The sitemap and its ETag per origin, valid while the listing read is the same one. */
299 const sitemapBodies = new Map<string, { rows: unknown; changed: string | null; body: string; etag: string }>();
300 const sitemapBodies = new Map<string, { rows: unknown; pages: unknown; released: string | null; changed: string | null; body: string; etag: string }>();
300301
301 /** GET /sitemap.xml — the front page, the changelog and every public repository. */
302 /** GET /sitemap.xml — the front page, the changelog, the guides and every public repository. */
302303 export async function sitemapXml(ctx: Ctx): Promise<Response> {
303304 // The rows are kept without an origin, and the origin is put on per answer,
304305 // so a caller who varies `Host` cannot make the forge read the rows again.
@@ -309,8 +310,13 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> {
309310 newestLanding(ctx.app.db, async () => (await readLandings(ctx.app))?.live[0]?.at),
310311 );
311312 const changed = newest === undefined ? null : dayOf(newest, "UTC");
313 const pages = await sitemapPages();
314 // The guides' lastmod where git has none: the newest release's day (frozen releases carry no
315 // time), then the start day.
outdated

On the preview this is null, so every guide and the three fileless pages are emitted with no lastmod. The fallback must give a day (the newest release, or the process start or build day) or box 1 stays unmet. Add a test with an empty lastChanges result and a known release time that asserts every guide row has a lastmod.

316 const at = shippedReleases().find((r) => r.at !== undefined)?.at;
317 const released = at === undefined ? null : dayOf(new Date(at).toISOString(), "UTC");
312318 let kept = sitemapBodies.get(origin);
313 if (kept === undefined || kept.rows !== repos || kept.changed !== changed) {
319 if (kept === undefined || kept.rows !== repos || kept.pages !== pages || kept.released !== released || kept.changed !== changed) {
314320 // Escaping is per character, so the escaped origin and the escaped path
315321 // join into the escaped URL.
316322 const body = timePhase(ctx.request, "render", () => {
@@ -324,6 +330,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> {
324330 `<url><loc>${o}/privacy</loc></url>`,
325331 `<url><loc>${o}/terms</loc></url>`,
326332 `<url><loc>${o}/docs</loc></url>`,
333 ...pages.map((p) => {
334 return `<url><loc>${o}${xml(p.path)}</loc><lastmod>${pageDay(p, released)}</lastmod></url>`;
335 }),
327336 ...repos.map((r) => {
328337 const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`;
329338 return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`;
@@ -332,7 +341,7 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> {
332341 return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`;
333342 });
334343 const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`);
335 kept = { rows: repos, changed, body, etag };
344 kept = { rows: repos, pages, released, changed, body, etag };
336345 // Origins come from the Host header: keep a handful, not one per guess.
337346 if (sitemapBodies.size >= 16) sitemapBodies.clear();
338347 sitemapBodies.set(origin, kept);
test/sitemap-guides.test.ts added +70
@@ -0,0 +1,70 @@
1 /** /sitemap.xml lists every guide /docs links, plus /templates, /security and /status (#13884). */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import { pageDay, parseLastChanges, sitemapPages } from "../src/web/sitemap_pages.ts";
9 import { docEntries } from "../src/view/docs.ts";
10
11 let dataDir: string;
12 let app: App;
13 let servers: Servers;
14 let body: string;
15
16 beforeAll(async () => {
17 dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-guides-"));
18 app = createApp({ dataDir, port: 0 });
19 servers = await serve(app);
20 body = await (await fetch(`${servers.url}/sitemap.xml`)).text();
21 });
22 afterAll(async () => {
23 servers.stop();
24 app.close();
25 await rm(dataDir, { recursive: true, force: true });
26 });
27
28 test("every doc the index lists is in the sitemap, and so are /templates, /security and /status", () => {
29 const entries = docEntries();
30 expect(entries.length).toBeGreaterThan(10);
31 for (const e of entries) expect(body).toContain(`<loc>${servers.url}/docs/${e.name}</loc>`);
32 for (const p of ["templates", "security", "status"]) expect(body).toContain(`<loc>${servers.url}/${p}</loc>`);
33 });
34
35 test("every lastmod is a real date, the file is well-formed and under 50,000 URLs", () => {

This only checks the lastmod values that exist; it never asserts that each guide and the three pages have one, which is what box 1 requires. Assert that every page URL has a lastmod, and test the no-git path.

36 const mods = [...body.matchAll(/<lastmod>([^<]*)<\/lastmod>/g)].map((m) => m[1]!);
37 for (const m of mods) {
38 expect(m).toMatch(/^\d{4}-\d{2}-\d{2}$/);
39 expect(Number.isNaN(Date.parse(m))).toBe(false);
40 }
41 const locs = body.match(/<url>/g)!.length;
42 expect(locs).toBe(body.match(/<\/url>/g)!.length);
43 expect(locs).toBeLessThan(50_000);
44 expect(body.startsWith('<?xml version="1.0" encoding="UTF-8"?>\n<urlset')).toBe(true);
45 expect(body.trimEnd().endsWith("</urlset>")).toBe(true);
46 });
47
48 test("the last commit's day is the first one the log names for a file", () => {
49 const days = parseLastChanges("@2026-11-02\ndocs/A.md\ndocs/B.md\n\n@2026-10-01\ndocs/A.md\ndocs/C.md\n");
50 expect(days.get("docs/A.md")).toBe("2026-11-02");
51 expect(days.get("docs/C.md")).toBe("2026-10-01");
52 expect(parseLastChanges("@garbage\ndocs/D.md\n").has("docs/D.md")).toBe(false);
53 });
54
55 test("every guide and the three pages carry a lastmod", () => {
56 const paths = [...docEntries().map((e) => `/docs/${e.name}`), "/templates", "/security", "/status"];
57 for (const p of paths) expect(body).toMatch(new RegExp(`<loc>${servers.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}${p}</loc><lastmod>\\d{4}-\\d{2}-\\d{2}</lastmod>`));
58 });
59
60 test("with no git history the day is the release's, else the day the process started", () => {
61 const none = { path: "/docs/x.md", lastmod: null };
62 expect(pageDay({ path: "/docs/x.md", lastmod: "2026-01-02" }, "2026-05-05", "2026-06-06")).toBe("2026-01-02");
63 expect(pageDay(none, "2026-05-05", "2026-06-06")).toBe("2026-05-05");
64 expect(pageDay(none, null, "2026-06-06")).toBe("2026-06-06");
65 });
66
67 test("a good git read is kept for the life of the process (a failed one is retried after a minute)", async () => {
68 const first = await sitemapPages();
69 expect(await sitemapPages()).toBe(first);
70 });
test/standards-sitemap.test.ts +9 −1
@@ -58,7 +58,15 @@ test("the sitemap lists / , /explore and the public repository, never the privat
5858 test("a push gives the repository a lastmod and changes the ETag; a repeat with the ETag is a bare 304", async () => {
5959 const first = await get("/sitemap.xml");
6060 const before = first.headers.get("etag")!;
61 expect(await first.text()).not.toContain("<lastmod>");
61 // Before a push the repositories have no lastmod; the guide pages and /changelog are the only
outdated

This weakens the old assertion to a regex on one repo, so it would no longer catch a stray lastmod on other rows. Keep it targeted, and make the 'no lastmod' meaning explicit for repos only.

62 // rows that do.
63 const rows = [...(await first.text()).matchAll(/<url><loc>([^<]*)<\/loc>(<lastmod>[^<]*<\/lastmod>)?<\/url>/g)];
64 expect(rows.length).toBeGreaterThan(8);
65 for (const [, loc, mod] of rows) {
66 const path = new URL(loc!).pathname;
67 const dated = path === "/changelog" || path.startsWith("/docs/") || ["/templates", "/security", "/status"].includes(path);
68 if (!dated) expect(mod).toBeUndefined();
69 }
6270 const again = await get("/sitemap.xml", { "if-none-match": before });
6371 expect(again.status).toBe(304);
6472 expect(await again.text()).toBe("");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs