Public: anyone can read this; the crew can change it.
/sitemap.xml lists /changelog, so what went live is findable from search #8343
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/8327-sitemap-xml-lists-changelog-so-what-went → main 1 commit fork
2 files
changed +38
−6
src/web/standards.ts +12 −6
| @@ -28,6 +28,8 @@ import { CRAWL_DELAY_S, DEEP_SEGMENTS } from "../http/crawl.ts"; | ||
| 28 | 28 | import { markPng, type MarkShape } from "../design/mark.ts"; |
| 29 | 29 | import { metaThemeColor } from "../design/themes.ts"; |
| 30 | 30 | import { BODY_MAX, TITLE_MAX } from "../core/reports.ts"; |
| 31 | import { dayOf } from "../release/days.ts"; | |
| 32 | import { readLandings } from "../release/landings.ts"; | |
| 31 | 33 | import { REPORT_LINK_TITLE } from "../actions/report_words.ts"; |
| 32 | 34 | import { sshTellCommand } from "../ssh/command.ts"; |
| 33 | 35 | import { llmsRepos, sitemapRepos } from "../core/listings.ts"; |
| @@ -38,7 +40,7 @@ import { SITE_LINE } from "./site_line.ts"; | ||
| 38 | 40 | import { mcpToolsSection } from "./llms_tools.ts"; |
| 39 | 41 | import { HARNESS_PLUGINS, mcpCard, mcpPointers } from "../core/mcp_card.ts"; |
| 40 | 42 | import { running } from "../kernel/version.ts"; |
| 41 | import { timePhase } from "../core/route_timing.ts"; | |
| 43 | import { timePhase, timePhaseAsync } from "../core/route_timing.ts"; | |
| 42 | 44 | import { shippedReleases, type Shipped } from "../release/shipped.ts"; |
| 43 | 45 | |
| 44 | 46 | const DAY = 24 * 60 * 60 * 1000; |
| @@ -188,16 +190,19 @@ function xml(s: string): string { | ||
| 188 | 190 | } |
| 189 | 191 | |
| 190 | 192 | /** The sitemap and its ETag per origin, valid while the listing read is the same one. */ |
| 191 | const sitemapBodies = new Map<string, { rows: unknown; body: string; etag: string }>(); | |
| 193 | const sitemapBodies = new Map<string, { rows: unknown; changed: string | null; body: string; etag: string }>(); | |
| 192 | 194 | |
| 193 | /** GET /sitemap.xml — the front page and every public repository. */ | |
| 194 | export function sitemapXml(ctx: Ctx): Response { | |
| 195 | /** GET /sitemap.xml — the front page, the changelog and every public repository. */ | |
| 196 | export async function sitemapXml(ctx: Ctx): Promise<Response> { | |
| 195 | 197 | // The rows are kept without an origin, and the origin is put on per answer, |
| 196 | 198 | // so a caller who varies `Host` cannot make the forge read the rows again. |
| 197 | 199 | const repos = timePhase(ctx.request, "repos", () => sitemapRepos(ctx.app.db)); |
| 198 | 200 | const origin = ctx.url.origin; |
| 201 | // The newest landing's day (UTC), the changelog's lastmod; none when nothing has landed. | |
| 202 | const newest = await timePhaseAsync(ctx.request, "landings", async () => (await readLandings(ctx.app).catch(() => undefined))?.live[0]?.at); | |
| 203 | const changed = newest === undefined ? null : dayOf(newest, "UTC"); | |
| 199 | 204 | let kept = sitemapBodies.get(origin); |
| 200 | if (kept === undefined || kept.rows !== repos) { | |
| 205 | if (kept === undefined || kept.rows !== repos || kept.changed !== changed) { | |
| 201 | 206 | // Escaping is per character, so the escaped origin and the escaped path |
| 202 | 207 | // join into the escaped URL. |
| 203 | 208 | const body = timePhase(ctx.request, "render", () => { |
| @@ -205,6 +210,7 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 205 | 210 | const urls = [ |
| 206 | 211 | `<url><loc>${o}/</loc></url>`, |
| 207 | 212 | `<url><loc>${o}/explore</loc></url>`, |
| 213 | `<url><loc>${o}/changelog</loc>${changed === null ? "" : `<lastmod>${changed}</lastmod>`}</url>`, | |
| 208 | 214 | ...repos.map((r) => { |
| 209 | 215 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; |
| 210 | 216 | return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`; |
| @@ -213,7 +219,7 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 213 | 219 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; |
| 214 | 220 | }); |
| 215 | 221 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); |
| 216 | kept = { rows: repos, body, etag }; | |
| 222 | kept = { rows: repos, changed, body, etag }; | |
| 217 | 223 | // Origins come from the Host header: keep a handful, not one per guess. |
| 218 | 224 | if (sitemapBodies.size >= 16) sitemapBodies.clear(); |
| 219 | 225 | sitemapBodies.set(origin, kept); |
test/standards-sitemap.test.ts +26
| @@ -79,3 +79,29 @@ test("robots.txt disallows the private surfaces and names the sitemap on the req | ||
| 79 | 79 | for (const p of ["/signin", "/signup", "/settings", "/search", "/api/"]) expect(lines).toContain(`Disallow: ${p}`); |
| 80 | 80 | expect(lines).toContain(`Sitemap: ${servers.url}/sitemap.xml`); |
| 81 | 81 | }); |
| 82 | ||
| 83 | test("the sitemap lists /changelog after /explore, with the newest landing's day as lastmod once one has landed", async () => { | |
| 84 | const before = await get("/sitemap.xml"); | |
| 85 | const bodyBefore = await before.text(); | |
| 86 | const etagBefore = before.headers.get("etag")!; | |
| 87 | const o = servers.url; | |
| 88 | expect(bodyBefore).toContain(`<loc>${o}/explore</loc></url>\n<url><loc>${o}/changelog</loc></url>`); | |
| 89 | ||
| 90 | const repoId = app.db.query<{ id: string }, []>("SELECT id FROM repo WHERE name = 'open'").get()!.id; | |
| 91 | const merge = (n: number, at: string) => { | |
| 92 | app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'Landed', '', ?4, 'closed', ?5)`).run(`ci${n}`, repoId, n + 100, alice.id, at); | |
| 93 | app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, '')`).run(`ci${n}`, `c/${n}`, repoId, at); | |
| 94 | }; | |
| 95 | merge(1, "2026-03-04T10:00:00.000Z"); | |
| 96 | const first = await get("/sitemap.xml", { "if-none-match": etagBefore }); | |
| 97 | expect(first.status).toBe(200); | |
| 98 | expect(await first.text()).toContain(`<loc>${o}/changelog</loc><lastmod>2026-03-04</lastmod></url>`); | |
| 99 | const etagFirst = first.headers.get("etag")!; | |
| 100 | expect(etagFirst).not.toBe(etagBefore); | |
| 101 | ||
| 102 | merge(2, "2026-03-09T23:00:00.000Z"); | |
| 103 | const second = await get("/sitemap.xml", { "if-none-match": etagFirst }); | |
| 104 | expect(second.status).toBe(200); | |
| 105 | expect(await second.text()).toContain("<lastmod>2026-03-09</lastmod>"); | |
| 106 | expect(second.headers.get("etag")).not.toBe(etagFirst); | |
| 107 | }); |
new above
offline