Public: anyone can read this; the crew can change it.
/sitemap.xml answers under 15 ms #6159
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6130-sitemap-xml-answers-under-15-ms → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +96
−17
src/web/standards.ts +36 −17
| @@ -175,26 +175,45 @@ function xml(s: string): string { | ||
| 175 | 175 | ); |
| 176 | 176 | } |
| 177 | 177 | |
| 178 | /** The sitemap and its ETag per origin, valid while the listing read is the same one. */ | |
| 179 | const sitemapBodies = new Map<string, { rows: unknown; body: string; etag: string }>(); | |
| 180 | ||
| 178 | 181 | /** GET /sitemap.xml — the front page and every public repository. */ |
| 179 | 182 | export function sitemapXml(ctx: Ctx): Response { |
| 180 | // Kept without an origin, and the origin is put on per answer, so a | |
| 181 | // caller who varies `Host` cannot make the forge read the rows again. | |
| 183 | // The rows are kept without an origin, and the origin is put on per answer, | |
| 184 | // so a caller who varies `Host` cannot make the forge read the rows again. | |
| 182 | 185 | const repos = timePhase(ctx.request, "repos", () => sitemapRepos(ctx.app.db)); |
| 183 | // Escaping is per character, so the escaped origin and the escaped path | |
| 184 | // join into the escaped URL. | |
| 185 | const body = timePhase(ctx.request, "render", () => { | |
| 186 | const origin = xml(ctx.url.origin); | |
| 187 | const urls = [ | |
| 188 | `<url><loc>${origin}/</loc></url>`, | |
| 189 | `<url><loc>${origin}/explore</loc></url>`, | |
| 190 | ...repos.map((r) => { | |
| 191 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; | |
| 192 | return `<url><loc>${origin}${xml(`/${r.path}`)}</loc>${mod}</url>`; | |
| 193 | }), | |
| 194 | ]; | |
| 195 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; | |
| 196 | }); | |
| 197 | return text(body, "application/xml; charset=utf-8"); | |
| 186 | const origin = ctx.url.origin; | |
| 187 | let kept = sitemapBodies.get(origin); | |
| 188 | if (kept === undefined || kept.rows !== repos) { | |
| 189 | // Escaping is per character, so the escaped origin and the escaped path | |
| 190 | // join into the escaped URL. | |
| 191 | const body = timePhase(ctx.request, "render", () => { | |
| 192 | const o = xml(origin); | |
| 193 | const urls = [ | |
| 194 | `<url><loc>${o}/</loc></url>`, | |
| 195 | `<url><loc>${o}/explore</loc></url>`, | |
| 196 | ...repos.map((r) => { | |
| 197 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; | |
| 198 | return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`; | |
| 199 | }), | |
| 200 | ]; | |
| 201 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; | |
| 202 | }); | |
| 203 | const etag = timePhase(ctx.request, "etag", () => `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`); | |
| 204 | kept = { rows: repos, body, etag }; | |
| 205 | // Origins come from the Host header: keep a handful, not one per guess. | |
| 206 | if (sitemapBodies.size >= 16) sitemapBodies.clear(); | |
| 207 | sitemapBodies.set(origin, kept); | |
| 208 | } | |
| 209 | const cache = "public, max-age=300"; | |
| 210 | const sent = ctx.request.headers.get("if-none-match"); | |
| 211 | if (sent !== null && sent.split(",").some((t) => t.trim() === kept.etag)) | |
| 212 | return new Response(null, { status: 304, headers: { etag: kept.etag, "cache-control": cache } }); | |
| 213 | const res = text(kept.body, "application/xml; charset=utf-8"); | |
| 214 | res.headers.set("etag", kept.etag); | |
| 215 | res.headers.set("cache-control", cache); | |
| 216 | return res; | |
| 198 | 217 | } |
| 199 | 218 | |
| 200 | 219 | /** GET /manifest.webmanifest — the forge as an app on a home screen. */ |
test/sitemap.test.ts added +60
| @@ -0,0 +1,60 @@ | ||
| 1 | /** /sitemap.xml: 100 public and 20 private repositories, built fast, cached with an ETag. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { forgetListings } from "../src/core/listings.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 11 | ||
| 12 | let dataDir: string; | |
| 13 | let app: App; | |
| 14 | let servers: Servers; | |
| 15 | ||
| 16 | beforeAll(async () => { | |
| 17 | dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-")); | |
| 18 | app = createApp({ dataDir, port: 0 }); | |
| 19 | servers = await serve(app); | |
| 20 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 21 | if (!a.ok) throw new Error(a.error); | |
| 22 | for (let i = 0; i < 120; i++) { | |
| 23 | const pub = i < 100; | |
| 24 | const r = await createRepo(app, a.value, { name: `${pub ? "pub" : "priv"}${i}`, visibility: pub ? "public" : "private" }); | |
| 25 | if (!r.ok) throw new Error(String(r.error)); | |
| 26 | } | |
| 27 | forgetListings(app.db); | |
| 28 | }); | |
| 29 | ||
| 30 | afterAll(async () => { | |
| 31 | servers.stop(); | |
| 32 | app.close(); | |
| 33 | await rm(dataDir, { recursive: true, force: true }); | |
| 34 | }); | |
| 35 | ||
| 36 | const phases = (h: string) => [...h.matchAll(/(\w+);dur=([\d.]+)/g)].reduce((n, m) => n + Number(m[2]), 0); | |
| 37 | ||
| 38 | test("lists the 100 public repositories and none of the 20 private ones, built in under 15 ms", async () => { | |
| 39 | forgetListings(app.db); | |
| 40 | const res = await fetch(`${servers.url}/sitemap.xml`); | |
| 41 | const body = await res.text(); | |
| 42 | expect(body.match(/\/alice\/pub\d+</g)?.length).toBe(100); | |
| 43 | expect(body).not.toContain("priv"); | |
| 44 | const timing = res.headers.get("server-timing") ?? ""; | |
| 45 | for (const phase of ["repos", "render", "etag"]) expect(timing).toContain(`${phase};dur=`); | |
| 46 | expect(phases(timing)).toBeLessThan(15); | |
| 47 | }); | |
| 48 | ||
| 49 | test("a repeat with If-None-Match gets a 304, a stale tag gets the document", async () => { | |
| 50 | const first = await fetch(`${servers.url}/sitemap.xml`); | |
| 51 | const etag = first.headers.get("etag")!; | |
| 52 | expect(etag).toMatch(/^"[0-9a-f]{32}"$/); | |
| 53 | await first.text(); | |
| 54 | const again = await fetch(`${servers.url}/sitemap.xml`, { headers: { "if-none-match": etag } }); | |
| 55 | expect(again.status).toBe(304); | |
| 56 | expect(await again.text()).toBe(""); | |
| 57 | const stale = await fetch(`${servers.url}/sitemap.xml`, { headers: { "if-none-match": '"nope"' } }); | |
| 58 | expect(stale.status).toBe(200); | |
| 59 | expect((await stale.text()).length).toBeGreaterThan(1000); | |
| 60 | }); |
new above
offline
The cache hit depends on sitemapRepos returning the same array reference until the listing is forgotten. That holds today, but a comment saying so would stop a later refactor that copies the rows from silently turning every request into a re-render.