Public: anyone can read this; the crew can change it.
/sitemap.xml answers from a cached read: 46 ms of landings becomes under 5 ms #12992
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12988-sitemap-xml-answers-from-a-cached-read-4 → main 3 commits fork
4 files
changed +47
−3
src/core/listings.ts +17 −1
| @@ -18,11 +18,12 @@ export const LLMS_LINES = 50; | ||
| 18 | 18 | interface Kept { |
| 19 | 19 | readonly sitemap: Fresh<readonly SitemapRepo[]>; |
| 20 | 20 | readonly llms: Fresh<LlmsListing>; |
| 21 | readonly landed: Fresh<Promise<string | undefined>>; | |
| 21 | 22 | } |
| 22 | 23 | const kept = new WeakMap<Database, Kept>(); |
| 23 | 24 | |
| 24 | 25 | /** Test seam: how many times each listing was read from the database. */ |
| 25 | export const listingBuilds = { sitemap: 0, llms: 0 }; | |
| 26 | export const listingBuilds = { sitemap: 0, llms: 0, landed: 0 }; | |
| 26 | 27 | |
| 27 | 28 | function holders(db: Database): Kept { |
| 28 | 29 | let k = kept.get(db); |
| @@ -30,6 +31,7 @@ function holders(db: Database): Kept { | ||
| 30 | 31 | k = { |
| 31 | 32 | sitemap: new Fresh(LISTING_MS, 1), |
| 32 | 33 | llms: new Fresh(LISTING_MS, 1), |
| 34 | landed: new Fresh(LISTING_MS, 1), | |
| 33 | 35 | }; |
| 34 | 36 | kept.set(db, k); |
| 35 | 37 | } |
| @@ -41,6 +43,20 @@ export function forgetListings(db: Database): void { | ||
| 41 | 43 | const k = kept.get(db); |
| 42 | 44 | k?.sitemap.clear(); |
| 43 | 45 | k?.llms.clear(); |
| 46 | k?.landed.clear(); | |
| 47 | } | |
| 48 | ||
| 49 | /** | |
| 50 | * When the newest landing was merged, read once a minute: the sitemap's | |
| 51 | * changelog lastmod. `read` is the landings read (it asks git); a failed | |
| 52 | * read is kept as "none" for the minute, not retried per request. Landings | |
| 53 | * list only public repositories, so a visibility change drops it with the rest. | |
| 54 | */ | |
| 55 | export function newestLanding(db: Database, read: () => Promise<string | undefined>): Promise<string | undefined> { | |
| 56 | return holders(db).landed.get("", () => { | |
| 57 | listingBuilds.landed++; | |
| 58 | return read().catch(() => undefined); | |
| 59 | }); | |
| 44 | 60 | } |
| 45 | 61 | |
| 46 | 62 | /** A listed repository: `owner/name` and its last push (ISO time), or null. */ |
src/web/standards.ts +4 −2
| @@ -38,7 +38,7 @@ import { takesOutside } from "../actions/contribute_token.ts"; | ||
| 38 | 38 | import { OFFER_PATCH_MAX, OFFERS_PER_SOURCE } from "../actions/offer.ts"; |
| 39 | 39 | import { REPORT_LINK_TITLE } from "../actions/report_words.ts"; |
| 40 | 40 | import { sshTellCommand } from "../ssh/command.ts"; |
| 41 | import { llmsRepos, sitemapRepos, type LlmsListing } from "../core/listings.ts"; | |
| 41 | import { llmsRepos, newestLanding, sitemapRepos, type LlmsListing } from "../core/listings.ts"; | |
| 42 | 42 | import { canRead, getRepo, type Repo } from "../core/repos.ts"; |
| 43 | 43 | import { publicForgeRepo } from "../core/forge_repo.ts"; |
| 44 | 44 | import type { Ctx } from "../http/context.ts"; |
| @@ -269,7 +269,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> { | ||
| 269 | 269 | const repos = timePhase(ctx.request, "repos", () => sitemapRepos(ctx.app.db)); |
| 270 | 270 | const origin = ctx.url.origin; |
| 271 | 271 | // The newest landing's day (UTC), the changelog's lastmod; none when nothing has landed. |
| 272 | const newest = await timePhaseAsync(ctx.request, "landings", async () => (await readLandings(ctx.app).catch(() => undefined))?.live[0]?.at); | |
| 272 | const newest = await timePhaseAsync(ctx.request, "landings", () => | |
| 273 | newestLanding(ctx.app.db, async () => (await readLandings(ctx.app))?.live[0]?.at), | |
| 274 | ); | |
| 273 | 275 | const changed = newest === undefined ? null : dayOf(newest, "UTC"); |
| 274 | 276 | let kept = sitemapBodies.get(origin); |
| 275 | 277 | if (kept === undefined || kept.rows !== repos || kept.changed !== changed) { |
test/sitemap-timing.test.ts +23
| @@ -50,3 +50,26 @@ test("the sitemap lists /explore exactly once, right after the front page", asyn | ||
| 50 | 50 | expect(locs.filter((l) => l.endsWith("/explore")).length).toBe(1); |
| 51 | 51 | expect(body).not.toContain("hidden"); |
| 52 | 52 | }); |
| 53 | ||
| 54 | test("the landings read behind the sitemap is made once a minute, a repeat is a 304, and a repository made private drops out at once", async () => { | |
| 55 | const { listingBuilds, forgetListings } = await import("../src/core/listings.ts"); | |
| 56 | const { getRepo } = await import("../src/core/repos.ts"); | |
| 57 | const { updateRepoSettings } = await import("../src/actions/repos.ts"); | |
| 58 | const alice = (await import("../src/core/users.ts")).getUserByName(app.db, "alice")!; | |
| 59 | forgetListings(app.db); | |
| 60 | const first = await fetch(`${servers.url}/sitemap.xml`); | |
| 61 | const etag = first.headers.get("etag")!; | |
| 62 | await first.text(); | |
| 63 | const sitemap = listingBuilds.sitemap; | |
| 64 | const landed = listingBuilds.landed; | |
| 65 | for (let i = 0; i < 20; i++) await fetch(`${servers.url}/sitemap.xml`).then((r) => r.text()); | |
| 66 | expect(listingBuilds.sitemap).toBe(sitemap); | |
| 67 | expect(listingBuilds.landed).toBe(landed); | |
| 68 | const again = await fetch(`${servers.url}/sitemap.xml`, { headers: { "if-none-match": etag } }); | |
| 69 | expect(again.status).toBe(304); | |
| 70 | const done = await updateRepoSettings(app, alice, getRepo(app.db, "alice", "two")!, { visibility: "private" }); | |
| 71 | if (!done.ok) throw new Error(String(done.error)); | |
| 72 | const after = await (await fetch(`${servers.url}/sitemap.xml`)).text(); | |
| 73 | expect(after).not.toContain("/alice/two</loc>"); | |
| 74 | expect(listingBuilds.landed).toBe(landed + 1); | |
| 75 | }); |
test/standards-sitemap.test.ts +3
| @@ -93,6 +93,8 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day | ||
| 93 | 93 | app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, '')`).run(`ci${n}`, `c/${n}`, repoId, at); |
| 94 | 94 | }; |
| 95 | 95 | merge(1, "2026-03-04T10:00:00.000Z"); |
| 96 | // The landing is kept for a minute by design; skip the wait. | |
| 97 | forgetListings(app.db); | |
| 96 | 98 | const first = await get("/sitemap.xml", { "if-none-match": etagBefore }); |
| 97 | 99 | expect(first.status).toBe(200); |
| 98 | 100 | expect(await first.text()).toContain(`<loc>${o}/changelog</loc><lastmod>2026-03-04</lastmod></url>`); |
| @@ -100,6 +102,7 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day | ||
| 100 | 102 | expect(etagFirst).not.toBe(etagBefore); |
| 101 | 103 | |
| 102 | 104 | merge(2, "2026-03-09T23:00:00.000Z"); |
| 105 | forgetListings(app.db); | |
| 103 | 106 | const second = await get("/sitemap.xml", { "if-none-match": etagFirst }); |
| 104 | 107 | expect(second.status).toBe(200); |
| 105 | 108 | expect(await second.text()).toContain("<lastmod>2026-03-09</lastmod>"); |
new above
offline