Public: anyone can read this; the crew can change it.

/sitemap.xml answers from a cached read: 46 ms of landings becomes under 5 ms #12992

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12988-sitemap-xml-answers-from-a-cached-read-4 → main 3 commits fork

  • Merged. mason/12988-sitemap-xml-answers-from-a-cached-read-4 was merged into main as 2cafde7.

Review

Sign in to review this change.
4 files changed +47 −3
src/core/listings.ts +17 −1
@@ -18,11 +18,12 @@ export const LLMS_LINES = 50;
1818 interface Kept {
1919 readonly sitemap: Fresh<readonly SitemapRepo[]>;
2020 readonly llms: Fresh<LlmsListing>;
21 readonly landed: Fresh<Promise<string | undefined>>;
2122 }
2223 const kept = new WeakMap<Database, Kept>();
2324
2425 /** Test seam: how many times each listing was read from the database. */
25 export const listingBuilds = { sitemap: 0, llms: 0 };
26 export const listingBuilds = { sitemap: 0, llms: 0, landed: 0 };
2627
2728 function holders(db: Database): Kept {
2829 let k = kept.get(db);
@@ -30,6 +31,7 @@ function holders(db: Database): Kept {
3031 k = {
3132 sitemap: new Fresh(LISTING_MS, 1),
3233 llms: new Fresh(LISTING_MS, 1),
34 landed: new Fresh(LISTING_MS, 1),
3335 };
3436 kept.set(db, k);
3537 }
@@ -41,6 +43,20 @@ export function forgetListings(db: Database): void {
4143 const k = kept.get(db);
4244 k?.sitemap.clear();
4345 k?.llms.clear();
46 k?.landed.clear();
47 }
48
49 /**
50 * When the newest landing was merged, read once a minute: the sitemap's
51 * changelog lastmod. `read` is the landings read (it asks git); a failed
52 * read is kept as "none" for the minute, not retried per request. Landings
53 * list only public repositories, so a visibility change drops it with the rest.
54 */
55 export function newestLanding(db: Database, read: () => Promise<string | undefined>): Promise<string | undefined> {
56 return holders(db).landed.get("", () => {
57 listingBuilds.landed++;
58 return read().catch(() => undefined);
59 });
4460 }
4561
4662 /** A listed repository: `owner/name` and its last push (ISO time), or null. */
src/web/standards.ts +4 −2
@@ -38,7 +38,7 @@ import { takesOutside } from "../actions/contribute_token.ts";
3838 import { OFFER_PATCH_MAX, OFFERS_PER_SOURCE } from "../actions/offer.ts";
3939 import { REPORT_LINK_TITLE } from "../actions/report_words.ts";
4040 import { sshTellCommand } from "../ssh/command.ts";
41 import { llmsRepos, sitemapRepos, type LlmsListing } from "../core/listings.ts";
41 import { llmsRepos, newestLanding, sitemapRepos, type LlmsListing } from "../core/listings.ts";
4242 import { canRead, getRepo, type Repo } from "../core/repos.ts";
4343 import { publicForgeRepo } from "../core/forge_repo.ts";
4444 import type { Ctx } from "../http/context.ts";
@@ -269,7 +269,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> {
269269 const repos = timePhase(ctx.request, "repos", () => sitemapRepos(ctx.app.db));
270270 const origin = ctx.url.origin;
271271 // The newest landing's day (UTC), the changelog's lastmod; none when nothing has landed.
272 const newest = await timePhaseAsync(ctx.request, "landings", async () => (await readLandings(ctx.app).catch(() => undefined))?.live[0]?.at);
272 const newest = await timePhaseAsync(ctx.request, "landings", () =>
273 newestLanding(ctx.app.db, async () => (await readLandings(ctx.app))?.live[0]?.at),
274 );
273275 const changed = newest === undefined ? null : dayOf(newest, "UTC");
274276 let kept = sitemapBodies.get(origin);
275277 if (kept === undefined || kept.rows !== repos || kept.changed !== changed) {
test/sitemap-timing.test.ts +23
@@ -50,3 +50,26 @@ test("the sitemap lists /explore exactly once, right after the front page", asyn
5050 expect(locs.filter((l) => l.endsWith("/explore")).length).toBe(1);
5151 expect(body).not.toContain("hidden");
5252 });
53
54 test("the landings read behind the sitemap is made once a minute, a repeat is a 304, and a repository made private drops out at once", async () => {
55 const { listingBuilds, forgetListings } = await import("../src/core/listings.ts");
56 const { getRepo } = await import("../src/core/repos.ts");
57 const { updateRepoSettings } = await import("../src/actions/repos.ts");
58 const alice = (await import("../src/core/users.ts")).getUserByName(app.db, "alice")!;
59 forgetListings(app.db);
60 const first = await fetch(`${servers.url}/sitemap.xml`);
61 const etag = first.headers.get("etag")!;
62 await first.text();
63 const sitemap = listingBuilds.sitemap;
64 const landed = listingBuilds.landed;
65 for (let i = 0; i < 20; i++) await fetch(`${servers.url}/sitemap.xml`).then((r) => r.text());
66 expect(listingBuilds.sitemap).toBe(sitemap);
67 expect(listingBuilds.landed).toBe(landed);
68 const again = await fetch(`${servers.url}/sitemap.xml`, { headers: { "if-none-match": etag } });
69 expect(again.status).toBe(304);
70 const done = await updateRepoSettings(app, alice, getRepo(app.db, "alice", "two")!, { visibility: "private" });
71 if (!done.ok) throw new Error(String(done.error));
72 const after = await (await fetch(`${servers.url}/sitemap.xml`)).text();
73 expect(after).not.toContain("/alice/two</loc>");
74 expect(listingBuilds.landed).toBe(landed + 1);
75 });
test/standards-sitemap.test.ts +3
@@ -93,6 +93,8 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day
9393 app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, '')`).run(`ci${n}`, `c/${n}`, repoId, at);
9494 };
9595 merge(1, "2026-03-04T10:00:00.000Z");
96 // The landing is kept for a minute by design; skip the wait.
97 forgetListings(app.db);
9698 const first = await get("/sitemap.xml", { "if-none-match": etagBefore });
9799 expect(first.status).toBe(200);
98100 expect(await first.text()).toContain(`<loc>${o}/changelog</loc><lastmod>2026-03-04</lastmod></url>`);
@@ -100,6 +102,7 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day
100102 expect(etagFirst).not.toBe(etagBefore);
101103
102104 merge(2, "2026-03-09T23:00:00.000Z");
105 forgetListings(app.db);
103106 const second = await get("/sitemap.xml", { "if-none-match": etagFirst });
104107 expect(second.status).toBe(200);
105108 expect(await second.text()).toContain("<lastmod>2026-03-09</lastmod>");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs