Public: anyone can read this; the crew can change it.

The sitemap lists /explore and every public repository's page #5685

Parked opened this
Closed without merging

scout/forge:scout/4109-the-sitemap-lists-explore-and-every-publ → main 8 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
5 files changed +159 −10
docs/STANDARDS.md +1 −1
@@ -10,7 +10,7 @@ its own mark. Nothing here is configured, and nothing here is optional.
1010 | `/.well-known/security.txt` | who to tell about a vulnerability (RFC 9116), with a fixed expiry | the first account's page here; an address only from `FORGE_SECURITY_CONTACT` |
1111 | `/.well-known/change-password` | where a password manager sends a person | `/settings/password` |
1212 | `/robots.txt` | crawlers stay out of accounts, search and the API; the sitemap | — |
13 | `/sitemap.xml` | the front page and every public repository, never a private one | `listReposVisibleTo(db, nobody)` |
13 | `/sitemap.xml` | `/`, `/explore` and every public repository with its last push as `lastmod` (50,000 URLs at most), never a private one | `sitemapEntries`, built body kept 60 s per origin |
1414 | `/manifest.webmanifest` | the forge as an installable app | — |
1515 | `/llms.txt` | the forge for an agent (llmstxt.org): how to report, how to offer a change, the API and MCP, the public repositories | `listReposVisibleTo(db, nobody)`, the report limits |
1616 | `/icons/forge-<size>.png` | the mark at 192, 512, 512-maskable and 180 (iOS), painted at boot | `design/mark.ts`, the favicon's two colours |
src/core/listings.ts +31 −5
@@ -9,13 +9,22 @@
99 import type { Database } from "bun:sqlite";
1010 import { Fresh } from "../kernel/fresh.ts";
1111 import { listReposVisibleTo } from "./repos.ts";
12 import { PUBLIC_VISIBLE } from "./visibility.ts";
1213
1314 const LISTING_MS = 60_000;
14 export const SITEMAP_ROWS = 5000;
15 /** The sitemap protocol allows 50,000 URLs; `/` and `/explore` take two. */
16 export const SITEMAP_URLS = 50_000;
17 export const SITEMAP_ROWS = SITEMAP_URLS - 2;
1518 export const LLMS_ROWS = 200;
1619
20 /** One repository's sitemap entry: `owner/name` and its last push, if any. */
21 export interface SitemapRepo {
22 readonly path: string;
23 readonly lastmod: string | null;
24 }
25
1726 interface Kept {
18 readonly sitemap: Fresh<readonly string[]>;
27 readonly sitemap: Fresh<readonly SitemapRepo[]>;
1928 readonly llms: Fresh<readonly string[]>;
2029 }
2130 const kept = new WeakMap<Database, Kept>();
@@ -42,14 +51,31 @@ export function forgetListings(db: Database): void {
4251 k?.llms.clear();
4352 }
4453
45 /** `owner/name` of the newest public repositories, at most `SITEMAP_ROWS`. */
46 export function sitemapRepos(db: Database): readonly string[] {
54 /**
55 * The newest public, unarchived repositories, at most `SITEMAP_ROWS`, each
56 * with the time of its last push (a repository never pushed to has none).
57 */
58 export function sitemapRepos(db: Database): readonly SitemapRepo[] {
4759 return holders(db).sitemap.get("", () => {
4860 listingBuilds.sitemap++;
49 return names(db, SITEMAP_ROWS);
61 return sitemapEntries(db, SITEMAP_ROWS);
5062 });
5163 }
5264
65 /** `sitemapRepos` without the cache, at most `limit` rows. */
66 export function sitemapEntries(db: Database, limit: number): SitemapRepo[] {
67 return db
68 .query<{ path: string; lastmod: string | null }, [number]>(
69 `SELECT user.username || '/' || repo.name AS path,
70 (SELECT MAX(event.created_at) FROM event
71 WHERE event.repo_id = repo.id AND event.kind = 'push') AS lastmod
72 FROM repo JOIN user ON user.id = repo.owner_id
73 WHERE ${PUBLIC_VISIBLE} AND repo.archived_at IS NULL
74 ORDER BY repo.id DESC LIMIT ?1`,
75 )
76 .all(limit);
77 }
78
5379 /** `owner/name` of the newest public repositories, at most `LLMS_ROWS`. */
5480 export function llmsRepos(db: Database): readonly string[] {
5581 return holders(db).llms.get("", () => {
src/core/migrations/20261022120000-event-last-push.ts added +6
@@ -0,0 +1,6 @@
1 // The sitemap asks each public repository for its newest push
2 // (`core/listings.ts`): with the kind and the time in the index that is one
3 // seek per repository, not a read of all its events. Additive.
4 export default `
5 CREATE INDEX event_repo_kind_time ON event(repo_id, kind, created_at);
6 `;
src/web/standards.ts +7 −3
@@ -5,7 +5,7 @@
55 * /.well-known/security.txt who to tell about a vulnerability (RFC 9116)
66 * /.well-known/change-password where a password manager sends someone
77 * /robots.txt what a crawler may index, and the sitemap
8 * /sitemap.xml every public repository
8 * /sitemap.xml the front page, the explorer, every public repository
99 * /releases.json the releases this forge has shipped, newest
1010 * first, for a child forge to read
1111 * /manifest.webmanifest the forge as an installable app
@@ -162,7 +162,7 @@ function xml(s: string): string {
162162 );
163163 }
164164
165 /** GET /sitemap.xml — the front page and every public repository. */
165 /** GET /sitemap.xml — the front page, the explorer and every public repository. */
166166 export function sitemapXml(ctx: Ctx): Response {
167167 // Kept without an origin, and the origin is put on per answer, so a
168168 // caller who varies `Host` cannot make the forge read the rows again.
@@ -173,7 +173,11 @@ export function sitemapXml(ctx: Ctx): Response {
173173 const origin = xml(ctx.url.origin);
174174 const urls = [
175175 `<url><loc>${origin}/</loc></url>`,
176 ...repos.map((r) => `<url><loc>${origin}${xml(`/${r}`)}</loc></url>`),
176 `<url><loc>${origin}/explore</loc></url>`,
177 ...repos.map(
178 (r) =>
179 `<url><loc>${origin}${xml(`/${r.path}`)}</loc>${r.lastmod === null ? "" : `<lastmod>${xml(r.lastmod)}</lastmod>`}</url>`,
180 ),
177181 ];
178182 return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`;
179183 });
test/standards.test.ts +114 −1
@@ -17,7 +17,12 @@ import { createRepo, updateRepoSettings } from "../src/actions/repos.ts";
1717 import { decodePng, pngHeader } from "../src/design/image/png.ts";
1818 import { markPng } from "../src/design/mark.ts";
1919 import { createUser, type User } from "../src/core/users.ts";
20 import { forgetListings, listingBuilds } from "../src/core/listings.ts";
20 import {
21 forgetListings,
22 listingBuilds,
23 SITEMAP_ROWS,
24 sitemapEntries,
25 } from "../src/core/listings.ts";
2126 import { createOrg, retireOrg } from "../src/core/orgs.ts";
2227 import { getRepo, moveArchived } from "../src/core/repos.ts";
2328 import { serve, type Servers } from "../src/http/server.ts";
@@ -142,6 +147,49 @@ describe("what the open web expects", () => {
142147 expect(body).not.toContain("alice/closed");
143148 });
144149
150 test("the sitemap lists /explore, and a pushed repository carries its last push as lastmod", async () => {
151 const r = await createRepo(app, alice, {
152 name: "stamped",
153 visibility: "public",
154 });
155 if (!r.ok) throw new Error(String(r.error));
156 const repo = getRepo(app.db, "alice", "stamped")!;
157 app.db
158 .query(
159 "INSERT INTO event (id, kind, repo_id, created_at) VALUES ('e-sm-1', 'push', ?1, '2026-03-04T05:06:07.000Z')",
160 )
161 .run(repo.id);
162 forgetListings(app.db);
163 const body = await fetch(`${base}/sitemap.xml`).then((x) => x.text());
164 expect(body).toContain(`<loc>${base}/explore</loc>`);
165 expect(body).toContain(
166 `<loc>${base}/alice/stamped</loc><lastmod>2026-03-04T05:06:07.000Z</lastmod>`,
167 );
168 expect(body).not.toContain("/explore?");
169 });
170
171 test("an archived repository is not in the sitemap, and the entry list honours its cap", async () => {
172 const r = await createRepo(app, alice, {
173 name: "boxed",
174 visibility: "public",
175 });
176 if (!r.ok) throw new Error(String(r.error));
177 expect(sitemapEntries(app.db, 1000).map((e) => e.path)).toContain(
178 "alice/boxed",
179 );
180 app.db
181 .query(
182 "UPDATE repo SET archived_at = '2026-01-01T00:00:00.000Z' WHERE name = 'boxed'",
183 )
184 .run();
185 forgetListings(app.db);
186 const body = await fetch(`${base}/sitemap.xml`).then((x) => x.text());
187 expect(body).not.toContain("alice/boxed");
188 expect(sitemapEntries(app.db, 1)).toHaveLength(1);
189 expect(sitemapEntries(app.db, 2)).toHaveLength(2);
190 expect(SITEMAP_ROWS + 2).toBe(50_000);
191 });
192
145193 test("sitemap and llms.txt are each built once for 50 requests, and a repository made private drops out at once", async () => {
146194 const mid = await createRepo(app, alice, {
147195 name: "midway",
@@ -323,3 +371,68 @@ describe("what the open web expects", () => {
323371 expect(csp).toMatch(/img-src [^;]*'self'/);
324372 });
325373 });
374
375 describe("the sitemap of a forge with a great many repositories", () => {
376 let dataDir: string;
377 let app: App;
378 let servers: Servers;
379
380 beforeAll(async () => {
381 dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-big-"));
382 app = createApp({ dataDir, port: 0 });
383 servers = await serve(app);
384 const a = await createUser(app.db, {
385 origin: "seed",
386 trusted: true,
387 username: "carol",
388 email: "[email protected]",
389 password: "a-good-passphrase",
390 adminIfFirst: true,
391 });
392 if (!a.ok) throw new Error(a.error);
393 // 50,100 public repositories with 6 events each, in SQL.
394 app.db.run(`
395 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 50100)
396 INSERT INTO repo (id, owner_id, name, visibility)
397 SELECT 'big' || i, (SELECT id FROM user LIMIT 1), 'r' || i, 'public' FROM n`);
398 app.db.run(`
399 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 6)
400 INSERT INTO event (id, kind, repo_id, created_at)
401 SELECT r.id || '-' || n.i, CASE WHEN n.i % 2 = 0 THEN 'push' ELSE 'issue.opened' END,
402 r.id, '2026-01-01T00:00:' || printf('%02d', n.i) || '.000Z'
403 FROM repo r, n WHERE r.id LIKE 'big%'`);
404 forgetListings(app.db);
405 });
406
407 afterAll(async () => {
408 servers.stop();
409 app.close();
410 await rm(dataDir, { recursive: true, force: true });
411 });
412
413 test("the last push is one index seek per repository", () => {
414 const plan = app.db
415 .query<{ detail: string }, []>(
416 `EXPLAIN QUERY PLAN SELECT MAX(event.created_at) FROM event
417 WHERE event.repo_id = 'big1' AND event.kind = 'push'`,
418 )
419 .all()
420 .map((r) => r.detail)
421 .join(" ");
422 expect(plan).toContain("event_repo_kind_time");
423 });
424
425 test("the cap holds and the built body is kept", async () => {
426 const entries = sitemapEntries(app.db, SITEMAP_ROWS);
427 expect(entries).toHaveLength(SITEMAP_ROWS);
428 expect(entries[0]!.lastmod).toBe("2026-01-01T00:00:06.000Z");
429
430 const url = servers.url;
431 const body = await fetch(`${url}/sitemap.xml`).then((r) => r.text());
432 expect(body.match(/<url>/g)).toHaveLength(50_000);
433 const before = listingBuilds.sitemap;
434 const again = await fetch(`${url}/sitemap.xml`).then((r) => r.text());
435 expect(again).toBe(body);
436 expect(listingBuilds.sitemap).toBe(before);
437 });
438 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs