Public: anyone can read this; the crew can change it.
The sitemap lists /explore and every public repository's page #5689
scout/forge:scout/4109-the-sitemap-lists-explore-and-every-publ → main 8 commits fork
5 files
changed +159
−10
docs/STANDARDS.md +1 −1
| @@ -10,7 +10,7 @@ its own mark. Nothing here is configured, and nothing here is optional. | ||
| 10 | 10 | | `/.well-known/security.txt` | who to tell about a vulnerability (RFC 9116), with a fixed expiry | the first account's page here; an address only from `FORGE_SECURITY_CONTACT` | |
| 11 | 11 | | `/.well-known/change-password` | where a password manager sends a person | `/settings/password` | |
| 12 | 12 | | `/robots.txt` | crawlers stay out of accounts, search and the API; the sitemap | — | |
| 13 | | `/sitemap.xml` | the front page and every public repository, never a private one | `listReposVisibleTo(db, nobody)` | | |
| 13 | | `/sitemap.xml` | `/`, `/explore` and every public repository with its last push as `lastmod` (50,000 URLs at most), never a private one | `sitemapEntries`, built body kept 60 s per origin | | |
| 14 | 14 | | `/manifest.webmanifest` | the forge as an installable app | — | |
| 15 | 15 | | `/llms.txt` | the forge for an agent (llmstxt.org): how to report, how to offer a change, the API and MCP, the public repositories | `listReposVisibleTo(db, nobody)`, the report limits | |
| 16 | 16 | | `/icons/forge-<size>.png` | the mark at 192, 512, 512-maskable and 180 (iOS), painted at boot | `design/mark.ts`, the favicon's two colours | |
src/core/listings.ts +31 −5
| @@ -9,13 +9,22 @@ | ||
| 9 | 9 | import type { Database } from "bun:sqlite"; |
| 10 | 10 | import { Fresh } from "../kernel/fresh.ts"; |
| 11 | 11 | import { listReposVisibleTo } from "./repos.ts"; |
| 12 | import { PUBLIC_VISIBLE } from "./visibility.ts"; | |
| 12 | 13 | |
| 13 | 14 | const LISTING_MS = 60_000; |
| 14 | export const SITEMAP_ROWS = 5000; | |
| 15 | /** The sitemap protocol allows 50,000 URLs; `/` and `/explore` take two. */ | |
| 16 | export const SITEMAP_URLS = 50_000; | |
| 17 | export const SITEMAP_ROWS = SITEMAP_URLS - 2; | |
| 15 | 18 | export const LLMS_ROWS = 200; |
| 16 | 19 | |
| 20 | /** One repository's sitemap entry: `owner/name` and its last push, if any. */ | |
| 21 | export interface SitemapRepo { | |
| 22 | readonly path: string; | |
| 23 | readonly lastmod: string | null; | |
| 24 | } | |
| 25 | ||
| 17 | 26 | interface Kept { |
| 18 | readonly sitemap: Fresh<readonly string[]>; | |
| 27 | readonly sitemap: Fresh<readonly SitemapRepo[]>; | |
| 19 | 28 | readonly llms: Fresh<readonly string[]>; |
| 20 | 29 | } |
| 21 | 30 | const kept = new WeakMap<Database, Kept>(); |
| @@ -42,14 +51,31 @@ export function forgetListings(db: Database): void { | ||
| 42 | 51 | k?.llms.clear(); |
| 43 | 52 | } |
| 44 | 53 | |
| 45 | /** `owner/name` of the newest public repositories, at most `SITEMAP_ROWS`. */ | |
| 46 | export function sitemapRepos(db: Database): readonly string[] { | |
| 54 | /** | |
| 55 | * The newest public, unarchived repositories, at most `SITEMAP_ROWS`, each | |
| 56 | * with the time of its last push (a repository never pushed to has none). | |
| 57 | */ | |
| 58 | export function sitemapRepos(db: Database): readonly SitemapRepo[] { | |
| 47 | 59 | return holders(db).sitemap.get("", () => { |
| 48 | 60 | listingBuilds.sitemap++; |
| 49 | return names(db, SITEMAP_ROWS); | |
| 61 | return sitemapEntries(db, SITEMAP_ROWS); | |
| 50 | 62 | }); |
| 51 | 63 | } |
| 52 | 64 | |
| 65 | /** `sitemapRepos` without the cache, at most `limit` rows. */ | |
| 66 | export function sitemapEntries(db: Database, limit: number): SitemapRepo[] { | |
| 67 | return db | |
| 68 | .query<{ path: string; lastmod: string | null }, [number]>( | |
| 69 | `SELECT user.username || '/' || repo.name AS path, | |
| 70 | (SELECT MAX(event.created_at) FROM event | |
| 71 | WHERE event.repo_id = repo.id AND event.kind = 'push') AS lastmod | |
| 72 | FROM repo JOIN user ON user.id = repo.owner_id | |
| 73 | WHERE ${PUBLIC_VISIBLE} AND repo.archived_at IS NULL | |
| 74 | ORDER BY repo.id DESC LIMIT ?1`, | |
| 75 | ) | |
| 76 | .all(limit); | |
| 77 | } | |
| 78 | ||
| 53 | 79 | /** `owner/name` of the newest public repositories, at most `LLMS_ROWS`. */ |
| 54 | 80 | export function llmsRepos(db: Database): readonly string[] { |
| 55 | 81 | return holders(db).llms.get("", () => { |
src/core/migrations/20261022120000-event-last-push.ts added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | // The sitemap asks each public repository for its newest push | |
| 2 | // (`core/listings.ts`): with the kind and the time in the index that is one | |
| 3 | // seek per repository, not a read of all its events. Additive. | |
| 4 | export default ` | |
| 5 | CREATE INDEX event_repo_kind_time ON event(repo_id, kind, created_at); | |
| 6 | `; |
src/web/standards.ts +7 −3
| @@ -5,7 +5,7 @@ | ||
| 5 | 5 | * /.well-known/security.txt who to tell about a vulnerability (RFC 9116) |
| 6 | 6 | * /.well-known/change-password where a password manager sends someone |
| 7 | 7 | * /robots.txt what a crawler may index, and the sitemap |
| 8 | * /sitemap.xml every public repository | |
| 8 | * /sitemap.xml the front page, the explorer, every public repository | |
| 9 | 9 | * /releases.json the releases this forge has shipped, newest |
| 10 | 10 | * first, for a child forge to read |
| 11 | 11 | * /manifest.webmanifest the forge as an installable app |
| @@ -162,7 +162,7 @@ function xml(s: string): string { | ||
| 162 | 162 | ); |
| 163 | 163 | } |
| 164 | 164 | |
| 165 | /** GET /sitemap.xml — the front page and every public repository. */ | |
| 165 | /** GET /sitemap.xml — the front page, the explorer and every public repository. */ | |
| 166 | 166 | export function sitemapXml(ctx: Ctx): Response { |
| 167 | 167 | // Kept without an origin, and the origin is put on per answer, so a |
| 168 | 168 | // caller who varies `Host` cannot make the forge read the rows again. |
| @@ -173,7 +173,11 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 173 | 173 | const origin = xml(ctx.url.origin); |
| 174 | 174 | const urls = [ |
| 175 | 175 | `<url><loc>${origin}/</loc></url>`, |
| 176 | ...repos.map((r) => `<url><loc>${origin}${xml(`/${r}`)}</loc></url>`), | |
| 176 | `<url><loc>${origin}/explore</loc></url>`, | |
| 177 | ...repos.map( | |
| 178 | (r) => | |
| 179 | `<url><loc>${origin}${xml(`/${r.path}`)}</loc>${r.lastmod === null ? "" : `<lastmod>${xml(r.lastmod)}</lastmod>`}</url>`, | |
| 180 | ), | |
| 177 | 181 | ]; |
| 178 | 182 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; |
| 179 | 183 | }); |
test/standards.test.ts +114 −1
| @@ -17,7 +17,12 @@ import { createRepo, updateRepoSettings } from "../src/actions/repos.ts"; | ||
| 17 | 17 | import { decodePng, pngHeader } from "../src/design/image/png.ts"; |
| 18 | 18 | import { markPng } from "../src/design/mark.ts"; |
| 19 | 19 | import { createUser, type User } from "../src/core/users.ts"; |
| 20 | import { forgetListings, listingBuilds } from "../src/core/listings.ts"; | |
| 20 | import { | |
| 21 | forgetListings, | |
| 22 | listingBuilds, | |
| 23 | SITEMAP_ROWS, | |
| 24 | sitemapEntries, | |
| 25 | } from "../src/core/listings.ts"; | |
| 21 | 26 | import { createOrg, retireOrg } from "../src/core/orgs.ts"; |
| 22 | 27 | import { getRepo, moveArchived } from "../src/core/repos.ts"; |
| 23 | 28 | import { serve, type Servers } from "../src/http/server.ts"; |
| @@ -142,6 +147,49 @@ describe("what the open web expects", () => { | ||
| 142 | 147 | expect(body).not.toContain("alice/closed"); |
| 143 | 148 | }); |
| 144 | 149 | |
| 150 | test("the sitemap lists /explore, and a pushed repository carries its last push as lastmod", async () => { | |
| 151 | const r = await createRepo(app, alice, { | |
| 152 | name: "stamped", | |
| 153 | visibility: "public", | |
| 154 | }); | |
| 155 | if (!r.ok) throw new Error(String(r.error)); | |
| 156 | const repo = getRepo(app.db, "alice", "stamped")!; | |
| 157 | app.db | |
| 158 | .query( | |
| 159 | "INSERT INTO event (id, kind, repo_id, created_at) VALUES ('e-sm-1', 'push', ?1, '2026-03-04T05:06:07.000Z')", | |
| 160 | ) | |
| 161 | .run(repo.id); | |
| 162 | forgetListings(app.db); | |
| 163 | const body = await fetch(`${base}/sitemap.xml`).then((x) => x.text()); | |
| 164 | expect(body).toContain(`<loc>${base}/explore</loc>`); | |
| 165 | expect(body).toContain( | |
| 166 | `<loc>${base}/alice/stamped</loc><lastmod>2026-03-04T05:06:07.000Z</lastmod>`, | |
| 167 | ); | |
| 168 | expect(body).not.toContain("/explore?"); | |
| 169 | }); | |
| 170 | ||
| 171 | test("an archived repository is not in the sitemap, and the entry list honours its cap", async () => { | |
| 172 | const r = await createRepo(app, alice, { | |
| 173 | name: "boxed", | |
| 174 | visibility: "public", | |
| 175 | }); | |
| 176 | if (!r.ok) throw new Error(String(r.error)); | |
| 177 | expect(sitemapEntries(app.db, 1000).map((e) => e.path)).toContain( | |
| 178 | "alice/boxed", | |
| 179 | ); | |
| 180 | app.db | |
| 181 | .query( | |
| 182 | "UPDATE repo SET archived_at = '2026-01-01T00:00:00.000Z' WHERE name = 'boxed'", | |
| 183 | ) | |
| 184 | .run(); | |
| 185 | forgetListings(app.db); | |
| 186 | const body = await fetch(`${base}/sitemap.xml`).then((x) => x.text()); | |
| 187 | expect(body).not.toContain("alice/boxed"); | |
| 188 | expect(sitemapEntries(app.db, 1)).toHaveLength(1); | |
| 189 | expect(sitemapEntries(app.db, 2)).toHaveLength(2); | |
| 190 | expect(SITEMAP_ROWS + 2).toBe(50_000); | |
| 191 | }); | |
| 192 | ||
| 145 | 193 | test("sitemap and llms.txt are each built once for 50 requests, and a repository made private drops out at once", async () => { |
| 146 | 194 | const mid = await createRepo(app, alice, { |
| 147 | 195 | name: "midway", |
| @@ -323,3 +371,68 @@ describe("what the open web expects", () => { | ||
| 323 | 371 | expect(csp).toMatch(/img-src [^;]*'self'/); |
| 324 | 372 | }); |
| 325 | 373 | }); |
| 374 | ||
| 375 | describe("the sitemap of a forge with a great many repositories", () => { | |
| 376 | let dataDir: string; | |
| 377 | let app: App; | |
| 378 | let servers: Servers; | |
| 379 | ||
| 380 | beforeAll(async () => { | |
| 381 | dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-big-")); | |
| 382 | app = createApp({ dataDir, port: 0 }); | |
| 383 | servers = await serve(app); | |
| 384 | const a = await createUser(app.db, { | |
| 385 | origin: "seed", | |
| 386 | trusted: true, | |
| 387 | username: "carol", | |
| 388 | email: "[email protected]", | |
| 389 | password: "a-good-passphrase", | |
| 390 | adminIfFirst: true, | |
| 391 | }); | |
| 392 | if (!a.ok) throw new Error(a.error); | |
| 393 | // 50,100 public repositories with 6 events each, in SQL. | |
| 394 | app.db.run(` | |
| 395 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 50100) | |
| 396 | INSERT INTO repo (id, owner_id, name, visibility) | |
| 397 | SELECT 'big' || i, (SELECT id FROM user LIMIT 1), 'r' || i, 'public' FROM n`); | |
| 398 | app.db.run(` | |
| 399 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 6) | |
| 400 | INSERT INTO event (id, kind, repo_id, created_at) | |
| 401 | SELECT r.id || '-' || n.i, CASE WHEN n.i % 2 = 0 THEN 'push' ELSE 'issue.opened' END, | |
| 402 | r.id, '2026-01-01T00:00:' || printf('%02d', n.i) || '.000Z' | |
| 403 | FROM repo r, n WHERE r.id LIKE 'big%'`); | |
| 404 | forgetListings(app.db); | |
| 405 | }); | |
| 406 | ||
| 407 | afterAll(async () => { | |
| 408 | servers.stop(); | |
| 409 | app.close(); | |
| 410 | await rm(dataDir, { recursive: true, force: true }); | |
| 411 | }); | |
| 412 | ||
| 413 | test("the last push is one index seek per repository", () => { | |
| 414 | const plan = app.db | |
| 415 | .query<{ detail: string }, []>( | |
| 416 | `EXPLAIN QUERY PLAN SELECT MAX(event.created_at) FROM event | |
| 417 | WHERE event.repo_id = 'big1' AND event.kind = 'push'`, | |
| 418 | ) | |
| 419 | .all() | |
| 420 | .map((r) => r.detail) | |
| 421 | .join(" "); | |
| 422 | expect(plan).toContain("event_repo_kind_time"); | |
| 423 | }); | |
| 424 | ||
| 425 | test("the cap holds and the built body is kept", async () => { | |
| 426 | const entries = sitemapEntries(app.db, SITEMAP_ROWS); | |
| 427 | expect(entries).toHaveLength(SITEMAP_ROWS); | |
| 428 | expect(entries[0]!.lastmod).toBe("2026-01-01T00:00:06.000Z"); | |
| 429 | ||
| 430 | const url = servers.url; | |
| 431 | const body = await fetch(`${url}/sitemap.xml`).then((r) => r.text()); | |
| 432 | expect(body.match(/<url>/g)).toHaveLength(50_000); | |
| 433 | const before = listingBuilds.sitemap; | |
| 434 | const again = await fetch(`${url}/sitemap.xml`).then((r) => r.text()); | |
| 435 | expect(again).toBe(body); | |
| 436 | expect(listingBuilds.sitemap).toBe(before); | |
| 437 | }); | |
| 438 | }); |
new above
offline