Public: anyone can read this; the crew can change it.
sitemap.xml and llms.txt are built at most once a minute, never once per stranger's request #3149
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3142-sitemap-xml-and-llms-txt-are-built-at-mo → main 4 commits fork 3 threads · 3 unresolved
7 files
changed +167
−35
src/core/accounts.ts +3
| @@ -12,6 +12,7 @@ | ||
| 12 | 12 | */ |
| 13 | 13 | |
| 14 | 14 | import type { Database } from "bun:sqlite"; |
| 15 | import { forgetListings } from "./listings.ts"; | |
| 15 | 16 | |
| 16 | 17 | export interface AccountRow { |
| 17 | 18 | readonly username: string; |
| @@ -138,6 +139,7 @@ export function setSuspended( | ||
| 138 | 139 | at: string | null, |
| 139 | 140 | ): void { |
| 140 | 141 | db.query("UPDATE user SET suspended_at = ?1 WHERE id = ?2").run(at, userId); |
| 142 | forgetListings(db); | |
| 141 | 143 | } |
| 142 | 144 | |
| 143 | 145 | /** Every way the account could prove it is itself, gone. */ |
| @@ -175,4 +177,5 @@ export function scrubAccount(db: Database, userId: string, at: string): void { | ||
| 175 | 177 | suspended_at = COALESCE(suspended_at, ?1), removed_at = ?1 |
| 176 | 178 | WHERE id = ?2`, |
| 177 | 179 | ).run(at, userId); |
| 180 | forgetListings(db); | |
| 178 | 181 | } |
src/core/listings.ts added +65
| @@ -0,0 +1,65 @@ | ||
| 1 | /** | |
| 2 | * The public repositories as sitemap.xml and llms.txt list them, read once a | |
| 3 | * minute however many ask. Kept here, not with the pages, because whatever | |
| 4 | * changes who may be listed — a repository made private or deleted, an | |
| 5 | * account suspended or removed, an organization retired — says so at its | |
| 6 | * own write (`forgetListings`), so no caller can forget to. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { Fresh } from "../kernel/fresh.ts"; | |
| 11 | import { listReposVisibleTo } from "./repos.ts"; | |
| 12 | ||
| 13 | const LISTING_MS = 60_000; | |
| 14 | export const SITEMAP_ROWS = 5000; | |
| 15 | export const LLMS_ROWS = 200; | |
| 16 | ||
| 17 | interface Kept { | |
| 18 | readonly sitemap: Fresh<readonly string[]>; | |
| 19 | readonly llms: Fresh<readonly string[]>; | |
| 20 | } | |
| 21 | const kept = new WeakMap<Database, Kept>(); | |
| 22 | ||
| 23 | /** Test seam: how many times each listing was read from the database. */ | |
| 24 | export const listingBuilds = { sitemap: 0, llms: 0 }; | |
| 25 | ||
| 26 | function holders(db: Database): Kept { | |
| 27 | let k = kept.get(db); | |
| 28 | if (k === undefined) { | |
| 29 | k = { | |
| 30 | sitemap: new Fresh(LISTING_MS, 1), | |
| 31 | llms: new Fresh(LISTING_MS, 1), | |
| 32 | }; | |
| 33 | kept.set(db, k); | |
| 34 | } | |
| 35 | return k; | |
| 36 | } | |
| 37 | ||
| 38 | /** Forget the kept listings: the next ask reads them again. */ | |
| 39 | export function forgetListings(db: Database): void { | |
| 40 | const k = kept.get(db); | |
| 41 | k?.sitemap.clear(); | |
| 42 | k?.llms.clear(); | |
| 43 | } | |
| 44 | ||
| 45 | /** `owner/name` of the newest public repositories, at most `SITEMAP_ROWS`. */ | |
| 46 | export function sitemapRepos(db: Database): readonly string[] { | |
| 47 | return holders(db).sitemap.get("", () => { | |
| 48 | listingBuilds.sitemap++; | |
| 49 | return names(db, SITEMAP_ROWS); | |
| 50 | }); | |
| 51 | } | |
| 52 | ||
| 53 | /** `owner/name` of the newest public repositories, at most `LLMS_ROWS`. */ | |
| 54 | export function llmsRepos(db: Database): readonly string[] { | |
| 55 | return holders(db).llms.get("", () => { | |
| 56 | listingBuilds.llms++; | |
| 57 | return names(db, LLMS_ROWS); | |
| 58 | }); | |
| 59 | } | |
| 60 | ||
| 61 | function names(db: Database, limit: number): string[] { | |
| 62 | return listReposVisibleTo(db, undefined, { limit }).map( | |
| 63 | (r) => `${r.ownerName}/${r.name}`, | |
| 64 | ); | |
| 65 | } |
src/core/orgs.ts +2
| @@ -10,6 +10,7 @@ import type { Database } from "bun:sqlite"; | ||
| 10 | 10 | import { isSafeName } from "../git/guard.ts"; |
| 11 | 11 | import { id } from "../kernel/id.ts"; |
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | import { forgetListings } from "./listings.ts"; | |
| 13 | 14 | import { reservedName } from "./reserved.ts"; |
| 14 | 15 | import { getUserById, getUserByName, type User } from "./users.ts"; |
| 15 | 16 | |
| @@ -243,6 +244,7 @@ export function retireOrg(db: Database, orgId: string, at: string): void { | ||
| 243 | 244 | "UPDATE user SET removed_at = ?1 WHERE id = ?2 AND kind = 'org'", |
| 244 | 245 | ).run(at, orgId); |
| 245 | 246 | })(); |
| 247 | forgetListings(db); | |
| 246 | 248 | } |
| 247 | 249 | |
| 248 | 250 | /** The organizations `userId` is the last active person owning: removing them would orphan these. */ |
src/core/repos.ts +3
| @@ -21,6 +21,7 @@ import { id } from "../kernel/id.ts"; | ||
| 21 | 21 | import { createLog } from "../kernel/log.ts"; |
| 22 | 22 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 23 | 23 | import { appsNaming, isAppMember } from "./app_members.ts"; |
| 24 | import { forgetListings } from "./listings.ts"; | |
| 24 | 25 | import { likeEscape, pageLimit } from "./issues.ts"; |
| 25 | 26 | import { |
| 26 | 27 | OWNER_STANDING, |
| @@ -787,6 +788,7 @@ export async function deleteRepo( | ||
| 787 | 788 | } |
| 788 | 789 | |
| 789 | 790 | db.query("DELETE FROM repo WHERE id = ?1").run(repo.id); |
| 791 | forgetListings(db); | |
| 790 | 792 | const dir = repoDir(reposRoot, repo.ownerName, repo.name); |
| 791 | 793 | // A pooled `cat-file` holds this directory open. Closing it first is what |
| 792 | 794 | // keeps the space actually free after the unlink instead of pinned to a |
| @@ -1093,6 +1095,7 @@ export function updateRepoSettings( | ||
| 1093 | 1095 | settings.decisionsBy ?? repo.decisionsBy, |
| 1094 | 1096 | (settings.plansNeedNod ?? repo.plansNeedNod) ? 1 : 0, |
| 1095 | 1097 | ); |
| 1098 | forgetListings(db); | |
| 1096 | 1099 | return getRepoById(db, repo.id)!; |
| 1097 | 1100 | } |
| 1098 | 1101 |
src/origins/sync.ts +2
| @@ -23,6 +23,7 @@ | ||
| 23 | 23 | */ |
| 24 | 24 | |
| 25 | 25 | import { existsSync } from "node:fs"; |
| 26 | import { forgetListings } from "../core/listings.ts"; | |
| 26 | 27 | import type { App } from "../app.ts"; |
| 27 | 28 | import { |
| 28 | 29 | cacheOriginRepos, |
| @@ -310,6 +311,7 @@ async function syncMirrorUnguarded( | ||
| 310 | 311 | app.db |
| 311 | 312 | .query("UPDATE repo SET visibility = ?1 WHERE id = ?2") |
| 312 | 313 | .run(wanted, repo.id); |
| 314 | forgetListings(app.db); | |
| 313 | 315 | app.bus.append({ |
| 314 | 316 | kind: "repo.settings_updated", |
| 315 | 317 | repoId: repo.id, |
src/web/standards.ts +7 −34
| @@ -23,9 +23,9 @@ | ||
| 23 | 23 | import releases from "../core/releases.json" with { type: "json" }; |
| 24 | 24 | import { markPng, type MarkShape } from "../design/mark.ts"; |
| 25 | 25 | import { BODY_MAX, TITLE_MAX } from "../core/reports.ts"; |
| 26 | import { canRead, getRepo, listReposVisibleTo } from "../core/repos.ts"; | |
| 26 | import { llmsRepos, sitemapRepos } from "../core/listings.ts"; | |
| 27 | import { canRead, getRepo } from "../core/repos.ts"; | |
| 27 | 28 | import type { App } from "../app.ts"; |
| 28 | import { Fresh } from "../kernel/fresh.ts"; | |
| 29 | 29 | import type { Ctx } from "../http/context.ts"; |
| 30 | 30 | import { notFound } from "../http/respond.ts"; |
| 31 | 31 | |
| @@ -140,24 +140,11 @@ function xml(s: string): string { | ||
| 140 | 140 | ); |
| 141 | 141 | } |
| 142 | 142 | |
| 143 | /** | |
| 144 | * A sitemap is up to five thousand rows, asked for by anyone: its rows are | |
| 145 | * read once a minute, however many ask. They are kept without an origin, | |
| 146 | * under one key, and the origin is put on per answer — so a caller who | |
| 147 | * varies `Host` (where `FORGE_URL` does not fix the origin) gets their own | |
| 148 | * origin on the same rows and cannot make the forge read them again. | |
| 149 | */ | |
| 150 | const SITEMAP_MS = 60_000; | |
| 151 | const sitemaps = new WeakMap<App, Fresh<readonly string[]>>(); | |
| 152 | ||
| 153 | 143 | /** GET /sitemap.xml — the front page and every public repository. */ |
| 154 | 144 | export function sitemapXml(ctx: Ctx): Response { |
| 155 | let kept = sitemaps.get(ctx.app); | |
| 156 | if (kept === undefined) { | |
| 157 | kept = new Fresh(SITEMAP_MS, 1); | |
| 158 | sitemaps.set(ctx.app, kept); | |
| 159 | } | |
| 160 | const paths = kept.get("", () => sitemapPaths(ctx.app)); | |
| 145 | // Kept without an origin, and the origin is put on per answer, so a | |
| 146 | // caller who varies `Host` cannot make the forge read the rows again. | |
| 147 | const paths = sitemapRepos(ctx.app.db).map((r) => xml(`/${r}`)); | |
| 161 | 148 | // Escaping is per character, so the escaped origin and the escaped path |
| 162 | 149 | // join into the escaped URL. |
| 163 | 150 | const origin = xml(ctx.url.origin); |
| @@ -171,13 +158,6 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 171 | 158 | ); |
| 172 | 159 | } |
| 173 | 160 | |
| 174 | /** Every public repository's path, escaped for XML, newest first. */ | |
| 175 | function sitemapPaths(app: App): readonly string[] { | |
| 176 | return listReposVisibleTo(app.db, undefined, { limit: 5000 }).map((r) => | |
| 177 | xml(`/${r.ownerName}/${r.name}`), | |
| 178 | ); | |
| 179 | } | |
| 180 | ||
| 181 | 161 | /** GET /manifest.webmanifest — the forge as an app on a home screen. */ |
| 182 | 162 | export function webManifest(ctx: Ctx): Response { |
| 183 | 163 | const name = ctx.app.config.edgeName ? "forge" : "forge"; |
| @@ -245,9 +225,6 @@ export function iconPng(ctx: Ctx): Response { | ||
| 245 | 225 | }); |
| 246 | 226 | } |
| 247 | 227 | |
| 248 | /** Public repositories `/llms.txt` lists by name; the sitemap has them all. */ | |
| 249 | const LLMS_REPOS = 200; | |
| 250 | ||
| 251 | 228 | /** |
| 252 | 229 | * GET /llms.txt — the forge as an agent should meet it (llmstxt.org): what |
outdated
Orphaned comment: the constant it documented (LLMS_REPOS) moved to listings.ts. Delete it. |
||
| 253 | 230 | * this is, how to report a problem with no account, how to offer a change, |
| @@ -256,9 +233,7 @@ const LLMS_REPOS = 200; | ||
| 256 | 233 | */ |
| 257 | 234 | export function llmsTxt(ctx: Ctx): Response { |
| 258 | 235 | const o = ctx.url.origin; |
| 259 | const repos = listReposVisibleTo(ctx.app.db, undefined, { | |
| 260 | limit: LLMS_REPOS, | |
| 261 | }); | |
| 236 | const repos = llmsRepos(ctx.app.db); | |
| 262 | 237 | return text( |
| 263 | 238 | [ |
| 264 | 239 | `# forge at ${ctx.url.host}`, |
| @@ -290,9 +265,7 @@ export function llmsTxt(ctx: Ctx): Response { | ||
| 290 | 265 | // Names and addresses only: a description is whatever its owner |
| 291 | 266 | // wrote, and an agent reading this file must not meet anyone's words |
| 292 | 267 | // as the forge's own. |
| 293 | ...repos.map( | |
| 294 | (r) => `- [${r.ownerName}/${r.name}](${o}/${r.ownerName}/${r.name})`, | |
| 295 | ), | |
| 268 | ...repos.map((r) => `- [${r}](${o}/${r})`), | |
| 296 | 269 | "", |
| 297 | 270 | ].join("\n"), |
| 298 | 271 | "text/markdown; charset=utf-8", |
test/standards.test.ts +85 −1
| @@ -11,10 +11,15 @@ import { mkdtemp, rm } from "node:fs/promises"; | ||
| 11 | 11 | import { tmpdir } from "node:os"; |
| 12 | 12 | import { join } from "node:path"; |
| 13 | 13 | import { createApp, type App } from "../src/app.ts"; |
| 14 | import { createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { promote } from "./helpers/members.ts"; | |
| 15 | import { suspendAccount, unsuspendAccount } from "../src/actions/accounts.ts"; | |
| 16 | import { createRepo, updateRepoSettings } from "../src/actions/repos.ts"; | |
| 15 | 17 | import { decodePng, pngHeader } from "../src/design/image/png.ts"; |
| 16 | 18 | import { markPng } from "../src/design/mark.ts"; |
| 17 | 19 | import { createUser, type User } from "../src/core/users.ts"; |
| 20 | import { forgetListings, listingBuilds } from "../src/core/listings.ts"; | |
| 21 | import { createOrg, retireOrg } from "../src/core/orgs.ts"; | |
| 22 | import { getRepo } from "../src/core/repos.ts"; | |
| 18 | 23 | import { serve, type Servers } from "../src/http/server.ts"; |
| 19 | 24 | |
| 20 | 25 | describe("what the open web expects", () => { |
| @@ -117,6 +122,85 @@ describe("what the open web expects", () => { | ||
| 117 | 122 | expect(body).not.toContain("alice/closed"); |
| 118 | 123 | }); |
| 119 | 124 | |
| 125 | test("sitemap and llms.txt are each built once for 50 requests, and a repository made private drops out at once", async () => { | |
| 126 | const mid = await createRepo(app, alice, { | |
| 127 | name: "midway", | |
| 128 | visibility: "public", | |
| 129 | }); | |
| 130 | if (!mid.ok) throw new Error(String(mid.error)); | |
| 131 | for (const path of ["/sitemap.xml", "/llms.txt"]) { | |
| 132 | const key = path === "/llms.txt" ? "llms" : "sitemap"; | |
| 133 | await fetch(`${base}${path}`).then((r) => r.text()); | |
| 134 | const before = listingBuilds[key]; | |
| 135 | for (let i = 0; i < 50; i++) | |
| 136 | await fetch(`${base}${path}`).then((r) => r.text()); | |
| 137 | expect(listingBuilds[key]).toBe(before); | |
| 138 | } | |
| 139 | const repo = getRepo(app.db, "alice", "midway")!; | |
| 140 | const done = await updateRepoSettings(app, alice, repo, { | |
| 141 | visibility: "private", | |
| 142 | }); | |
| 143 | expect(done.ok).toBe(true); | |
| 144 | for (const path of ["/sitemap.xml", "/llms.txt"]) { | |
| 145 | const body = await fetch(`${base}${path}`).then((r) => r.text()); | |
| 146 | expect(body).not.toContain("alice/midway"); | |
| 147 | expect(body).toContain("alice/open"); | |
| 148 | } | |
| 149 | }); | |
| 150 | ||
| 151 | test("a suspended account's repositories leave the cached listings at once", async () => { | |
| 152 | const bob = await createUser(app.db, { | |
| 153 | origin: "seed", | |
| 154 | trusted: true, | |
| 155 | username: "bobby", | |
| 156 | email: "[email protected]", | |
| 157 | password: "a-good-passphrase", | |
| 158 | }); | |
| 159 | if (!bob.ok) throw new Error(bob.error); | |
| 160 | promote(app.db, bob.value); | |
| 161 | const r = await createRepo(app, bob.value, { | |
| 162 | name: "shown", | |
| 163 | visibility: "public", | |
| 164 | }); | |
| 165 | if (!r.ok) throw new Error(String(r.error)); | |
| 166 | const lists = async () => | |
| 167 | ( | |
| 168 | await Promise.all( | |
| 169 | ["/sitemap.xml", "/llms.txt"].map((p) => | |
| 170 | fetch(`${base}${p}`).then((x) => x.text()), | |
| 171 | ), | |
| 172 | ) | |
| 173 | ).join("\n"); | |
| 174 | await lists(); | |
| 175 | await suspendAccount(app, alice, "bobby", { confirmed: true }); | |
| 176 | expect(await lists()).not.toContain("bobby/shown"); | |
| 177 | unsuspendAccount(app, alice, "bobby"); | |
| 178 | expect(await lists()).toContain("bobby/shown"); | |
| 179 | }); | |
| 180 | ||
| 181 | test("a retired organization's repositories leave the cached listings at once", async () => { | |
| 182 | const org = createOrg(app.db, "gone-co", alice.id); | |
| 183 | if (!org.ok) throw new Error(org.error); | |
| 184 | const made = await createRepo(app, org.value, { | |
| 185 | name: "ledger", | |
| 186 | visibility: "public", | |
| 187 | }); | |
| 188 | if (!made.ok) throw new Error(String(made.error)); | |
| 189 | const lists = async () => | |
| 190 | ( | |
| 191 | await Promise.all( | |
| 192 | ["/sitemap.xml", "/llms.txt"].map((p) => | |
| 193 | fetch(`${base}${p}`).then((x) => x.text()), | |
| 194 | ), | |
| 195 | ) | |
| 196 | ).join("\n"); | |
| 197 | // A new repository joins the listings when they are next read. | |
| 198 | forgetListings(app.db); | |
| 199 | expect(await lists()).toContain("gone-co/ledger"); | |
| 200 | retireOrg(app.db, org.value.id, new Date().toISOString()); | |
| 201 | expect(await lists()).not.toContain("gone-co/ledger"); | |
| 202 | }); | |
| 203 | ||
| 120 | 204 | test("the manifest is installable and its icons are real PNGs", async () => { |
| 121 | 205 | const res = await fetch(`${base}/manifest.webmanifest`); |
| 122 | 206 | expect(res.headers.get("content-type")).toContain( |
new above
offline
Doubled blank line left by the deleted sitemapPaths.