Public: anyone can read this; the crew can change it.
/sitemap.xml gives each repository a lastmod from its last push, and a suspended account's repositories never appear #5760
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/5694-sitemap-xml-gives-each-repository-a-last → main 2 commits fork
3 files
changed +98
−5
src/core/listings.ts +29 −4
| @@ -15,7 +15,7 @@ export const SITEMAP_ROWS = 5000; | ||
| 15 | 15 | export const LLMS_ROWS = 200; |
| 16 | 16 | |
| 17 | 17 | interface Kept { |
| 18 | readonly sitemap: Fresh<readonly string[]>; | |
| 18 | readonly sitemap: Fresh<readonly SitemapRepo[]>; | |
| 19 | 19 | readonly llms: Fresh<readonly string[]>; |
| 20 | 20 | } |
| 21 | 21 | const kept = new WeakMap<Database, Kept>(); |
| @@ -42,11 +42,36 @@ export function forgetListings(db: Database): void { | ||
| 42 | 42 | k?.llms.clear(); |
| 43 | 43 | } |
| 44 | 44 | |
| 45 | /** `owner/name` of the newest public repositories, at most `SITEMAP_ROWS`. */ | |
| 46 | export function sitemapRepos(db: Database): readonly string[] { | |
| 45 | /** A listed repository: `owner/name` and its last push (ISO time), or null. */ | |
| 46 | export interface SitemapRepo { | |
| 47 | readonly path: string; | |
| 48 | readonly pushedAt: string | null; | |
| 49 | } | |
| 50 | ||
| 51 | /** | |
| 52 | * The newest public repositories, at most `SITEMAP_ROWS`, each with the time | |
| 53 | * of its last push. Who may be listed is `listReposVisibleTo`'s answer, so a | |
| 54 | * private, archived or suspended owner's repository is never here. | |
| 55 | */ | |
| 56 | export function sitemapRepos(db: Database): readonly SitemapRepo[] { | |
| 47 | 57 | return holders(db).sitemap.get("", () => { |
| 48 | 58 | listingBuilds.sitemap++; |
| 49 | return names(db, SITEMAP_ROWS); | |
| 59 | const repos = listReposVisibleTo(db, undefined, { | |
| 60 | limit: SITEMAP_ROWS, | |
| 61 | withoutArchived: true, | |
| 62 | }); | |
| 63 | const pushed = new Map( | |
| 64 | db | |
| 65 | .query<{ repo_id: string; at: string }, []>( | |
| 66 | "SELECT repo_id, MAX(created_at) AS at FROM event WHERE kind = 'push' AND repo_id IS NOT NULL GROUP BY repo_id", | |
| 67 | ) | |
| 68 | .all() | |
| 69 | .map((r) => [r.repo_id, r.at]), | |
| 70 | ); | |
| 71 | return repos.map((r) => ({ | |
| 72 | path: `${r.ownerName}/${r.name}`, | |
| 73 | pushedAt: pushed.get(r.id) ?? null, | |
| 74 | })); | |
| 50 | 75 | }); |
| 51 | 76 | } |
| 52 | 77 |
src/web/standards.ts +4 −1
| @@ -174,7 +174,10 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 174 | 174 | const urls = [ |
| 175 | 175 | `<url><loc>${origin}/</loc></url>`, |
| 176 | 176 | `<url><loc>${origin}/explore</loc></url>`, |
| 177 | ...repos.map((r) => `<url><loc>${origin}${xml(`/${r}`)}</loc></url>`), | |
| 177 | ...repos.map((r) => { | |
| 178 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; | |
| 179 | return `<url><loc>${origin}${xml(`/${r.path}`)}</loc>${mod}</url>`; | |
| 180 | }), | |
| 178 | 181 | ]; |
| 179 | 182 | return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`; |
| 180 | 183 | }); |
test/sitemap-lastmod.test.ts added +65
| @@ -0,0 +1,65 @@ | ||
| 1 | /** /sitemap.xml gives a pushed repository a lastmod, and drops a suspended account's. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { suspendAccount } from "../src/actions/accounts.ts"; | |
| 8 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { getRepo } from "../src/core/repos.ts"; | |
| 10 | import { createUser, type User } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import { promote } from "./helpers/members.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: App; | |
| 16 | let servers: Servers; | |
| 17 | let alice: User; | |
| 18 | let bob: User; | |
| 19 | ||
| 20 | const sitemap = async () => (await fetch(`${servers.url}/sitemap.xml`)).text(); | |
| 21 | const entry = (body: string, path: string) => | |
| 22 | body.match(new RegExp(`<url><loc>[^<]*/${path}</loc>(.*?)</url>`))?.[1]; | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | dataDir = await mkdtemp(join(tmpdir(), "forge-sitemap-lastmod-")); | |
| 26 | app = createApp({ dataDir, port: 0 }); | |
| 27 | servers = await serve(app); | |
| 28 | const make = async (username: string, adminIfFirst: boolean) => { | |
| 29 | const u = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst }); | |
| 30 | if (!u.ok) throw new Error(u.error); | |
| 31 | promote(app.db, u.value); | |
| 32 | return u.value; | |
| 33 | }; | |
| 34 | alice = await make("alice", true); | |
| 35 | bob = await make("bob", false); | |
| 36 | for (const [u, name] of [[alice, "pushed"], [alice, "empty"], [bob, "tools"]] as const) { | |
| 37 | const r = await createRepo(app, u, { name, visibility: "public" }); | |
| 38 | if (!r.ok) throw new Error(String(r.error)); | |
| 39 | } | |
| 40 | for (const [u, owner, name] of [[alice, "alice", "pushed"], [bob, "bob", "tools"]] as const) { | |
| 41 | const c = await commitToBranch(app, u, getRepo(app.db, owner, name)!, { message: "Start", files: [{ path: "README.md", content: "hi\n" }] }); | |
| 42 | if (!c.ok) throw new Error(String(c.error)); | |
| 43 | } | |
| 44 | }); | |
| 45 | ||
| 46 | afterAll(async () => { | |
| 47 | servers.stop(); | |
| 48 | app.close(); | |
| 49 | await rm(dataDir, { recursive: true, force: true }); | |
| 50 | }); | |
| 51 | ||
| 52 | test("a pushed public repository has its lastmod as a date", async () => { | |
| 53 | expect(entry(await sitemap(), "alice/pushed")).toMatch(/^<lastmod>\d{4}-\d{2}-\d{2}<\/lastmod>$/); | |
| 54 | }); | |
| 55 | ||
| 56 | test("a never-pushed repository has no lastmod", async () => { | |
| 57 | expect(entry(await sitemap(), "alice/empty")).toBe(""); | |
| 58 | }); | |
| 59 | ||
| 60 | test("after its owner is suspended the repository is no longer listed", async () => { | |
| 61 | expect(await sitemap()).toContain("/bob/tools</loc>"); | |
| 62 | const r = await suspendAccount(app, alice, "bob"); | |
| 63 | if (!r.ok) throw new Error(String(r.error)); | |
| 64 | expect(await sitemap()).not.toContain("/bob/tools"); | |
| 65 | }); |
new above
offline