Public: anyone can read this; the crew can change it.
A test pins /robots.txt and /sitemap.xml: public repositories listed with lastmod, private never, and If-None-Match answers 304 #6637
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6626-a-test-pins-robots-txt-and-sitemap-xml-p → main 1 commit fork 1 thread · 1 unresolved
1 file
changed +81
−0
test/standards-sitemap.test.ts added +81
| @@ -0,0 +1,81 @@ | ||
| 1 | /** /robots.txt and /sitemap.xml: public repositories with lastmod, never a private one, 304 on a repeat. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { forgetListings } from "../src/core/listings.ts"; | |
| 9 | import { getRepo } from "../src/core/repos.ts"; | |
| 10 | import { createUser, type User } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | ||
| 13 | let dataDir: string; | |
| 14 | let app: App; | |
| 15 | let servers: Servers; | |
| 16 | let alice: User; | |
| 17 | ||
| 18 | const get = (path: string, headers: Record<string, string> = {}) => fetch(`${servers.url}${path}`, { headers }); | |
| 19 | const push = async (name: string) => { | |
| 20 | const c = await commitToBranch(app, alice, getRepo(app.db, "alice", name)!, { message: "Start", files: [{ path: "README.md", content: "hi\n" }] }); | |
| 21 | if (!c.ok) throw new Error(String(c.error)); | |
| 22 | }; | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | dataDir = await mkdtemp(join(tmpdir(), "forge-standards-sitemap-")); | |
| 26 | app = createApp({ dataDir, port: 0 }); | |
| 27 | servers = await serve(app); | |
| 28 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 29 | if (!u.ok) throw new Error(u.error); | |
| 30 | alice = u.value; | |
| 31 | for (const [name, visibility] of [["open", "public"], ["secret", "private"]] as const) { | |
| 32 | const r = await createRepo(app, alice, { name, visibility }); | |
| 33 | if (!r.ok) throw new Error(String(r.error)); | |
| 34 | } | |
| 35 | }); | |
| 36 | ||
| 37 | afterAll(async () => { | |
| 38 | servers.stop(); | |
| 39 | app.close(); | |
| 40 | await rm(dataDir, { recursive: true, force: true }); | |
| 41 | }); | |
| 42 | ||
| 43 | test("the sitemap lists / , /explore and the public repository, never the private one", async () => { | |
| 44 | const res = await get("/sitemap.xml"); | |
| 45 | expect(res.status).toBe(200); | |
| 46 | expect(res.headers.get("content-type")).toContain("application/xml"); | |
| 47 | const body = await res.text(); | |
| 48 | const o = servers.url; | |
| 49 | expect(body).toContain(`<loc>${o}/</loc>`); | |
| 50 | expect(body).toContain(`<loc>${o}/explore</loc>`); | |
| 51 | expect(body).toContain(`<loc>${o}/alice/open</loc>`); | |
| 52 | expect(body).not.toContain("secret"); | |
| 53 | }); | |
| 54 | ||
| 55 | test("a push gives the repository a lastmod and changes the ETag; a repeat with the ETag is a bare 304", async () => { | |
| 56 | const first = await get("/sitemap.xml"); | |
| 57 | const before = first.headers.get("etag")!; | |
| 58 | expect(await first.text()).not.toContain("<lastmod>"); | |
| 59 | const again = await get("/sitemap.xml", { "if-none-match": before }); | |
| 60 | expect(again.status).toBe(304); | |
| 61 | expect(await again.text()).toBe(""); | |
| 62 | ||
| 63 | await push("open"); | |
| 64 | await push("secret"); | |
| 65 | // The listing is kept for a minute by design; skip the wait. | |
| 66 | forgetListings(app.db); | |
| 67 | const after = await get("/sitemap.xml", { "if-none-match": before }); | |
| 68 | expect(after.status).toBe(200); | |
| 69 | expect(after.headers.get("etag")).not.toBe(before); | |
| 70 | const body = await after.text(); | |
| 71 | expect(body).toMatch(/<loc>[^<]*\/alice\/open<\/loc><lastmod>\d{4}-\d{2}-\d{2}<\/lastmod>/); | |
| 72 | expect(body).not.toContain("secret"); | |
| 73 | }); | |
| 74 | ||
| 75 | test("robots.txt disallows the private surfaces and names the sitemap on the request's origin", async () => { | |
| 76 | const res = await get("/robots.txt"); | |
| 77 | expect(res.status).toBe(200); | |
| 78 | const lines = (await res.text()).split("\n"); | |
| 79 | for (const p of ["/signin", "/signup", "/settings", "/search", "/api/"]) expect(lines).toContain(`Disallow: ${p}`); | |
| 80 | expect(lines).toContain(`Sitemap: ${servers.url}/sitemap.xml`); | |
| 81 | }); |
new above
offline
The description says a push to the private repository changes nothing, but both pushes happen before the single ETag comparison. Push to secret first, forgetListings, and assert the ETag equals before; then push to open and assert it differs.