Public: anyone can read this; the crew can change it.
The sitemap and llms.txt tell an agent the truth: every public page is listed, and an unstamped build says so plainly #13195
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13190-the-sitemap-and-llms-txt-tell-an-agent-t → main 5 commits fork
4 files
changed +59
−12
src/core/v1_proof.ts +1 −1
| @@ -124,7 +124,7 @@ export const probes: readonly Probe[] = [ | ||
| 124 | 124 | item: "production names the commit it runs, so it can be told from main", |
| 125 | 125 | run: async (get) => { |
| 126 | 126 | const got = await bodyOk(get, "/llms.txt"); |
| 127 | const m = /Running \S+ at ([0-9a-f]{7,40})/.exec(got.text); | |
| 127 | const m = /Running [^\n]+? at ([0-9a-f]{7,40})/.exec(got.text); | |
| 128 | 128 | expect(m !== null, "/llms.txt does not name the running commit"); |
| 129 | 129 | return `running ${m?.[1] ?? ""}`; |
| 130 | 130 | }, |
src/web/standards.ts +25 −9
| @@ -53,7 +53,7 @@ import { threeDoors } from "./doors_words.ts"; | ||
| 53 | 53 | import { agentCard, toolsKey, apiCatalog, cardsLine, DISCOVERY, mcpCard, offeredPlugins, mcpPointers, offeredPluginRepo, pluginSource } from "../core/mcp_card.ts"; |
| 54 | 54 | import { forgeDocument } from "../core/standards_index.ts"; |
| 55 | 55 | import { TAGS } from "../contract/served.ts"; |
| 56 | import { running } from "../kernel/version.ts"; | |
| 56 | import { running, UNSTAMPED } from "../kernel/version.ts"; | |
| 57 | 57 | import { GUIDE } from "../view/guide.ts"; |
| 58 | 58 | import { loadMcp } from "../http/mcp/load.ts"; |
| 59 | 59 | import { timePhase, timePhaseAsync } from "../core/route_timing.ts"; |
| @@ -131,7 +131,10 @@ export function securityTxt(ctx: Ctx): Response { | ||
| 131 | 131 | ); |
| 132 | 132 | } |
| 133 | 133 | |
| 134 | /** A discovery document and its ETag, per kind and origin, made again only when its inputs read differently. */ | |
| 134 | /** | |
| 135 | * A discovery document and its ETag, per kind and origin, made again only when its inputs read | |
| 136 | * differently. | |
| 137 | */ | |
| 135 | 138 | const discoveryBodies = new Map<string, { inputs: string; body: string; etag: string }>(); |
| 136 | 139 | |
| 137 | 140 | /** |
| @@ -147,7 +150,8 @@ function discovery(ctx: Ctx, kind: string, inputs: string, make: () => unknown, | ||
| 147 | 150 | const etag = `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`; |
| 148 | 151 | kept = { inputs, body, etag }; |
| 149 | 152 | // Origins come from the Host header: keep a handful, not one per guess. |
| 150 | // The oldest goes first (a Map keeps insertion order), so junk hosts never flush the real ones at once. | |
| 153 | // The oldest goes first (a Map keeps insertion order), so junk hosts never flush the real ones | |
| 154 | // at once. | |
| 151 | 155 | discoveryBodies.delete(key); |
| 152 | 156 | if (discoveryBodies.size >= 48) discoveryBodies.delete(discoveryBodies.keys().next().value!); |
| 153 | 157 | discoveryBodies.set(key, kept); |
| @@ -177,12 +181,18 @@ export async function agentCardJson(ctx: Ctx): Promise<Response> { | ||
| 177 | 181 | return discovery(ctx, `agent-card ${signedIn}`, JSON.stringify([now.version, toolsKey()]), () => agentCard(ctx.url.origin, now, signedIn), "application/json", signedIn ? "private, max-age=3600" : "public, max-age=3600", true, "Cookie, Authorization"); |
| 178 | 182 | } |
| 179 | 183 | |
| 180 | /** GET /.well-known/api-catalog — RFC 9727: one linkset naming each API's description and docs. Anonymous. */ | |
| 184 | /** | |
| 185 | * GET /.well-known/api-catalog — RFC 9727: one linkset naming each API's description and docs. | |
| 186 | * Anonymous. | |
| 187 | */ | |
| 181 | 188 | export function apiCatalogJson(ctx: Ctx): Response { |
| 182 | 189 | return discovery(ctx, "api-catalog", "", () => apiCatalog(ctx.url.origin), 'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"', "public, max-age=300"); |
| 183 | 190 | } |
| 184 | 191 | |
| 185 | /** GET /.well-known/forge — the standards this forge implements, by id and version (docs/standards/). */ | |
| 192 | /** | |
| 193 | * GET /.well-known/forge — the standards this forge implements, by id and version | |
| 194 | * (docs/standards/). | |
| 195 | */ | |
| 186 | 196 | export function forgeJson(ctx: Ctx): Response { |
| 187 | 197 | const version = running().version; |
| 188 | 198 | return discovery(ctx, "forge", version, () => forgeDocument(version), "application/json", "public, max-age=3600"); |
| @@ -296,6 +306,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> { | ||
| 296 | 306 | `<url><loc>${o}/explore</loc></url>`, |
| 297 | 307 | `<url><loc>${o}/changelog</loc>${changed === null ? "" : `<lastmod>${changed}</lastmod>`}</url>`, |
| 298 | 308 | `<url><loc>${o}/agents</loc></url>`, |
| 309 | `<url><loc>${o}/pricing</loc></url>`, | |
| 310 | `<url><loc>${o}/privacy</loc></url>`, | |
| 311 | `<url><loc>${o}/docs</loc></url>`, | |
| 299 | 312 | ...repos.map((r) => { |
| 300 | 313 | const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`; |
| 301 | 314 | return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`; |
| @@ -414,12 +427,15 @@ export const appleTouchIcon = fixedIcon("forge-180"); | ||
| 414 | 427 | /** `Running 0.7.51 at a1b2c3d`: which build these instructions describe. */ |
| 415 | 428 | function runningSentence(): string { |
| 416 | 429 | const r = running(); |
| 417 | return r.commit === null | |
| 418 | ? `Running ${r.version}` | |
| 419 | : `Running ${r.version} at ${r.commit.slice(0, 7)}`; | |
| 430 | // A bare `dev` tells an agent nothing: say plainly that the build is unstamped. | |
| 431 | const what = r.version === UNSTAMPED ? "a build from source" : r.version; | |
| 432 | return r.commit === null ? `Running ${what}` : `Running ${what} at ${r.commit.slice(0, 7)}`; | |
| 420 | 433 | } |
| 421 | 434 | |
| 422 | /** llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and the plugin source being public are the same. */ | |
| 435 | /** | |
| 436 | * llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and | |
| 437 | * the plugin source being public are the same. | |
| 438 | */ | |
| 423 | 439 | const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; tools: number; body: string; etag: string }>(); |
| 424 | 440 | |
| 425 | 441 | /** |
test/sitemap.test.ts +1 −1
| @@ -44,7 +44,7 @@ test("lists the 100 public repositories and none of the 20 private ones, built i | ||
| 44 | 44 | body = await res.text(); |
| 45 | 45 | } |
| 46 | 46 | expect(body.match(/\/alice\/pub\d+</g)?.length).toBe(100); |
| 47 | expect(body).not.toContain("priv"); | |
| 47 | expect(body).not.toMatch(/\/alice\/priv/); | |
| 48 | 48 | const timing = res.headers.get("server-timing") ?? ""; |
| 49 | 49 | for (const phase of ["repos", "render", "etag"]) expect(timing).toContain(`${phase};dur=`); |
| 50 | 50 | // Counted, not timed: a cold build reads once for all 120 repositories, not once each. |
test/standards-sitemap.test.ts +32 −1
| @@ -1,4 +1,7 @@ | ||
| 1 | /** /robots.txt and /sitemap.xml: public repositories with lastmod, never a private one, 304 on a repeat. */ | |
| 1 | /** | |
| 2 | * /robots.txt and /sitemap.xml: public repositories with lastmod, never a private one, 304 on a | |
| 3 | * repeat. | |
| 4 | */ | |
| 2 | 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; |
| 3 | 6 | import { mkdtemp, rm } from "node:fs/promises"; |
| 4 | 7 | import { tmpdir } from "node:os"; |
| @@ -108,3 +111,31 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day | ||
| 108 | 111 | expect(await second.text()).toContain("<lastmod>2026-03-09</lastmod>"); |
| 109 | 112 | expect(second.headers.get("etag")).not.toBe(etagFirst); |
| 110 | 113 | }); |
| 114 | ||
| 115 | test("every <loc> in the sitemap answers an anonymous caller 200, and the public pages are listed", async () => { | |
| 116 | const body = await (await get("/sitemap.xml")).text(); | |
| 117 | const locs = [...body.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!); | |
| 118 | for (const p of ["/pricing", "/privacy", "/docs"]) expect(locs).toContain(`${servers.url}${p}`); | |
| 119 | for (const loc of locs) { | |
| 120 | const res = await fetch(loc, { redirect: "manual", headers: { cookie: "" } }); | |
| 121 | await res.text(); | |
| 122 | expect([loc, res.status]).toEqual([loc, 200]); | |
| 123 | } | |
| 124 | }); | |
| 125 | ||
| 126 | test("llms.txt names an unstamped build plainly and a stamped one by version", async () => { | |
| 127 | const keys = ["FORGE_VERSION", "FORGE_COMMIT", "FORGE_APP_COMMIT"] as const; | |
| 128 | const saved = keys.map((k) => process.env[k]); | |
| 129 | try { | |
| 130 | delete process.env["FORGE_COMMIT"]; | |
| 131 | process.env["FORGE_APP_COMMIT"] = "0123456789abcdef0123"; | |
| 132 | process.env["FORGE_VERSION"] = "dev"; | |
| 133 | let line = (await (await get("/llms.txt")).text()).split("\n")[2]; | |
| 134 | expect(line).toBe("Running a build from source at 0123456"); | |
| 135 | process.env["FORGE_VERSION"] = "v1.2.3"; | |
| 136 | line = (await (await get("/llms.txt")).text()).split("\n")[2]; | |
| 137 | expect(line).toBe("Running 1.2.3 at 0123456"); | |
| 138 | } finally { | |
| 139 | keys.forEach((k, i) => (saved[i] === undefined ? delete process.env[k] : (process.env[k] = saved[i]))); | |
| 140 | } | |
| 141 | }); |
new above
offline