Public: anyone can read this; the crew can change it.

The sitemap and llms.txt tell an agent the truth: every public page is listed, and an unstamped build says so plainly #13195

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13190-the-sitemap-and-llms-txt-tell-an-agent-t → main 5 commits fork

  • Merged. scout/13190-the-sitemap-and-llms-txt-tell-an-agent-t was merged into main as aa10491.

Review

Sign in to review this change.
4 files changed +59 −12
src/core/v1_proof.ts +1 −1
@@ -124,7 +124,7 @@ export const probes: readonly Probe[] = [
124124 item: "production names the commit it runs, so it can be told from main",
125125 run: async (get) => {
126126 const got = await bodyOk(get, "/llms.txt");
127 const m = /Running \S+ at ([0-9a-f]{7,40})/.exec(got.text);
127 const m = /Running [^\n]+? at ([0-9a-f]{7,40})/.exec(got.text);
128128 expect(m !== null, "/llms.txt does not name the running commit");
129129 return `running ${m?.[1] ?? ""}`;
130130 },
src/web/standards.ts +25 −9
@@ -53,7 +53,7 @@ import { threeDoors } from "./doors_words.ts";
5353 import { agentCard, toolsKey, apiCatalog, cardsLine, DISCOVERY, mcpCard, offeredPlugins, mcpPointers, offeredPluginRepo, pluginSource } from "../core/mcp_card.ts";
5454 import { forgeDocument } from "../core/standards_index.ts";
5555 import { TAGS } from "../contract/served.ts";
56 import { running } from "../kernel/version.ts";
56 import { running, UNSTAMPED } from "../kernel/version.ts";
5757 import { GUIDE } from "../view/guide.ts";
5858 import { loadMcp } from "../http/mcp/load.ts";
5959 import { timePhase, timePhaseAsync } from "../core/route_timing.ts";
@@ -131,7 +131,10 @@ export function securityTxt(ctx: Ctx): Response {
131131 );
132132 }
133133
134 /** A discovery document and its ETag, per kind and origin, made again only when its inputs read differently. */
134 /**
135 * A discovery document and its ETag, per kind and origin, made again only when its inputs read
136 * differently.
137 */
135138 const discoveryBodies = new Map<string, { inputs: string; body: string; etag: string }>();
136139
137140 /**
@@ -147,7 +150,8 @@ function discovery(ctx: Ctx, kind: string, inputs: string, make: () => unknown,
147150 const etag = `"${new Bun.CryptoHasher("sha256").update(body).digest("hex").slice(0, 32)}"`;
148151 kept = { inputs, body, etag };
149152 // Origins come from the Host header: keep a handful, not one per guess.
150 // The oldest goes first (a Map keeps insertion order), so junk hosts never flush the real ones at once.
153 // The oldest goes first (a Map keeps insertion order), so junk hosts never flush the real ones
154 // at once.
151155 discoveryBodies.delete(key);
152156 if (discoveryBodies.size >= 48) discoveryBodies.delete(discoveryBodies.keys().next().value!);
153157 discoveryBodies.set(key, kept);
@@ -177,12 +181,18 @@ export async function agentCardJson(ctx: Ctx): Promise<Response> {
177181 return discovery(ctx, `agent-card ${signedIn}`, JSON.stringify([now.version, toolsKey()]), () => agentCard(ctx.url.origin, now, signedIn), "application/json", signedIn ? "private, max-age=3600" : "public, max-age=3600", true, "Cookie, Authorization");
178182 }
179183
180 /** GET /.well-known/api-catalog — RFC 9727: one linkset naming each API's description and docs. Anonymous. */
184 /**
185 * GET /.well-known/api-catalog — RFC 9727: one linkset naming each API's description and docs.
186 * Anonymous.
187 */
181188 export function apiCatalogJson(ctx: Ctx): Response {
182189 return discovery(ctx, "api-catalog", "", () => apiCatalog(ctx.url.origin), 'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"', "public, max-age=300");
183190 }
184191
185 /** GET /.well-known/forge — the standards this forge implements, by id and version (docs/standards/). */
192 /**
193 * GET /.well-known/forge — the standards this forge implements, by id and version
194 * (docs/standards/).
195 */
186196 export function forgeJson(ctx: Ctx): Response {
187197 const version = running().version;
188198 return discovery(ctx, "forge", version, () => forgeDocument(version), "application/json", "public, max-age=3600");
@@ -296,6 +306,9 @@ export async function sitemapXml(ctx: Ctx): Promise<Response> {
296306 `<url><loc>${o}/explore</loc></url>`,
297307 `<url><loc>${o}/changelog</loc>${changed === null ? "" : `<lastmod>${changed}</lastmod>`}</url>`,
298308 `<url><loc>${o}/agents</loc></url>`,
309 `<url><loc>${o}/pricing</loc></url>`,
310 `<url><loc>${o}/privacy</loc></url>`,
311 `<url><loc>${o}/docs</loc></url>`,
299312 ...repos.map((r) => {
300313 const mod = r.pushedAt === null ? "" : `<lastmod>${r.pushedAt.slice(0, 10)}</lastmod>`;
301314 return `<url><loc>${o}${xml(`/${r.path}`)}</loc>${mod}</url>`;
@@ -414,12 +427,15 @@ export const appleTouchIcon = fixedIcon("forge-180");
414427 /** `Running 0.7.51 at a1b2c3d`: which build these instructions describe. */
415428 function runningSentence(): string {
416429 const r = running();
417 return r.commit === null
418 ? `Running ${r.version}`
419 : `Running ${r.version} at ${r.commit.slice(0, 7)}`;
430 // A bare `dev` tells an agent nothing: say plainly that the build is unstamped.
431 const what = r.version === UNSTAMPED ? "a build from source" : r.version;
432 return r.commit === null ? `Running ${what}` : `Running ${what} at ${r.commit.slice(0, 7)}`;
420433 }
421434
422 /** llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and the plugin source being public are the same. */
435 /**
436 * llms.txt and its ETag per origin, valid while the listing read, the build, the tell command and
437 * the plugin source being public are the same.
438 */
423439 const llmsBodies = new Map<string, { repos: LlmsListing; running: string; tell: string | undefined; plugins: string | undefined; plans: string; tools: number; body: string; etag: string }>();
424440
425441 /**
test/sitemap.test.ts +1 −1
@@ -44,7 +44,7 @@ test("lists the 100 public repositories and none of the 20 private ones, built i
4444 body = await res.text();
4545 }
4646 expect(body.match(/\/alice\/pub\d+</g)?.length).toBe(100);
47 expect(body).not.toContain("priv");
47 expect(body).not.toMatch(/\/alice\/priv/);
4848 const timing = res.headers.get("server-timing") ?? "";
4949 for (const phase of ["repos", "render", "etag"]) expect(timing).toContain(`${phase};dur=`);
5050 // Counted, not timed: a cold build reads once for all 120 repositories, not once each.
test/standards-sitemap.test.ts +32 −1
@@ -1,4 +1,7 @@
1 /** /robots.txt and /sitemap.xml: public repositories with lastmod, never a private one, 304 on a repeat. */
1 /**
2 * /robots.txt and /sitemap.xml: public repositories with lastmod, never a private one, 304 on a
3 * repeat.
4 */
25 import { afterAll, beforeAll, expect, test } from "bun:test";
36 import { mkdtemp, rm } from "node:fs/promises";
47 import { tmpdir } from "node:os";
@@ -108,3 +111,31 @@ test("the sitemap lists /changelog after /explore, with the newest landing's day
108111 expect(await second.text()).toContain("<lastmod>2026-03-09</lastmod>");
109112 expect(second.headers.get("etag")).not.toBe(etagFirst);
110113 });
114
115 test("every <loc> in the sitemap answers an anonymous caller 200, and the public pages are listed", async () => {
116 const body = await (await get("/sitemap.xml")).text();
117 const locs = [...body.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!);
118 for (const p of ["/pricing", "/privacy", "/docs"]) expect(locs).toContain(`${servers.url}${p}`);
119 for (const loc of locs) {
120 const res = await fetch(loc, { redirect: "manual", headers: { cookie: "" } });
121 await res.text();
122 expect([loc, res.status]).toEqual([loc, 200]);
123 }
124 });
125
126 test("llms.txt names an unstamped build plainly and a stamped one by version", async () => {
127 const keys = ["FORGE_VERSION", "FORGE_COMMIT", "FORGE_APP_COMMIT"] as const;
128 const saved = keys.map((k) => process.env[k]);
129 try {
130 delete process.env["FORGE_COMMIT"];
131 process.env["FORGE_APP_COMMIT"] = "0123456789abcdef0123";
132 process.env["FORGE_VERSION"] = "dev";
133 let line = (await (await get("/llms.txt")).text()).split("\n")[2];
134 expect(line).toBe("Running a build from source at 0123456");
135 process.env["FORGE_VERSION"] = "v1.2.3";
136 line = (await (await get("/llms.txt")).text()).split("\n")[2];
137 expect(line).toBe("Running 1.2.3 at 0123456");
138 } finally {
139 keys.forEach((k, i) => (saved[i] === undefined ? delete process.env[k] : (process.env[k] = saved[i])));
140 }
141 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs