Public: anyone can read this; the crew can change it.

Every /api/v1 and /mcp answer, including errors and 404s, carries the security floor headers: one test sweeps them #14715

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14701-every-api-v1-and-mcp-answer-including-er → main 2 commits fork

  • Merged. mason/14701-every-api-v1-and-mcp-answer-including-er was merged into main as 5fd34c9.

Review

Sign in to review this change.
1 file changed +272 −0
test/api-security-floor.test.ts added +272
@@ -0,0 +1,272 @@
1 /**
2 * Every /api/v1 and /mcp answer carries the security floor (#14701):
3 * Content-Security-Policy, X-Content-Type-Options and Referrer-Policy, on the
4 * 200s and on the refusals alike.
5 *
6 * The routes are read from the contract as the forge serves it
7 * (`GET /contract/openapi.json`, asked as a site admin so the document is
8 * whole), never from a list here, so a route added to the contract is swept
9 * without editing this file. Each operation is asked signed out (a 200 from
10 * the open reads, a 401 from the rest, a 404 where a filled parameter names
11 * nothing), its GETs signed in too, each path under a method it does not
12 * declare (a 405), and each operation that takes a body with a malformed
13 * JSON one (a 400). The MCP side is swept the same way from its own list:
14 * `tools/list`, then every tool called by a stranger, a malformed body, a
15 * notification's bodiless 202 and the methods /mcp does not take. A 429 is
16 * reached once, through the sign-in window.
17 *
18 * Exempt, and why: nothing is skipped by kind. A stream (an SSE read, a raw
19 * download) is judged by its headers alone and its body is cancelled, since
20 * the floor is laid before the first byte; a HEAD is asked of every GET and
21 * judged the same way, having no body. The parameters are filled from one
22 * seeded repository; a name the seed has no value for is filled with a
23 * placeholder, whose not-found answer must carry the floor as well.
24 *
25 * Should a route ever have to be skipped, skip it by name, never by kind:
26 * filter it out of `ops` in `beforeAll` beside a comment saying why it cannot
27 * carry the floor, and name it in the list above, so the exemption is read in
28 * one place and reviewed with the change that needs it.
29 */
30 import { afterAll, beforeAll, expect, test } from "bun:test";
31 import { mkdtemp, rm } from "node:fs/promises";
32 import { tmpdir } from "node:os";
33 import { join } from "node:path";
34 import { createApp, type App } from "../src/app.ts";
35 import { createRepo } from "../src/core/repos.ts";
36 import { createSession } from "../src/core/sessions.ts";
37 import { createToken } from "../src/core/tokens.ts";
38 import { serve, type Servers } from "../src/http/server.ts";
39 import { STRANGER_LIMITS, setStrangerLimits } from "../src/http/strangers.ts";
40 import { limitValue, setLimit } from "../src/settings/limits.ts";
41 import { seedAdmin } from "./helpers/admin.ts";
42 import { quietLogs } from "./helpers/globals.ts";
43
44 quietLogs();
45
46 const FLOOR = ["content-security-policy", "x-content-type-options", "referrer-policy"] as const;
47 const METHODS = ["get", "post", "put", "patch", "delete"] as const;
48 const IN_FLIGHT = 16;
49
50 /**
51 * What each path parameter is filled with: the seeded admin, its repository, or a well-formed value
52 * nothing seeds.
53 */
54 const FILL: Readonly<Record<string, string>> = {
55 username: "ada", owner: "ada", org: "ada", scope: "ada", user: "ada",
56 name: "notes", repo: "notes", number: "1", n: "1", id: "1", job: "1",
57 branch: "main", ref: "main", oid: "0".repeat(40), sha: "0".repeat(40),
58 path: "README.md", file: "README.md", env: "prod", team: "core", label: "v1",
59 version: "1", chapter: "1", slug: "board", receipt: "r", key: "k", host: "example.com",
60 };
61 const PLACEHOLDER = "x";
62
63 type Op = { method: string; path: string; body: boolean };
64 type Ask = { name: string; path: string; init: RequestInit; want?: number };
65 /** An answer; `want`, when the ask named one, is the status it had to be. */
66 type Seen = { name: string; status: number; want?: number; missing: string[] };
67
68 let dataDir = "";
69 let app: App, servers: Servers;
70 let cookie = "";
71 let bearer = "";
72 let ops: Op[] = [];
73
74 beforeAll(async () => {
75 dataDir = await mkdtemp(join(tmpdir(), "forge-api-floor-"));
76 app = createApp({ dataDir, port: 0 });
77 servers = await serve(app);
78 const admin = await seedAdmin(app, { username: "ada", email: "[email protected]", password: "correct-horse-battery" });
79 if (!admin.ok) throw new Error(admin.error);
80 const made = await createRepo(app.db, app.reposRoot, { ownerId: admin.value.id, name: "notes", visibility: "public" });
81 if (!made.ok) throw new Error(String(made.error));
82 cookie = `forge_session=${createSession(app.db, admin.value.id).token}`;
83 bearer = `Bearer ${createToken(app.db, admin.value.id, "floor", "admin").token}`;
84 // The sweeps ask several hundred times signed out from one address; past
85 // the tokenless allowance every answer would be the limiter's 429, not the
86 // handler's. The 429 is reached on purpose in its own test.
87 if (!setLimit(app.db, "api.anonymous_per_source", 100_000, "seed").ok) throw new Error("limit");
88 setStrangerLimits(app, {
89 perAddress: { ...STRANGER_LIMITS.perAddress, api: 100_000 },
90 forge: { ...STRANGER_LIMITS.forge, api: 100_000 },
91 });
92 const res = await fetch(`${servers.url}/contract/openapi.json`, { headers: { cookie, "accept-encoding": "identity" } });
93 for (const h of FLOOR) if (!res.headers.get(h)) throw new Error(`the contract itself lacks ${h}`);
94 const doc = (await res.json()) as { paths: Record<string, Record<string, { requestBody?: unknown }>> };
95 ops = Object.entries(doc.paths).flatMap(([path, byMethod]) =>
96 METHODS.filter((m) => byMethod[m] !== undefined).map((m) => ({ method: m.toUpperCase(), path, body: byMethod[m]!.requestBody !== undefined })),
97 );
98 }, 30_000);
99
100 afterAll(async () => {
101 servers?.stop();
102 app?.close();
103 await rm(dataDir, { recursive: true, force: true });
104 });
105
106 const fill = (path: string): string => path.replace(/\{(\w+)\}/g, (_, k: string) => FILL[k] ?? PLACEHOLDER);
107 const own = () => ({ origin: servers.url, "accept-encoding": "identity" });
108 const machine = (path: string) => path.startsWith("/api/") || path === "/mcp";
109
110 /**
111 * One answer: its status and the floor headers it lacks. A body is cancelled, never read: a
112 * stream's headers are its answer.
113 */
114 async function ask(a: Ask): Promise<Seen> {
115 const res = await fetch(`${servers.url}${a.path}`, { ...a.init, redirect: "manual", signal: AbortSignal.timeout(15_000) });
116 await res.body?.cancel().catch(() => undefined);
117 return { name: a.name, status: res.status, ...(a.want === undefined ? {} : { want: a.want }), missing: FLOOR.filter((h) => !res.headers.get(h)) };
118 }
119
120 /** Every ask, `IN_FLIGHT` at a time; the work is bounded by the contract, never by a clock. */
121 async function sweep(asks: Ask[]): Promise<Seen[]> {
122 const seen: Seen[] = [];
123 let next = 0;
124 const worker = async () => {
125 while (next < asks.length) {
126 const a = asks[next++]!;
127 seen.push(await ask(a));
128 }
129 };
130 await Promise.all(Array.from({ length: IN_FLIGHT }, worker));
131 return seen;
132 }
133
134 /** Each answer that lacks a header or is not the status its ask wanted, named with its request. */
135 const misses = (seen: Seen[]): string[] =>
136 seen.flatMap((s) => [
137 ...s.missing.map((h) => `${s.name} (${s.status}) lacks ${h}`),
138 ...(s.want !== undefined && s.status !== s.want ? [`${s.name} answered ${s.status}, wanted ${s.want}`] : []),
139 ]);
140 const statuses = (seen: Seen[]): Set<number> => new Set(seen.map((s) => s.status));
141
142 /**
143 * The sweep reached the answers named, so a pass is not a sweep of one kind,
144 * and no limiter stood in front of the handlers.
145 */
146 function reached(seen: Seen[], wanted: number[]): void {
147 const got = statuses(seen);
148 for (const status of wanted) expect(got.has(status)).toBe(true);
149 expect(seen.filter((s) => s.status === 429).map((s) => s.name)).toEqual([]);
150 }
151
152 test("the contract names API routes of every method to sweep", () => {
153 const swept = ops.filter((o) => machine(o.path));
154 expect(swept.length).toBeGreaterThan(100);
155 expect(new Set(swept.map((o) => o.method))).toEqual(new Set(["GET", "POST", "PUT", "PATCH", "DELETE"]));
156 });
157
158 test("every GET and HEAD in the contract, signed out and signed in, carries the floor", async () => {
159 const gets = ops.filter((o) => o.method === "GET" && machine(o.path));
160 const asks: Ask[] = gets.flatMap((o) => {
161 const path = fill(o.path);
162 return [
163 { name: `GET ${o.path} signed out`, path, init: { headers: own() } },
164 { name: `HEAD ${o.path} signed out`, path, init: { method: "HEAD", headers: own() } },
165 { name: `GET ${o.path} signed in`, path, init: { headers: { ...own(), cookie } } },
166 ];
167 });
168 const seen = await sweep(asks);
169 expect(misses(seen)).toEqual([]);
170
171 reached(seen, [200, 401, 404]);
172 }, 60_000);
173
174 test("every mutation in the contract, signed out with a malformed body, carries the floor", async () => {
175 const writes = ops.filter((o) => o.method !== "GET" && machine(o.path));
176 const asks: Ask[] = writes.map((o) => ({
177 name: `${o.method} ${o.path} signed out`,
178 path: fill(o.path),
179 init: { method: o.method, headers: { ...own(), "content-type": "application/json" }, body: "{not json" },
180 }));
181 const seen = await sweep(asks);
182 expect(misses(seen)).toEqual([]);
183 reached(seen, [401]);
184 }, 60_000);
185
186 test("every operation that takes a body, signed in with malformed JSON, carries the floor", async () => {
187 // Only the ones that declare a body: they read it before they act, so
188 // nothing in the seed is changed by the sweep.
189 const bodied = ops.filter((o) => o.body && o.method !== "GET" && machine(o.path));
190 const asks: Ask[] = bodied.map((o) => ({
191 name: `${o.method} ${o.path} malformed`,
192 path: fill(o.path),
193 init: { method: o.method, headers: { ...own(), cookie, "content-type": "application/json" }, body: "{not json" },
194 }));
195 const seen = await sweep(asks);
196 expect(misses(seen)).toEqual([]);
197 reached(seen, [400]);
198 }, 60_000);
199
200 test("every contract path under a method it does not declare carries the floor", async () => {
201 const byPath = new Map<string, Set<string>>();
202 for (const o of ops) if (machine(o.path)) byPath.set(o.path, (byPath.get(o.path) ?? new Set()).add(o.method));
203 const asks: Ask[] = [];
204 for (const [path, declared] of byPath) {
205 const other = ["PUT", "PATCH", "DELETE", "POST"].find((m) => !declared.has(m));
206 if (other !== undefined) asks.push({ name: `${other} ${path} undeclared`, path: fill(path), init: { method: other, headers: own() } });
207 }
208 const seen = await sweep(asks);
209 expect(misses(seen)).toEqual([]);
210 reached(seen, [405]);
211 }, 60_000);
212
213 test("a path the contract does not name, and a 429, carry the floor", async () => {
214 const seen = await sweep([
215 { name: "GET unknown API path", path: "/api/v1/no/such/route/at/all", init: { headers: own() }, want: 404 },
216 { name: "GET unknown repository", path: "/api/v1/repos/ada/nothing-here", init: { headers: own() }, want: 404 },
217 ]);
218 // One password attempt a minute for the whole forge: the first fills the
219 // window (and is a refusal that must carry the floor too), the second is a
220 // 429. The limit is put back after, so no later test inherits it.
221 const before = limitValue(app.db, "signin.per_forge");
222 expect(setLimit(app.db, "signin.per_forge", 1, "seed").ok).toBe(true);
223 try {
224 const signin = (name: string, want?: number): Ask => ({
225 name,
226 path: "/api/v1/sessions",
227 init: { method: "POST", headers: { ...own(), "content-type": "application/json" }, body: '{"username":"ada","password":"wrong"}' },
228 ...(want === undefined ? {} : { want }),
229 });
230 seen.push(await ask(signin("POST /api/v1/sessions filling the window")));
231 seen.push(await ask(signin("POST /api/v1/sessions past the window", 429)));
232 } finally {
233 expect(setLimit(app.db, "signin.per_forge", before, "seed").ok).toBe(true);
234 }
235 expect(seen.length).toBe(4);
236 expect(seen[2]!.status).not.toBe(429);
237 expect(misses(seen)).toEqual([]);
238 });
239
240 /** One JSON-RPC message to /mcp, with the admin's token or as a stranger. */
241 const rpc = (name: string, message: unknown, signedIn: boolean): Ask => ({
242 name,
243 path: "/mcp",
244 init: {
245 method: "POST",
246 headers: { ...own(), "content-type": "application/json", accept: "application/json, text/event-stream", ...(signedIn ? { authorization: bearer } : {}) },
247 body: typeof message === "string" ? message : JSON.stringify(message),
248 },
249 });
250
251 test("every MCP answer, each tool called by a stranger and every refusal, carries the floor", async () => {
252 const listed = await fetch(`${servers.url}/mcp`, rpc("tools/list", { jsonrpc: "2.0", id: 1, method: "tools/list" }, true).init);
253 for (const h of FLOOR) expect(listed.headers.get(h)).toBeTruthy();
254 const tools = ((await listed.json()) as { result: { tools: { name: string }[] } }).result.tools.map((t) => t.name);
255 expect(tools.length).toBeGreaterThan(5);
256 const asks: Ask[] = [
257 rpc("MCP initialize", { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "floor", version: "1" } } }, true),
258 rpc("MCP notification", { jsonrpc: "2.0", method: "notifications/initialized" }, true),
259 rpc("MCP malformed JSON", "{not json", true),
260 rpc("MCP malformed JSON, stranger", "{not json", false),
261 rpc("MCP unknown method", { jsonrpc: "2.0", id: 2, method: "no/such" }, true),
262 rpc("MCP unknown tool", { jsonrpc: "2.0", id: 3, method: "tools/call", params: { name: "no_such_tool", arguments: {} } }, false),
263 { name: "GET /mcp", path: "/mcp", init: { headers: own() } },
264 { name: "GET /mcp as a browser", path: "/mcp", init: { headers: { ...own(), accept: "text/html" } } },
265 { name: "DELETE /mcp", path: "/mcp", init: { method: "DELETE", headers: own() } },
266 { name: "PUT /mcp", path: "/mcp", init: { method: "PUT", headers: own() } },
267 ...tools.map((tool, i) => rpc(`MCP tools/call ${tool}, stranger`, { jsonrpc: "2.0", id: 10 + i, method: "tools/call", params: { name: tool, arguments: {} } }, false)),
268 ];
269 const seen = await sweep(asks);
270 expect(misses(seen)).toEqual([]);
271 reached(seen, [200, 202, 405]);
272 }, 60_000);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs