Public: anyone can read this; the crew can change it.
Every MCP tool answers hostile arguments with a named refusal, never a 500: a fuzz test walks tools/list #11026
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/11021-every-mcp-tool-answers-hostile-arguments → main 3 commits fork 2 threads · 2 unresolved
1 file
changed +292
−0
test/fuzz-mcp.test.ts added +292
| @@ -0,0 +1,292 @@ | ||
| 1 | /** | |
| 2 | * Every MCP tool answers hostile arguments with a named refusal, never a | |
| 3 | * 500, a throw or a hang (#11021). One forge boots; `tools/list` is read as | |
| 4 | * a signed-in non-admin; each listed tool is then called with seeded | |
| 5 | * hostile arguments built from its own inputSchema: no arguments, the | |
| 6 | * arguments not an object, wrong types, unknown keys (`__proto__` among | |
| 7 | * them), 10 kB strings, negative and huge numbers, `../` paths and deeply | |
| 8 | * nested objects. Every answer must be a JSON-RPC result (`isError` or not) | |
| 9 | * or a JSON-RPC error with a message, inside a second. | |
| 10 | * | |
| 11 | * Anonymous callers send every name the forge has (`toolNames()`): each is | |
| 12 | * the 401 that names `report`, except exactly the token-free tools, `report` | |
| 13 | * and `start` (the keys of `TOKEN_FREE_HANDLERS`, so the next one is fuzzed | |
| 14 | * without an edit here). Those are fuzzed anonymously from their own | |
| 15 | * schemas, the only caller that reaches their real path; a refused call | |
| 16 | * must leave no account and no repository behind, and `start` with a | |
| 17 | * hostile name must be refused. | |
| 18 | * | |
| 19 | * Exclusions, named rather than skipped silently: the admin-only tools are | |
| 20 | * not listed to the member, so they are fuzzed only as the "no tool named" | |
| 21 | * refusal a member gets for them (the last test); their argument checks run | |
| 22 | * behind the admin scope. Nothing here can reach outside the test forge: | |
| 23 | * the forge has no runner, no cloud account and no egress configured. | |
| 24 | * | |
| 25 | * Rerun one failure with `FUZZ_SEED=<n> bun test test/fuzz-mcp.test.ts`. | |
| 26 | */ | |
| 27 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 28 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 29 | import { tmpdir } from "node:os"; | |
| 30 | import { join } from "node:path"; | |
| 31 | import { createApp, type App } from "../src/app.ts"; | |
| 32 | import { createRepo } from "../src/actions/repos.ts"; | |
| 33 | import { createToken } from "../src/core/tokens.ts"; | |
| 34 | import { createUser } from "../src/core/users.ts"; | |
| 35 | import { TOKEN_FREE_HANDLERS, toolNames } from "../src/http/mcp.ts"; | |
| 36 | import { TOOLS } from "../src/http/mcp/tools.ts"; | |
| 37 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 38 | import { int, pick, rng, SEED, type Rand } from "./helpers/fuzz.ts"; | |
| 39 | import { listAt } from "./mcp-list.ts"; | |
| 40 | ||
| 41 | /** How long one call may take before it counts as a hang (ms). */ | |
| 42 | const CALL_MS = 1000; | |
| 43 | /** Random cases per tool, on top of the fixed ones. */ | |
| 44 | const RANDOM_CASES = 4; | |
| 45 | ||
| 46 | type Schema = { type?: string | string[]; properties?: Record<string, Schema>; required?: string[]; items?: Schema; enum?: unknown[] }; | |
| 47 | ||
| 48 | const BIG = "x".repeat(10 * 1024); | |
| 49 | const nested = (depth: number): unknown => (depth === 0 ? { a: "../" } : { a: nested(depth - 1), b: [nested(0)] }); | |
| 50 | ||
| 51 | /** Hostile values in no particular type, for any field. */ | |
| 52 | const ANY: unknown[] = [ | |
| 53 | null, | |
| 54 | true, | |
| 55 | 0, | |
| 56 | -1, | |
| 57 | -2_147_483_649, | |
| 58 | 1e308, | |
| 59 | Number.MAX_SAFE_INTEGER + 2, | |
| 60 | 0.5, | |
| 61 | "", | |
| 62 | " ", | |
| 63 | BIG, | |
| 64 | "../../../etc/passwd", | |
| 65 | "..", | |
| 66 | "/", | |
| 67 | "bob/../alice/secret", | |
| 68 | "alice/open/../../x", | |
| 69 | // Real names, so a case gets past "not found" to the argument checks behind it. | |
| 70 | "alice/open", | |
| 71 | "alice/secret", | |
| 72 | "bob", | |
| 73 | "main", | |
| 74 | "%2e%2e%2f%2e%2e%2f", | |
| 75 | "a\u0000b", | |
| 76 | "<script>alert(1)</script>", | |
| 77 | "'; drop table users; --", | |
| 78 | "\ud800", | |
| 79 | [], | |
| 80 | ["../", 1, null], | |
| 81 | {}, | |
| 82 | { __proto__: null, toString: 1 }, | |
| 83 | nested(40), | |
| 84 | ]; | |
| 85 | ||
| 86 | /** Wrong for the declared type: a string where a number is wanted, and so on. */ | |
| 87 | function wrongFor(schema: Schema): unknown[] { | |
| 88 | const types = [schema.type].flat(); | |
| 89 | const out: unknown[] = []; | |
| 90 | if (types.includes("string")) out.push(42, ["a"], { s: "a" }, BIG, "../../..", ""); | |
| 91 | if (types.includes("integer") || types.includes("number")) | |
| 92 | out.push("12", -1, -1e308, 1e308, 0.25, Number.MAX_SAFE_INTEGER * 4, { n: 1 }); | |
| 93 | if (types.includes("boolean")) out.push("true", 1, null); | |
| 94 | if (types.includes("array")) out.push("a,b", { 0: "a" }, Array.from({ length: 2000 }, () => "../"), [nested(20)]); | |
| 95 | if (types.includes("object")) out.push("{}", [], nested(40)); | |
| 96 | if (out.length === 0) out.push(...ANY.slice(0, 6)); | |
| 97 | return out; | |
| 98 | } | |
| 99 | ||
| 100 | /** The fixed cases for a schema, then seeded random ones. */ | |
| 101 | function cases(name: string, schema: Schema, r: Rand): unknown[] { | |
| 102 | const props = Object.entries(schema.properties ?? {}); | |
| 103 | const all: unknown[] = [undefined, {}, "not-an-object", [1, 2], null]; | |
| 104 | // Every field at once: each wrong type, a 10 kB string, a `../` path. | |
| 105 | for (const pickAt of [0, 3, 4]) { | |
| 106 | const args: Record<string, unknown> = {}; | |
| 107 | for (const [k, s] of props) args[k] = wrongFor(s)[pickAt % wrongFor(s).length]; | |
| 108 | all.push(args); | |
| 109 | } | |
| 110 | // Unknown keys, `__proto__` set as an own key (JSON.parse makes it one). | |
| 111 | all.push(JSON.parse(`{"__proto__":{"admin":true,"scope":"admin"},"constructor":{"prototype":{}},"zz${name}":1}`)); | |
| 112 | // Huge and negative numbers in every numeric field, required present. | |
| 113 | const numeric: Record<string, unknown> = {}; | |
| 114 | for (const [k, s] of props) numeric[k] = [s.type].flat().some((t) => t === "integer" || t === "number") ? -1e308 : "../"; | |
| 115 | all.push(numeric); | |
| 116 | // Seeded: each field drawn from its wrong values or the general pool, some left out. | |
| 117 | for (let i = 0; i < RANDOM_CASES; i++) { | |
| 118 | const args: Record<string, unknown> = {}; | |
| 119 | for (const [k, s] of props) { | |
| 120 | if (r() < 0.25) continue; | |
| 121 | args[k] = r() < 0.5 ? pick(r, wrongFor(s)) : pick(r, ANY); | |
| 122 | } | |
| 123 | if (r() < 0.3) args[`extra${int(r, 100)}`] = pick(r, ANY); | |
| 124 | all.push(args); | |
| 125 | } | |
| 126 | return all; | |
| 127 | } | |
| 128 | ||
| 129 | const seedFor = (name: string): number => { | |
| 130 | let h = SEED >>> 0; | |
| 131 | for (const c of name) h = Math.imul(h ^ c.charCodeAt(0), 16777619) >>> 0; | |
| 132 | return h; | |
| 133 | }; | |
| 134 | ||
| 135 | describe("MCP fuzz: hostile arguments get a named refusal", () => { | |
| 136 | let dataDir: string; | |
| 137 | let app: App; | |
| 138 | let servers: Servers; | |
| 139 | let member: string; | |
| 140 | let bobId: string; | |
| 141 | let id = 1; | |
| 142 | let spent = 0; | |
| 143 | ||
| 144 | /** | |
| 145 | * The member's token, a fresh one every 400 calls: the forge holds each | |
| 146 | * token to 600 requests a minute (and fewer proposals), and the fuzz is | |
| 147 | * about argument checks, not that limiter, which has its own tests. | |
| 148 | */ | |
| 149 | const memberToken = (): string => { | |
| 150 | if (++spent % 400 === 0) member = createToken(app.db, bobId, `rw${spent}`, "rw").token; | |
| 151 | return member; | |
| 152 | }; | |
| 153 | ||
| 154 | /** One tools/call, raw: the HTTP status, the parsed body, and how long it took. */ | |
| 155 | const call = async (token: string | undefined, name: string, args: unknown) => { | |
| 156 | const started = performance.now(); | |
| 157 | const params = args === undefined ? { name } : { name, arguments: args }; | |
| 158 | const res = await fetch(`${servers.url}/mcp`, { | |
| 159 | method: "POST", | |
| 160 | headers: { | |
| 161 | "content-type": "application/json", | |
| 162 | accept: "application/json, text/event-stream", | |
| 163 | ...(token === undefined ? {} : { authorization: `Bearer ${token}` }), | |
| 164 | }, | |
| 165 | body: JSON.stringify({ jsonrpc: "2.0", id: id++, method: "tools/call", params }), | |
| 166 | signal: AbortSignal.timeout(CALL_MS * 5), | |
| 167 | }); | |
| 168 | const text = await res.text(); | |
| 169 | let body: any; | |
| 170 | try { | |
| 171 | body = JSON.parse(text); | |
| 172 | } catch { | |
| 173 | body = { unparsed: text.slice(0, 200) }; | |
| 174 | } | |
| 175 | return { status: res.status, body, ms: performance.now() - started }; | |
| 176 | }; | |
| 177 | ||
| 178 | /** Why an answer is not a named refusal or a result, or undefined when it is. */ | |
| 179 | const flaw = (a: { status: number; body: any; ms: number }): string | undefined => { | |
| 180 | if (a.ms > CALL_MS) return `took ${Math.round(a.ms)} ms`; | |
| 181 | if (a.status !== 200) return `HTTP ${a.status}: ${JSON.stringify(a.body).slice(0, 200)}`; | |
| 182 | const b = a.body; | |
| 183 | if (b?.jsonrpc !== "2.0") return `not JSON-RPC: ${JSON.stringify(b).slice(0, 200)}`; | |
| 184 | if (b.error !== undefined) | |
| 185 | return typeof b.error.message === "string" && b.error.message.length > 0 ? undefined : "an error without a message"; | |
| 186 | const content = b.result?.content; | |
| 187 | if (!Array.isArray(content) || typeof content[0]?.text !== "string") return "a result without text content"; | |
| 188 | if (b.result.isError === true && content[0].text.trim().length === 0) return "isError without a message"; | |
| 189 | return undefined; | |
| 190 | }; | |
| 191 | ||
| 192 | beforeAll(async () => { | |
| 193 | dataDir = await mkdtemp(join(tmpdir(), "forge-fuzz-mcp-")); | |
| 194 | app = createApp({ dataDir, port: 0 }); | |
| 195 | servers = await serve(app); | |
| 196 | const mk = async (username: string, first = false) => { | |
| 197 | const u = await createUser(app.db, { | |
| 198 | origin: "seed", | |
| 199 | trusted: true, | |
| 200 | username, | |
| 201 | email: `${username}@forge.local`, | |
| 202 | password: "a-good-passphrase", | |
| 203 | adminIfFirst: first, | |
| 204 | }); | |
| 205 | if (!u.ok) throw new Error(u.error); | |
| 206 | return u.value; | |
| 207 | }; | |
| 208 | const alice = await mk("alice", true); | |
| 209 | const bob = await mk("bob"); | |
| 210 | bobId = bob.id; | |
| 211 | member = createToken(app.db, bob.id, "rw", "rw").token; | |
| 212 | for (const [owner, name, visibility] of [[alice, "secret", "private"], [alice, "open", "public"]] as const) { | |
| 213 | const made = await createRepo(app, owner, { name, visibility }); | |
| 214 | if (!made.ok) throw new Error(String(made.error)); | |
| 215 | } | |
| 216 | }, 30_000); | |
| 217 | ||
| 218 | afterAll(async () => { | |
| 219 | await servers.stop(); | |
| 220 | app.close(); | |
| 221 | await rm(dataDir, { recursive: true, force: true }); | |
| 222 | }); | |
| 223 | ||
| 224 | test("as a signed-in member, every listed tool answers every case with a result or a named error", async () => { | |
| 225 | const listed: { name: string; inputSchema: Schema }[] = (await listAt(servers.url, member)).result.tools; | |
| 226 | expect(listed.length).toBeGreaterThan(50); | |
| 227 | expect(listed.map((t) => t.name)).toContain("report"); | |
| 228 | const flaws: string[] = []; | |
| 229 | for (const tool of listed) { | |
| 230 | const r = rng(seedFor(tool.name)); | |
| 231 | for (const args of cases(tool.name, tool.inputSchema, r)) { | |
| 232 | let problem: string | undefined; | |
| 233 | try { | |
| 234 | problem = flaw(await call(memberToken(), tool.name, args)); | |
| 235 | } catch (error) { | |
| 236 | problem = `threw: ${String(error)}`; | |
| 237 | } | |
| 238 | if (problem !== undefined) | |
| 239 | flaws.push(`${tool.name} ${JSON.stringify(args)?.slice(0, 160)}: ${problem}`); | |
| 240 | } | |
| 241 | } | |
| 242 | expect(flaws).toEqual([]); | |
| 243 | }, 60_000); | |
| 244 | ||
| 245 | test("anonymous reaches exactly the token-free tools, and each refuses hostile arguments by name, leaving nothing behind", async () => { | |
| 246 | const free = Object.keys(TOKEN_FREE_HANDLERS).sort(); | |
| 247 | expect(free).toEqual(["report", "start"]); | |
| 248 | const reached: string[] = []; | |
outdated
Fuzz |
||
| 249 | for (const name of toolNames()) { | |
| 250 | const a = await call(undefined, name, { repo: "../../etc", title: BIG, __proto__: { admin: true } }); | |
| 251 | if (a.status === 401) { | |
| 252 | expect(typeof a.body.message).toBe("string"); | |
| 253 | expect(a.body.message).toContain("report"); | |
| 254 | continue; | |
| 255 | } | |
| 256 | expect([name, flaw(a)]).toEqual([name, undefined]); | |
| 257 | reached.push(name); | |
| 258 | } | |
| 259 | expect(reached.sort()).toEqual(free); | |
| 260 | ||
| 261 | // Each token-free tool's schema as the forge lists it, so none is fuzzed from a copy. | |
| 262 | const listed: { name: string; inputSchema: Schema }[] = (await listAt(servers.url, member)).result.tools; | |
| 263 | const counts = () => | |
| 264 | app.db.query<{ users: number; repos: number }, []>("SELECT (SELECT count(*) FROM user) AS users, (SELECT count(*) FROM repo) AS repos").get(); | |
| 265 | for (const name of free) { | |
| 266 | const schema = listed.find((t) => t.name === name)?.inputSchema; | |
| 267 | expect([name, schema === undefined]).toEqual([name, false]); | |
| 268 | const r = rng(seedFor(name)); | |
| 269 | for (const args of cases(name, schema as Schema, r)) { | |
| 270 | const label = `${name} ${JSON.stringify(args)?.slice(0, 160)}`; | |
| 271 | const before = counts(); | |
| 272 | const a = await call(undefined, name, args); | |
| 273 | expect([label, flaw(a)]).toEqual([label, undefined]); | |
| 274 | const refused = a.body.error !== undefined || a.body.result?.isError === true; | |
| 275 | if (refused) expect([label, counts()]).toEqual([label, before]); | |
| 276 | const given = typeof args === "object" && args !== null ? (args as Record<string, unknown>)["name"] : undefined; | |
| 277 | const badName = typeof given !== "string" || given.trim() === "" || given.trim().length > 40 || !/[a-z0-9]/i.test(given); | |
| 278 | if (name === "start" && badName) expect([label, refused]).toEqual([label, true]); | |
| 279 | } | |
| 280 | } | |
| 281 | }, 30_000); | |
| 282 | ||
| 283 | test("an admin-only tool, named by a member, is the same refusal as a tool that does not exist", async () => { | |
| 284 | const listed = new Set((await listAt(servers.url, member)).result.tools.map((t: { name: string }) => t.name)); | |
| 285 | const hidden = TOOLS.filter((t) => !listed.has(t.name)); | |
| 286 | expect(hidden.length).toBeGreaterThan(0); | |
| 287 | for (const t of hidden) { | |
| 288 | const a = await call(memberToken(), t.name, { "../": BIG }); | |
| 289 | expect([t.name, flaw(a)]).toEqual([t.name, undefined]); | |
| 290 | } | |
| 291 | }); | |
| 292 | }); |
Add
START_TOOLhere.startis token-free and creates an account, repo and app, so anonymous must reach["report", "start"]and both must be fuzzed. Better, derive the list fromTOKEN_FREE_HANDLERSortoolNames().