Public: anyone can read this; the crew can change it.
An MCP tool error names the next call to try, in the same words the API's hint uses #12709
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12701-an-mcp-tool-error-names-the-next-call-to → main 5 commits fork 2 threads · 2 unresolved
6 files
changed +171
−24
src/http/mcp.ts +11 −9
| @@ -35,7 +35,7 @@ import { TOKEN_FREE_TOOLS } from "../core/mcp_card.ts"; | ||
| 35 | 35 | import { exploreRun, searchRun } from "./mcp/reads.ts"; |
| 36 | 36 | import { greppable, searchKind } from "../core/search.ts"; |
| 37 | 37 | import { strangerCodeSearch } from "./strangers.ts"; |
| 38 | import { retryAfterOf } from "./respond.ts"; | |
| 38 | import { hintForCode, retryAfterOf } from "./respond.ts"; | |
| 39 | 39 | import { proofOf } from "./report_filer.ts"; |
| 40 | 40 | import { fileReport } from "../actions/reports.ts"; |
| 41 | 41 | import { START_HINT, START_WORDS, startBuilding } from "../actions/start.ts"; |
| @@ -47,7 +47,7 @@ import { claimDone, completeWrite, dropWrite, recordWrite } from "../core/conver | ||
| 47 | 47 | import { type Args, alreadyDone, notAdminTool, visible, plansOnly, str, REPO_ARG, lean, fenceAnswer, heldWrite, askedFirst, readThrough } from "./mcp/shared.ts"; |
| 48 | 48 | import { CONVERSATION_BRIEF } from "../crew/conversation_brief.ts"; |
| 49 | 49 | import { TOOLS } from "./mcp/tools.ts"; |
| 50 | import { READ_ONLY_REFUSAL, noToolNamed, withNextStep } from "./mcp/hints.ts"; | |
| 50 | import { NO_TOOL_HINT, READ_ONLY_REFUSAL, hintOf, hintOfText, noToolNamed, withNextStep } from "./mcp/hints.ts"; | |
| 51 | 51 | import { DEFAULT_TOOLS, wantsAll } from "./mcp/catalog.ts"; |
| 52 | 52 | import { OFFER_TOOL, offerAnswer } from "./mcp/offer.ts"; |
| 53 | 53 | import { OFFER_REQUEST_MAX } from "../actions/offer.ts"; |
| @@ -127,10 +127,10 @@ function rpcOne( | ||
| 127 | 127 | : {}; |
| 128 | 128 | const id = m["id"] ?? null; |
| 129 | 129 | const reply = (result: unknown) => ({ jsonrpc: "2.0", id, result }); |
| 130 | const refuse = (code: number, text: string) => ({ | |
| 130 | const refuse = (code: number, text: string, hint?: string) => ({ | |
| 131 | 131 | jsonrpc: "2.0", |
| 132 | 132 | id, |
| 133 | error: { code, message: text }, | |
| 133 | error: { code, message: text, ...(hint === undefined ? {} : { data: { hint } }) }, | |
| 134 | 134 | }); |
| 135 | 135 | if (m["jsonrpc"] !== "2.0" || typeof m["method"] !== "string") |
| 136 | 136 | return refuse(-32600, "not a JSON-RPC 2.0 request"); |
| @@ -220,7 +220,7 @@ function rpcOne( | ||
| 220 | 220 | : {}; |
| 221 | 221 | // An admin's tool stays out of an everyday account's list, and a call to |
| 222 | 222 | // it by name is told how to reach it before anything is held or asked (#11818). |
| 223 | if (tool?.adminOnly === true && !user.isAdmin) return refuse(-32602, notAdminTool(tool.name)); | |
| 223 | if (tool?.adminOnly === true && !user.isAdmin) return refuse(-32602, notAdminTool(tool.name), NO_TOOL_HINT); | |
| 224 | 224 | // Before the scope floor, so an admin tool the forge's agent was |
| 225 | 225 | // steered to is held as a confirm too, not merely refused (#7649), and |
| 226 | 226 | // one the person asked for waits for their yes (#7650). |
| @@ -228,13 +228,14 @@ function rpcOne( | ||
| 228 | 228 | tool === undefined ? undefined : (askedFirst(ctx, user, tool, args) ?? heldWrite(ctx, user, tool, args)); |
| 229 | 229 | if (held !== undefined) return reply(held); |
| 230 | 230 | if (tool === undefined || !visible(tool, ctx, user)) |
| 231 | return refuse(-32602, noToolNamed(String(params["name"]))); | |
| 231 | return refuse(-32602, noToolNamed(String(params["name"])), NO_TOOL_HINT); | |
| 232 | 232 | if (tool.writes && !plansOnly(tool, args) && !scopeAllows(ctx.auth.scope ?? "ro", "rw")) |
| 233 | 233 | return reply({ |
| 234 | 234 | content: [ |
| 235 | 235 | { type: "text", text: READ_ONLY_REFUSAL }, |
| 236 | 236 | ], |
| 237 | 237 | isError: true, |
| 238 | hint: hintForCode("forbidden"), | |
| 238 | 239 | }); |
| 239 | 240 | // The forge's agent's write is recorded before it runs; one a cut turn |
| 240 | 241 | // already sent is answered with what was done, not done again (#8254). |
| @@ -270,6 +271,7 @@ function rpcOne( | ||
| 270 | 271 | }, |
| 271 | 272 | ], |
| 272 | 273 | isError: true, |
| 274 | hint: hintOf(tool, answer.error), | |
| 273 | 275 | }, |
| 274 | 276 | ); |
| 275 | 277 | }); |
| @@ -369,7 +371,7 @@ async function readAnswer(ctx: Ctx, id: unknown, name: string, args: Args): Prom | ||
| 369 | 371 | const text = (t: string, isError = false, extra: Record<string, unknown> = {}) => ({ |
| 370 | 372 | jsonrpc: "2.0", |
| 371 | 373 | id, |
| 372 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}), ...extra }, | |
| 374 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}), ...extra }, | |
| 373 | 375 | }); |
| 374 | 376 | let held: ReturnType<typeof strangerCodeSearch> | undefined; |
| 375 | 377 | if (name === "search" && typeof args["q"] === "string") { |
| @@ -429,7 +431,7 @@ function reportAnswer(ctx: Ctx, id: unknown, args: Args): unknown { | ||
| 429 | 431 | const text = (t: string, isError = false) => ({ |
| 430 | 432 | jsonrpc: "2.0", |
| 431 | 433 | id, |
| 432 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) }, | |
| 434 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) }, | |
| 433 | 435 | }); |
| 434 | 436 | const [owner, name, ...rest] = str(args["repo"]).split("/"); |
| 435 | 437 | const repo = |
| @@ -476,7 +478,7 @@ async function startAnswer(ctx: Ctx, id: unknown, args: Args): Promise<unknown> | ||
| 476 | 478 | const text = (t: string, isError = false) => ({ |
| 477 | 479 | jsonrpc: "2.0", |
| 478 | 480 | id, |
| 479 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) }, | |
| 481 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) }, | |
| 480 | 482 | }); |
| 481 | 483 | if (args["action"] === "contribute") { |
| 482 | 484 | const made = await mintContributeToken(ctx.app, args, clientBucket(ctx.request), ctx.url.origin); |
src/http/mcp/hints.ts +41 −14
| @@ -3,6 +3,8 @@ | ||
| 3 | 3 | * and the tool's own schema only, so it never repeats a secret or another person's data. |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | import { hintForCode } from "../respond.ts"; | |
| 7 | ||
| 6 | 8 | type Schema = { required?: readonly string[]; properties?: Record<string, unknown> }; |
| 7 | 9 | |
| 8 | 10 | /** The argument names a tool needs, or failing that the ones it takes. */ |
| @@ -12,23 +14,33 @@ function argsOf(schema: unknown): string[] { | ||
| 12 | 14 | return names.slice(0, 4); |
| 13 | 15 | } |
| 14 | 16 | |
| 17 | /** The one step for a refusal code; `tool` is absent when the handler has no tool schema. */ | |
| 18 | function stepFor(tool: { name: string; inputSchema?: unknown } | undefined, code: string): string { | |
| 19 | const name = tool === undefined ? "the same tool" : `\`${tool.name}\``; | |
| 20 | const args = tool === undefined ? [] : argsOf(tool.inputSchema); | |
| 21 | const list = args.length === 0 ? "" : ` (${args.map((a) => `\`${a}\``).join(", ")})`; | |
| 22 | if (code === "forbidden" || code === "unauthorized") | |
| 23 | return "`whoami_can` says what this token and account may do; use one that may, or ask someone who can, and repeat the call."; | |
| 24 | if (code === "not_found") | |
| 25 | return `check the id or name against what a read of ${name} lists, or call \`find_tool\` for the tool that lists it, then repeat.`; | |
| 26 | if (code === "rate_limited" || code === "too_many") return `wait a few seconds and call ${name} again.`; | |
| 27 | if (code === "conflict" || code.startsWith("no_")) return `read the current state with ${name} or \`find_tool\` before trying again.`; | |
| 28 | if (code === "invalid") return `call ${name} again with its arguments${list} set as tools/list describes.`; | |
| 29 | return `read the message, change what it names, and call ${name} again; \`find_tool\` lists the other tools.`; | |
| 30 | } | |
| 31 | ||
| 15 | 32 | /** `message` with a "Next:" step added unless it already carries one. */ |
| 16 | 33 | export function withNextStep(tool: { name: string; inputSchema?: unknown }, code: string, message: string): string { |
| 17 | 34 | if (/\bnext:/i.test(message)) return message; |
| 18 | const name = `\`${tool.name}\``; | |
| 19 | const args = argsOf(tool.inputSchema); | |
| 20 | const list = args.length === 0 ? "" : ` (${args.map((a) => `\`${a}\``).join(", ")})`; | |
| 21 | let step: string; | |
| 22 | if (code === "forbidden" || code === "unauthorized") | |
| 23 | step = "`whoami_can` says what this token and account may do; use one that may, or ask someone who can, and repeat the call."; | |
| 24 | else if (code === "not_found") | |
| 25 | step = `check the id or name against what a read of ${name} lists, or call \`find_tool\` for the tool that lists it, then repeat.`; | |
| 26 | else if (code === "rate_limited" || code === "too_many") | |
| 27 | step = `wait a few seconds and call ${name} again.`; | |
| 28 | else if (code === "conflict" || code.startsWith("no_")) | |
| 29 | step = `read the current state with ${name} or \`find_tool\` before trying again.`; | |
| 30 | else step = `call ${name} again with its arguments${list} set as tools/list describes.`; | |
| 31 | return `${message}${/[.!?]$/.test(message) ? "" : "."} Next: ${step}`; | |
| 35 | return `${message}${/[.!?]$/.test(message) ? "" : "."} Next: ${stepFor(tool, code)}`; | |
| 36 | } | |
| 37 | ||
| 38 | /** | |
| 39 | * The one-line `hint` of a refused call: the REST words where they hold for a | |
| 40 | * tool too (`hintForCode`), else the tool's own next step. | |
| 41 | */ | |
| 42 | export function hintOf(tool: { name: string; inputSchema?: unknown }, code: string): string { | |
| 43 | return hintForCode(code) ?? stepFor(tool, code).replace(/^./, (c) => c.toUpperCase()); | |
| 32 | 44 | } |
| 33 | 45 | |
| 34 | 46 | /** What a read-only token is told when it tries to act. */ |
| @@ -38,3 +50,18 @@ export const READ_ONLY_REFUSAL = | ||
| 38 | 50 | /** What a name no tool answers to is told. */ |
| 39 | 51 | export const noToolNamed = (name: string): string => |
| 40 | 52 | `no tool named ${name}. Next: call tools/list, or \`find_tool\` with a word about what you want, for the names this token may call.`; |
| 53 | ||
| 54 | /** The `hint` beside a refusal for a tool the caller cannot call. */ | |
| 55 | export const NO_TOOL_HINT = "Call tools/list, or `find_tool` with a word about what you want, for the names this token may call."; | |
| 56 | ||
| 57 | /** | |
| 58 | * The `hint` of a token-free handler's refusal: its own "Next:" step, else the words for its | |
| 59 | * leading code. | |
| 60 | */ | |
| 61 | export function hintOfText(text: string): string { | |
| 62 | const code = /^([a-z_]+): /.exec(text)?.[1]; | |
| 63 | const next = /\bNext: (.*)$/s.exec(text)?.[1]; | |
| 64 | if (next !== undefined && next.length < 200) return next; | |
| 65 | const rest = code === undefined ? undefined : hintForCode(code); | |
| 66 | return rest ?? stepFor(undefined, code ?? "").replace(/^./, (c) => c.toUpperCase()); | |
| 67 | } |
src/http/mcp/offer.ts +2 −1
| @@ -10,6 +10,7 @@ import { getRepo } from "../../core/repos.ts"; | ||
| 10 | 10 | import type { Ctx } from "../context.ts"; |
| 11 | 11 | import { clientBucket } from "../peer.ts"; |
| 12 | 12 | import { retryAfterOf } from "../respond.ts"; |
| 13 | import { hintOfText } from "./hints.ts"; | |
| 13 | 14 | import { REPO_ARG, str, type Args } from "./shared.ts"; |
| 14 | 15 | |
| 15 | 16 | /** A full queue drains as people read offers; no clock says when: a minute, as the HTTP door. */ |
| @@ -39,7 +40,7 @@ export async function offerAnswer(ctx: Ctx, id: unknown, args: Args): Promise<un | ||
| 39 | 40 | const text = (t: string, isError = false) => ({ |
| 40 | 41 | jsonrpc: "2.0", |
| 41 | 42 | id, |
| 42 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) }, | |
| 43 | result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) }, | |
| 43 | 44 | }); |
| 44 | 45 | const notFound = () => text(`not_found: no such public repository. ${NEXT}`, true); |
| 45 | 46 | const [owner, name, ...rest] = str(args["repo"]).split("/"); |
src/http/respond.ts +11
| @@ -63,6 +63,17 @@ const HINTS_BY_CODE: Record<string, string> = { | ||
| 63 | 63 | no_route: "No route matches this path: pick one from GET /contract/openapi.json and check the spelling.", |
| 64 | 64 | }; |
| 65 | 65 | |
| 66 | /** The codes whose REST hint reads the same to a tool caller: no route, header or verb in it. */ | |
| 67 | const TOOL_SAFE_CODES = ["held", "checks_required", "not_admin", "forbidden"]; | |
| 68 | ||
| 69 | /** | |
| 70 | * The REST hint for an error code a tool refusal shares, or undefined when the | |
| 71 | * words name a route (an MCP caller gets its own step from `mcp/hints.ts`). | |
| 72 | */ | |
| 73 | export function hintForCode(code: string): string | undefined { | |
| 74 | return TOOL_SAFE_CODES.includes(code) ? HINTS_BY_CODE[code] : undefined; | |
| 75 | } | |
This returns the REST wording, including |
||
| 76 | ||
| 66 | 77 | /** Every 401, 403, 404, 409 and 422 error body carries `hint`; a site's own is kept. */ |
| 67 | 78 | export function withHint(data: unknown, status: number): unknown { |
| 68 | 79 | const fallback = HINTS[status]; |
test/mcp-error-hints.test.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | /** | |
| 2 | * A refused MCP tool call carries a one-line `hint` (#12701): the REST API's words | |
| 3 | * for the code (`hintForCode`), under 200 bytes, with none of the caller's input. | |
| 4 | * The tools come from tools/list, so a new tool is covered without editing this. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { createToken } from "../src/core/tokens.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { askingOf } from "../src/perf/asking.ts"; | |
| 14 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 15 | import { hintOf, hintOfText } from "../src/http/mcp/hints.ts"; | |
| 16 | import { hintForCode } from "../src/http/respond.ts"; | |
| 17 | import { quietLogs } from "./helpers/globals.ts"; | |
| 18 | ||
| 19 | quietLogs(); | |
| 20 | let dir: string, app: App, servers: Servers, member: string, ro: string; | |
| 21 | ||
| 22 | const rpc = async (tok: string, method: string, params: object = {}) => { | |
| 23 | const res = await fetch(`${servers.url}/mcp`, { | |
| 24 | method: "POST", | |
| 25 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${tok}` }, | |
| 26 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }), | |
| 27 | }); | |
| 28 | return (await res.json()) as any; | |
| 29 | }; | |
| 30 | ||
| 31 | beforeAll(async () => { | |
| 32 | dir = await mkdtemp(join(tmpdir(), "forge-mcp-errhints-")); | |
| 33 | app = createApp({ dataDir: dir, port: 0 }); | |
| 34 | servers = await serve(app); | |
| 35 | const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 36 | if (!b.ok) throw new Error("user"); | |
| 37 | member = createToken(app.db, b.value.id, "t", "admin").token; | |
| 38 | ro = createToken(app.db, b.value.id, "t", "ro").token; | |
| 39 | }); | |
| 40 | afterAll(async () => { | |
| 41 | askingOf(app).forget(); | |
| 42 | await servers.stop(); | |
| 43 | await rm(dir, { recursive: true, force: true }); | |
| 44 | }); | |
| 45 | ||
| 46 | const okHint = (h: unknown) => { | |
| 47 | expect(typeof h).toBe("string"); | |
| 48 | expect((h as string).trim()).not.toBe(""); | |
| 49 | expect(Buffer.byteLength(h as string)).toBeLessThan(200); | |
| 50 | }; | |
| 51 | ||
| 52 | test("the REST hint for a code is the one an MCP refusal reuses", () => { | |
| 53 | expect(hintForCode("held")).toContain("hold"); | |
| 54 | for (const code of ["not_found", "invalid", "unauthorized", "rate_limited"]) expect(hintForCode(code)).toBeUndefined(); | |
| 55 | expect(hintOf({ name: "x" }, "not_found")).not.toContain("/contract/"); | |
| 56 | expect(hintOfText("rate_limited: slow down")).toMatch(/wait/i); | |
| 57 | expect(hintOfText("something else")).not.toContain("tools/list, or"); | |
| 58 | }); | |
| 59 | ||
| 60 | test("every tool called with an invalid argument that refuses carries a short hint", async () => { | |
| 61 | const names: string[] = []; | |
| 62 | let cursor: string | undefined; | |
| 63 | do { | |
| 64 | const r = await rpc(member, "tools/list", { all: true, ...(cursor === undefined ? {} : { cursor }) }); | |
| 65 | names.push(...r.result.tools.map((t: any) => t.name)); | |
| 66 | cursor = r.result.nextCursor; | |
| 67 | } while (cursor !== undefined); | |
| 68 | expect(names.length).toBeGreaterThan(20); | |
| 69 | const args = { repo: 42, kind: "zzz", id: -1, name: {}, q: [] }; | |
| 70 | let refused = 0; | |
| 71 | for (let i = 0; i < names.length; i += 40) { | |
| 72 | await Promise.all( | |
| 73 | names.slice(i, i + 40).map(async (name) => { | |
| 74 | const r = await rpc(member, "tools/call", { name, arguments: args }); | |
| 75 | expect(r.result !== undefined || r.error !== undefined).toBe(true); | |
| 76 | if (r.result === undefined || r.result.isError !== true) return; | |
| 77 | refused++; | |
| 78 | okHint(r.result.hint); | |
| 79 | expect(r.result.hint).not.toContain("/contract/"); | |
| 80 | expect(r.result.hint).not.toContain("Authorization"); | |
| 81 | expect(r.result.hint).not.toContain("zzz"); | |
| 82 | }), | |
| 83 | ); | |
| 84 | } | |
| 85 | expect(refused).toBeGreaterThan(10); | |
| 86 | }, 120000); | |
| 87 | ||
| 88 | test("a read-only token's refusal and an unknown tool carry a hint", async () => { | |
| 89 | const w = await rpc(ro, "tools/call", { name: "say", arguments: { text: "x" } }); | |
| 90 | expect(w.result.isError).toBe(true); | |
| 91 | okHint(w.result.hint); | |
| 92 | const n = await rpc(member, "tools/call", { name: "no_such<b>", arguments: {} }); | |
| 93 | okHint(n.error.data.hint); | |
| 94 | expect(n.error.data.hint).not.toContain("no_such"); | |
| 95 | }); |
new above
offline
The NO_TOOL_HINT fallback is wrong for refusals unrelated to tool names, such as a rate limit. Use a fitting default.