Public: anyone can read this; the crew can change it.

An MCP tool error names the next call to try, in the same words the API's hint uses #12709

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12701-an-mcp-tool-error-names-the-next-call-to → main 5 commits fork 2 threads · 2 unresolved

  • Merged. mason/12701-an-mcp-tool-error-names-the-next-call-to was merged into main as cf41fa1.

Review

Sign in to review this change.
6 files changed +171 −24
src/http/mcp.ts +11 −9
@@ -35,7 +35,7 @@ import { TOKEN_FREE_TOOLS } from "../core/mcp_card.ts";
3535 import { exploreRun, searchRun } from "./mcp/reads.ts";
3636 import { greppable, searchKind } from "../core/search.ts";
3737 import { strangerCodeSearch } from "./strangers.ts";
38 import { retryAfterOf } from "./respond.ts";
38 import { hintForCode, retryAfterOf } from "./respond.ts";
3939 import { proofOf } from "./report_filer.ts";
4040 import { fileReport } from "../actions/reports.ts";
4141 import { START_HINT, START_WORDS, startBuilding } from "../actions/start.ts";
@@ -47,7 +47,7 @@ import { claimDone, completeWrite, dropWrite, recordWrite } from "../core/conver
4747 import { type Args, alreadyDone, notAdminTool, visible, plansOnly, str, REPO_ARG, lean, fenceAnswer, heldWrite, askedFirst, readThrough } from "./mcp/shared.ts";
4848 import { CONVERSATION_BRIEF } from "../crew/conversation_brief.ts";
4949 import { TOOLS } from "./mcp/tools.ts";
50 import { READ_ONLY_REFUSAL, noToolNamed, withNextStep } from "./mcp/hints.ts";
50 import { NO_TOOL_HINT, READ_ONLY_REFUSAL, hintOf, hintOfText, noToolNamed, withNextStep } from "./mcp/hints.ts";
5151 import { DEFAULT_TOOLS, wantsAll } from "./mcp/catalog.ts";
5252 import { OFFER_TOOL, offerAnswer } from "./mcp/offer.ts";
5353 import { OFFER_REQUEST_MAX } from "../actions/offer.ts";
@@ -127,10 +127,10 @@ function rpcOne(
127127 : {};
128128 const id = m["id"] ?? null;
129129 const reply = (result: unknown) => ({ jsonrpc: "2.0", id, result });
130 const refuse = (code: number, text: string) => ({
130 const refuse = (code: number, text: string, hint?: string) => ({
131131 jsonrpc: "2.0",
132132 id,
133 error: { code, message: text },
133 error: { code, message: text, ...(hint === undefined ? {} : { data: { hint } }) },
134134 });
135135 if (m["jsonrpc"] !== "2.0" || typeof m["method"] !== "string")
136136 return refuse(-32600, "not a JSON-RPC 2.0 request");
@@ -220,7 +220,7 @@ function rpcOne(
220220 : {};
221221 // An admin's tool stays out of an everyday account's list, and a call to
222222 // it by name is told how to reach it before anything is held or asked (#11818).
223 if (tool?.adminOnly === true && !user.isAdmin) return refuse(-32602, notAdminTool(tool.name));
223 if (tool?.adminOnly === true && !user.isAdmin) return refuse(-32602, notAdminTool(tool.name), NO_TOOL_HINT);
224224 // Before the scope floor, so an admin tool the forge's agent was
225225 // steered to is held as a confirm too, not merely refused (#7649), and
226226 // one the person asked for waits for their yes (#7650).
@@ -228,13 +228,14 @@ function rpcOne(
228228 tool === undefined ? undefined : (askedFirst(ctx, user, tool, args) ?? heldWrite(ctx, user, tool, args));
229229 if (held !== undefined) return reply(held);
230230 if (tool === undefined || !visible(tool, ctx, user))
231 return refuse(-32602, noToolNamed(String(params["name"])));
231 return refuse(-32602, noToolNamed(String(params["name"])), NO_TOOL_HINT);
232232 if (tool.writes && !plansOnly(tool, args) && !scopeAllows(ctx.auth.scope ?? "ro", "rw"))
233233 return reply({
234234 content: [
235235 { type: "text", text: READ_ONLY_REFUSAL },
236236 ],
237237 isError: true,
238 hint: hintForCode("forbidden"),
238239 });
239240 // The forge's agent's write is recorded before it runs; one a cut turn
240241 // already sent is answered with what was done, not done again (#8254).
@@ -270,6 +271,7 @@ function rpcOne(
270271 },
271272 ],
272273 isError: true,
274 hint: hintOf(tool, answer.error),
273275 },
274276 );
275277 });
@@ -369,7 +371,7 @@ async function readAnswer(ctx: Ctx, id: unknown, name: string, args: Args): Prom
369371 const text = (t: string, isError = false, extra: Record<string, unknown> = {}) => ({
370372 jsonrpc: "2.0",
371373 id,
372 result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}), ...extra },
374 result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}), ...extra },
373375 });
374376 let held: ReturnType<typeof strangerCodeSearch> | undefined;
375377 if (name === "search" && typeof args["q"] === "string") {
@@ -429,7 +431,7 @@ function reportAnswer(ctx: Ctx, id: unknown, args: Args): unknown {
429431 const text = (t: string, isError = false) => ({
430432 jsonrpc: "2.0",
431433 id,
432 result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) },
434 result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) },
433435 });
434436 const [owner, name, ...rest] = str(args["repo"]).split("/");
435437 const repo =
@@ -476,7 +478,7 @@ async function startAnswer(ctx: Ctx, id: unknown, args: Args): Promise<unknown>
476478 const text = (t: string, isError = false) => ({
477479 jsonrpc: "2.0",
478480 id,
479 result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) },
481 result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) },
480482 });
481483 if (args["action"] === "contribute") {
482484 const made = await mintContributeToken(ctx.app, args, clientBucket(ctx.request), ctx.url.origin);
src/http/mcp/hints.ts +41 −14
@@ -3,6 +3,8 @@
33 * and the tool's own schema only, so it never repeats a secret or another person's data.
44 */
55
6 import { hintForCode } from "../respond.ts";
7
68 type Schema = { required?: readonly string[]; properties?: Record<string, unknown> };
79
810 /** The argument names a tool needs, or failing that the ones it takes. */
@@ -12,23 +14,33 @@ function argsOf(schema: unknown): string[] {
1214 return names.slice(0, 4);
1315 }
1416
17 /** The one step for a refusal code; `tool` is absent when the handler has no tool schema. */
18 function stepFor(tool: { name: string; inputSchema?: unknown } | undefined, code: string): string {
19 const name = tool === undefined ? "the same tool" : `\`${tool.name}\``;
20 const args = tool === undefined ? [] : argsOf(tool.inputSchema);
21 const list = args.length === 0 ? "" : ` (${args.map((a) => `\`${a}\``).join(", ")})`;
22 if (code === "forbidden" || code === "unauthorized")
23 return "`whoami_can` says what this token and account may do; use one that may, or ask someone who can, and repeat the call.";
24 if (code === "not_found")
25 return `check the id or name against what a read of ${name} lists, or call \`find_tool\` for the tool that lists it, then repeat.`;
26 if (code === "rate_limited" || code === "too_many") return `wait a few seconds and call ${name} again.`;
27 if (code === "conflict" || code.startsWith("no_")) return `read the current state with ${name} or \`find_tool\` before trying again.`;
28 if (code === "invalid") return `call ${name} again with its arguments${list} set as tools/list describes.`;
29 return `read the message, change what it names, and call ${name} again; \`find_tool\` lists the other tools.`;
30 }
31
1532 /** `message` with a "Next:" step added unless it already carries one. */
1633 export function withNextStep(tool: { name: string; inputSchema?: unknown }, code: string, message: string): string {
1734 if (/\bnext:/i.test(message)) return message;
18 const name = `\`${tool.name}\``;
19 const args = argsOf(tool.inputSchema);
20 const list = args.length === 0 ? "" : ` (${args.map((a) => `\`${a}\``).join(", ")})`;
21 let step: string;
22 if (code === "forbidden" || code === "unauthorized")
23 step = "`whoami_can` says what this token and account may do; use one that may, or ask someone who can, and repeat the call.";
24 else if (code === "not_found")
25 step = `check the id or name against what a read of ${name} lists, or call \`find_tool\` for the tool that lists it, then repeat.`;
26 else if (code === "rate_limited" || code === "too_many")
27 step = `wait a few seconds and call ${name} again.`;
28 else if (code === "conflict" || code.startsWith("no_"))
29 step = `read the current state with ${name} or \`find_tool\` before trying again.`;
30 else step = `call ${name} again with its arguments${list} set as tools/list describes.`;
31 return `${message}${/[.!?]$/.test(message) ? "" : "."} Next: ${step}`;
35 return `${message}${/[.!?]$/.test(message) ? "" : "."} Next: ${stepFor(tool, code)}`;
36 }
37
38 /**
39 * The one-line `hint` of a refused call: the REST words where they hold for a
40 * tool too (`hintForCode`), else the tool's own next step.
41 */
42 export function hintOf(tool: { name: string; inputSchema?: unknown }, code: string): string {
43 return hintForCode(code) ?? stepFor(tool, code).replace(/^./, (c) => c.toUpperCase());
3244 }
3345
3446 /** What a read-only token is told when it tries to act. */
@@ -38,3 +50,18 @@ export const READ_ONLY_REFUSAL =
3850 /** What a name no tool answers to is told. */
3951 export const noToolNamed = (name: string): string =>
4052 `no tool named ${name}. Next: call tools/list, or \`find_tool\` with a word about what you want, for the names this token may call.`;
53
54 /** The `hint` beside a refusal for a tool the caller cannot call. */
55 export const NO_TOOL_HINT = "Call tools/list, or `find_tool` with a word about what you want, for the names this token may call.";
56
57 /**
58 * The `hint` of a token-free handler's refusal: its own "Next:" step, else the words for its
59 * leading code.
60 */
61 export function hintOfText(text: string): string {
62 const code = /^([a-z_]+): /.exec(text)?.[1];
63 const next = /\bNext: (.*)$/s.exec(text)?.[1];
64 if (next !== undefined && next.length < 200) return next;
65 const rest = code === undefined ? undefined : hintForCode(code);
66 return rest ?? stepFor(undefined, code ?? "").replace(/^./, (c) => c.toUpperCase());
outdated

The NO_TOOL_HINT fallback is wrong for refusals unrelated to tool names, such as a rate limit. Use a fitting default.

67 }
src/http/mcp/offer.ts +2 −1
@@ -10,6 +10,7 @@ import { getRepo } from "../../core/repos.ts";
1010 import type { Ctx } from "../context.ts";
1111 import { clientBucket } from "../peer.ts";
1212 import { retryAfterOf } from "../respond.ts";
13 import { hintOfText } from "./hints.ts";
1314 import { REPO_ARG, str, type Args } from "./shared.ts";
1415
1516 /** A full queue drains as people read offers; no clock says when: a minute, as the HTTP door. */
@@ -39,7 +40,7 @@ export async function offerAnswer(ctx: Ctx, id: unknown, args: Args): Promise<un
3940 const text = (t: string, isError = false) => ({
4041 jsonrpc: "2.0",
4142 id,
42 result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}) },
43 result: { content: [{ type: "text", text: t }], ...(isError ? { isError, hint: hintOfText(t) } : {}) },
4344 });
4445 const notFound = () => text(`not_found: no such public repository. ${NEXT}`, true);
4546 const [owner, name, ...rest] = str(args["repo"]).split("/");
src/http/mcp/shared.ts +11
@@ -143,6 +143,7 @@ export function heldWrite(ctx: Ctx, user: User, tool: Tool, args: Args): Record<
143143 },
144144 ],
145145 isError: true,
146 hint: confirm === null ? CONFIRMS_FULL_HINT : CONFIRM_HINT,
146147 };
147148 }
148149
@@ -160,6 +161,14 @@ export function alreadyDone(tool: string, result: string | null): Record<string,
160161 };
161162 }
162163
164 /**
165 * The hints beside a confirm that waits, a confirm that could not be kept, and arguments too big
166 * to ask about.
167 */
168 const CONFIRM_HINT = "Say the sentence in `does` to the person and point at the buttons; their yes does it once, so call nothing else for it.";
169 const CONFIRMS_FULL_HINT = "Wait for the person to answer a confirm already waiting in this conversation, then repeat the call.";
170 const TOO_BIG_HINT = "Repeat the call with less in it, split across smaller calls the person can read.";
171
163172 /** A call whose arguments a person could not read whole is refused, not kept. */
164173 const tooBig = (tool: string): Record<string, unknown> => ({
165174 content: [
@@ -169,6 +178,7 @@ const tooBig = (tool: string): Record<string, unknown> => ({
169178 },
170179 ],
171180 isError: true,
181 hint: TOO_BIG_HINT,
172182 });
173183
174184 /** What an asked-first call tells the forge's agent, beside the confirm's id and its sentence. */
@@ -205,6 +215,7 @@ export function askedFirst(ctx: Ctx, user: User, tool: Tool, args: Args): Record
205215 },
206216 ],
207217 isError: true,
218 hint: confirm === null ? CONFIRMS_FULL_HINT : CONFIRM_HINT,
208219 };
209220 }
210221
src/http/respond.ts +11
@@ -63,6 +63,17 @@ const HINTS_BY_CODE: Record<string, string> = {
6363 no_route: "No route matches this path: pick one from GET /contract/openapi.json and check the spelling.",
6464 };
6565
66 /** The codes whose REST hint reads the same to a tool caller: no route, header or verb in it. */
67 const TOOL_SAFE_CODES = ["held", "checks_required", "not_admin", "forbidden"];
68
69 /**
70 * The REST hint for an error code a tool refusal shares, or undefined when the
71 * words name a route (an MCP caller gets its own step from `mcp/hints.ts`).
72 */
73 export function hintForCode(code: string): string | undefined {
74 return TOOL_SAFE_CODES.includes(code) ? HINTS_BY_CODE[code] : undefined;
75 }

This returns the REST wording, including GET /contract/openapi.json and Authorization: Bearer, which an MCP caller cannot call. Give the tool door its own transport-neutral wording.

76
6677 /** Every 401, 403, 404, 409 and 422 error body carries `hint`; a site's own is kept. */
6778 export function withHint(data: unknown, status: number): unknown {
6879 const fallback = HINTS[status];
test/mcp-error-hints.test.ts added +95
@@ -0,0 +1,95 @@
1 /**
2 * A refused MCP tool call carries a one-line `hint` (#12701): the REST API's words
3 * for the code (`hintForCode`), under 200 bytes, with none of the caller's input.
4 * The tools come from tools/list, so a new tool is covered without editing this.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { createToken } from "../src/core/tokens.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { askingOf } from "../src/perf/asking.ts";
14 import { serve, type Servers } from "../src/http/server.ts";
15 import { hintOf, hintOfText } from "../src/http/mcp/hints.ts";
16 import { hintForCode } from "../src/http/respond.ts";
17 import { quietLogs } from "./helpers/globals.ts";
18
19 quietLogs();
20 let dir: string, app: App, servers: Servers, member: string, ro: string;
21
22 const rpc = async (tok: string, method: string, params: object = {}) => {
23 const res = await fetch(`${servers.url}/mcp`, {
24 method: "POST",
25 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${tok}` },
26 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
27 });
28 return (await res.json()) as any;
29 };
30
31 beforeAll(async () => {
32 dir = await mkdtemp(join(tmpdir(), "forge-mcp-errhints-"));
33 app = createApp({ dataDir: dir, port: 0 });
34 servers = await serve(app);
35 const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
36 if (!b.ok) throw new Error("user");
37 member = createToken(app.db, b.value.id, "t", "admin").token;
38 ro = createToken(app.db, b.value.id, "t", "ro").token;
39 });
40 afterAll(async () => {
41 askingOf(app).forget();
42 await servers.stop();
43 await rm(dir, { recursive: true, force: true });
44 });
45
46 const okHint = (h: unknown) => {
47 expect(typeof h).toBe("string");
48 expect((h as string).trim()).not.toBe("");
49 expect(Buffer.byteLength(h as string)).toBeLessThan(200);
50 };
51
52 test("the REST hint for a code is the one an MCP refusal reuses", () => {
53 expect(hintForCode("held")).toContain("hold");
54 for (const code of ["not_found", "invalid", "unauthorized", "rate_limited"]) expect(hintForCode(code)).toBeUndefined();
55 expect(hintOf({ name: "x" }, "not_found")).not.toContain("/contract/");
56 expect(hintOfText("rate_limited: slow down")).toMatch(/wait/i);
57 expect(hintOfText("something else")).not.toContain("tools/list, or");
58 });
59
60 test("every tool called with an invalid argument that refuses carries a short hint", async () => {
61 const names: string[] = [];
62 let cursor: string | undefined;
63 do {
64 const r = await rpc(member, "tools/list", { all: true, ...(cursor === undefined ? {} : { cursor }) });
65 names.push(...r.result.tools.map((t: any) => t.name));
66 cursor = r.result.nextCursor;
67 } while (cursor !== undefined);
68 expect(names.length).toBeGreaterThan(20);
69 const args = { repo: 42, kind: "zzz", id: -1, name: {}, q: [] };
70 let refused = 0;
71 for (let i = 0; i < names.length; i += 40) {
72 await Promise.all(
73 names.slice(i, i + 40).map(async (name) => {
74 const r = await rpc(member, "tools/call", { name, arguments: args });
75 expect(r.result !== undefined || r.error !== undefined).toBe(true);
76 if (r.result === undefined || r.result.isError !== true) return;
77 refused++;
78 okHint(r.result.hint);
79 expect(r.result.hint).not.toContain("/contract/");
80 expect(r.result.hint).not.toContain("Authorization");
81 expect(r.result.hint).not.toContain("zzz");
82 }),
83 );
84 }
85 expect(refused).toBeGreaterThan(10);
86 }, 120000);
87
88 test("a read-only token's refusal and an unknown tool carry a hint", async () => {
89 const w = await rpc(ro, "tools/call", { name: "say", arguments: { text: "x" } });
90 expect(w.result.isError).toBe(true);
91 okHint(w.result.hint);
92 const n = await rpc(member, "tools/call", { name: "no_such<b>", arguments: {} });
93 okHint(n.error.data.hint);
94 expect(n.error.data.hint).not.toContain("no_such");
95 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs