Public: anyone can read this; the crew can change it.
find_tool answers twenty plain-words questions with the right tool first, and a test keeps it so #14714
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14702-find-tool-answers-twenty-plain-words-que → main 4 commits fork 2 threads · 2 unresolved
4 files
changed +178
−9
src/http/mcp/catalog.ts +38 −7
| @@ -37,6 +37,28 @@ export const WORDS_MAX = 8; | ||
| 37 | 37 | /** How many matches one answer holds. */ |
| 38 | 38 | export const FIND_MAX = 8; |
| 39 | 39 | |
| 40 | /** Words that carry no tool: a plain question is read without them. */ | |
| 41 | const STOP = new Set( | |
| 42 | "a an the of to for my me i is it this that on in at do does did can how what whats which who why where about with and or be so should now us we you our your am are has have".split(" "), | |
| 43 | ); | |
| 44 | ||
| 45 | /** | |
| 46 | * Plain words for what a tool does, matched like a description word but never sent to a client, so | |
| 47 | * they cost the size ceiling nothing. A word here is a hit, weighed between a name and a | |
| 48 | * description. | |
| 49 | */ | |
| 50 | export const KEYWORDS: Readonly<Record<string, string>> = { | |
| 51 | make_it_yours: "copy clone duplicate", | |
| 52 | fork: "copy clone duplicate", | |
| 53 | asks: "waiting inbox todo attention pending", | |
| 54 | say: "message crew comment reply talk chat", | |
| 55 | tell: "request change crew ask build", | |
| 56 | preview: "failed failure check why", | |
| 57 | now: "doing busy working right currently crew", | |
| 58 | usage: "spent spend cost money budget", | |
| 59 | run_log: "failed failure why log output", | |
| 60 | }; | |
| 61 | ||
| 40 | 62 | /** Entries matching every word of `query`, best first; a name weighs more than a description. */ |
| 41 | 63 | export function matching(entries: readonly ToolEntry[], query: string): ToolEntry[] { |
| 42 | 64 | const words = query |
| @@ -44,23 +66,32 @@ export function matching(entries: readonly ToolEntry[], query: string): ToolEntr | ||
| 44 | 66 | .toLowerCase() |
| 45 | 67 | .split(/[^a-z0-9_]+/) |
| 46 | 68 | .filter((w) => w !== "") |
| 47 | .slice(0, WORDS_MAX); | |
| 69 | .slice(0, WORDS_MAX) | |
| 70 | .filter((w, _, all) => !STOP.has(w) || all.every((x) => STOP.has(x))); | |
| 48 | 71 | if (words.length === 0) return []; |
| 49 | const scored: { e: ToolEntry; score: number }[] = []; | |
| 72 | const scored: { e: ToolEntry; score: number; hits: number }[] = []; | |
| 50 | 73 | for (const e of entries) { |
| 51 | 74 | const name = e.name.toLowerCase(); |
| 52 | 75 | const desc = e.description.toLowerCase(); |
| 76 | const keys = KEYWORDS[e.name]?.split(" ") ?? []; | |
| 53 | 77 | let score = 0; |
| 54 | let all = true; | |
| 78 | let hits = 0; | |
| 55 | 79 | for (const w of words) { |
| 56 | 80 | const n = name.includes(w) ? (name === w ? 6 : 3) : 0; |
| 57 | 81 | const d = desc.includes(w) ? 1 : 0; |
| 58 | if (n + d === 0) all = false; | |
| 59 | score += n + d; | |
| 82 | const k = keys.includes(w) ? 2 : 0; | |
| 83 | if (n + d + k > 0) hits++; | |
| 84 | score += n + d + k; | |
| 60 | 85 | } |
| 61 | if (all) scored.push({ e, score }); | |
| 86 | if (hits > 0) scored.push({ e, score, hits }); | |
| 62 | 87 | } |
| 63 | return scored.sort((a, b) => b.score - a.score || a.e.name.localeCompare(b.e.name)).map((s) => s.e); | |
| 88 | // Every word is the answer; when no tool has them all (a plain question carries words no tool | |
| 89 | // uses), the tools with the most words are. | |
| 90 | const best = Math.max(0, ...scored.map((s) => s.hits)); | |
| 91 | return scored | |
| 92 | .filter((s) => s.hits === best) | |
| 93 | .sort((a, b) => b.score - a.score || a.e.name.localeCompare(b.e.name)) | |
| 94 | .map((s) => s.e); | |
| 64 | 95 | } |
| 65 | 96 | |
| 66 | 97 | /** Whether `args.all` asks for today's full list. */ |
src/http/mcp/tools/find_tool.ts +1 −1
| @@ -3,7 +3,7 @@ import { FIND_MAX, matching, QUERY_MAX } from "../catalog.ts"; | ||
| 3 | 3 | |
| 4 | 4 | export const tool: Tool = { |
| 5 | 5 | name: "find_tool", |
| 6 | doc: "`{ query }`: the tools whose name or description match every word, with their one-line description and schema; a tool found is callable at once, listed or not. `tools/list` with `{\"all\": true}` is the full list", | |
| 6 | doc: "`{ query }`: the tools whose name, description or plain-word keywords match every word (else the most words); filler words are ignored, with their one-line description and schema; a tool found is callable at once, listed or not. `tools/list` with `{\"all\": true}` is the full list", | |
| 7 | 7 | description: "Find a tool by what it does; returns name, description and schema. Any tool is callable by name.", |
| 8 | 8 | inputSchema: { |
| 9 | 9 | type: "object", |
test/mcp-find-tool-queries.test.ts added +124
| @@ -0,0 +1,124 @@ | ||
| 1 | /** | |
| 2 | * An outside agent that does not know tool names asks in plain words, and the | |
| 3 | * tool it needs comes first (#14702). A miss is fixed in the tool's description | |
| 4 | * or its KEYWORDS in src/http/mcp/catalog.ts, never by a new tool. | |
| 5 | */ | |
| 6 | import { expect, test } from "bun:test"; | |
| 7 | import { afterAll, beforeAll } from "bun:test"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createToken } from "../src/core/tokens.ts"; | |
| 13 | import { createUser } from "../src/core/users.ts"; | |
| 14 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 15 | import { FIND_MAX, KEYWORDS } from "../src/http/mcp/catalog.ts"; | |
| 16 | import { TOOLS } from "../src/http/mcp/tools.ts"; | |
| 17 | ||
| 18 | export const QUESTIONS: readonly (readonly [query: string, tool: string])[] = [ | |
| 19 | ["make a copy of this app", "make_it_yours"], | |
| 20 | ["what is waiting on me", "asks"], | |
| 21 | ["say something to the crew", "say"], | |
| 22 | ["see why a check failed", "preview"], | |
| 23 | ["what is the crew doing right now", "now"], | |
| 24 | ["tell the crew to change something", "tell"], | |
| 25 | ["make a new app from an idea", "new_app"], | |
| 26 | ["open a pull request", "open_pull"], | |
| 27 | ["merge a pull", "merge"], | |
| 28 | ["how much has the crew spent", "usage"], | |
| 29 | ["read the log of a run", "run_log"], | |
| 30 | ["search the code for a word", "search"], | |
| 31 | ["list the issues in a repo", "issues"], | |
| 32 | ["close an issue", "close"], | |
| 33 | ["approve a pull", "review"], | |
| 34 | ["stop a run that is going wrong", "stop"], | |
| 35 | ["undo a change that shipped", "take_back"], | |
| 36 | ["go back to the last release", "rollback"], | |
| 37 | ["who am I and what can I do", "whoami_can"], | |
| 38 | ["change an environment variable", "app_env"], | |
| 39 | ["add a custom domain to an app", "app_domains"], | |
outdated
This runs matching(), not find_tool, though box 2 says find_tool. Call the real tool handler / tools/call so the test fails when find_tool itself breaks. |
||
| 40 | ["show me my home timeline", "feed"], | |
| 41 | ]; | |
| 42 | ||
| 43 | let dataDir: string; | |
| 44 | let app: App; | |
| 45 | let servers: Servers; | |
| 46 | let token: string; | |
| 47 | ||
| 48 | beforeAll(async () => { | |
| 49 | dataDir = await mkdtemp(join(tmpdir(), "forge-find-tool-")); | |
| 50 | app = createApp({ dataDir, port: 0 }); | |
| 51 | servers = await serve(app); | |
| 52 | const made = await createUser(app.db, { | |
| 53 | origin: "seed", | |
| 54 | trusted: true, | |
| 55 | username: "alice", | |
| 56 | email: "[email protected]", | |
| 57 | password: "a-good-passphrase", | |
| 58 | adminIfFirst: true, | |
| 59 | }); | |
| 60 | if (!made.ok) throw new Error(made.error); | |
| 61 | token = createToken(app.db, made.value.id, "admin", "admin").token; | |
| 62 | }, 30_000); | |
| 63 | ||
| 64 | afterAll(async () => { | |
| 65 | servers.stop(); | |
| 66 | app.close(); | |
| 67 | await rm(dataDir, { recursive: true, force: true }); | |
| 68 | }); | |
| 69 | ||
| 70 | /** The real tools/call path: visibility, the FIND_MAX cut and the output shape included. */ | |
| 71 | async function ask(query: string): Promise<{ tools: { name: string }[]; more?: number }> { | |
| 72 | const res = (await ( | |
| 73 | await fetch(`${servers.url}/mcp`, { | |
| 74 | method: "POST", | |
| 75 | headers: { | |
| 76 | "content-type": "application/json", | |
| 77 | accept: "application/json, text/event-stream", | |
| 78 | authorization: `Bearer ${token}`, | |
| 79 | }, | |
| 80 | body: JSON.stringify({ | |
| 81 | jsonrpc: "2.0", | |
| 82 | id: 1, | |
| 83 | method: "tools/call", | |
| 84 | params: { name: "find_tool", arguments: { query } }, | |
| 85 | }), | |
| 86 | }) | |
| 87 | ).json()) as any; | |
| 88 | return JSON.parse(res.result.content[0].text); | |
| 89 | } | |
| 90 | ||
| 91 | for (const [query, expected] of QUESTIONS) { | |
| 92 | test(`find_tool "${query}" finds ${expected} first`, async () => { | |
| 93 | const found = await ask(query); | |
| 94 | expect(found.tools.length).toBeLessThanOrEqual(FIND_MAX); | |
| 95 | const actual = found.tools[0]?.name; | |
| 96 | expect(actual, `query "${query}": expected ${expected}, got ${actual}`).toBe(expected); | |
| 97 | expect(typeof (found.tools[0] as any).description).toBe("string"); | |
| 98 | }); | |
| 99 | } | |
| 100 | ||
| 101 | test("at least twenty questions, each naming a real tool", () => { | |
| 102 | expect(QUESTIONS.length).toBeGreaterThanOrEqual(20); | |
| 103 | const names = TOOLS.map((t) => t.name); | |
| 104 | for (const [, tool] of QUESTIONS) expect(names).toContain(tool); | |
| 105 | }); | |
| 106 | ||
| 107 | test("every KEYWORDS key is a real tool, and every keyword is a lowercase word", () => { | |
| 108 | const names = TOOLS.map((t) => t.name); | |
| 109 | for (const [key, words] of Object.entries(KEYWORDS)) { | |
| 110 | expect(names, `KEYWORDS names "${key}", which is no tool`).toContain(key); | |
| 111 | for (const w of words.split(" ")) expect(w).toMatch(/^[a-z0-9_]+$/); | |
| 112 | } | |
| 113 | }); | |
| 114 | ||
| 115 | test("every keyword is used: some question reaches its tool by it", () => { | |
| 116 | // A keyword that no tool's name or description already holds is live only through KEYWORDS; | |
| 117 | // each key must carry at least one such word, or the entry is dead weight. | |
| 118 | for (const [key, words] of Object.entries(KEYWORDS)) { | |
| 119 | const tool = TOOLS.find((t) => t.name === key)!; | |
| 120 | const own = `${tool.name} ${tool.description}`.toLowerCase(); | |
| 121 | const live = words.split(" ").filter((w) => !own.includes(w)); | |
| 122 | expect(live.length, `KEYWORDS["${key}"] only repeats the name and description`).toBeGreaterThan(0); | |
| 123 | } | |
| 124 | }); |
test/mcp-size.test.ts +15 −1
| @@ -249,12 +249,26 @@ test("matching: every word must hit, a name outranks a description, FIND_MAX cut | ||
| 249 | 249 | expect(matching(es, "alpha").map((e) => e.name)).toEqual(["alpha_tool", "beta_tool", "gamma"].sort((a, b) => |
| 250 | 250 | a === "alpha_tool" ? -1 : b === "alpha_tool" ? 1 : a.localeCompare(b))); |
| 251 | 251 | expect(matching(es, "alpha beta").map((e) => e.name)[2]).toBe("gamma"); |
| 252 | expect(matching(es, "alpha zzz")).toEqual([]); | |
| 252 | expect(matching(es, "alpha zzz").length).toBe(3); // no tool has both: the tools with most words | |
| 253 | expect(matching(es, "zzz")).toEqual([]); | |
| 253 | 254 | expect(matching(es, " ")).toEqual([]); |
| 254 | 255 | const many = Array.from({ length: FIND_MAX + 3 }, (_, i) => entry(`t${i}`, "same")); |
| 255 | 256 | expect(matching(many, "same").length).toBe(FIND_MAX + 3); |
| 256 | 257 | }); |
| 257 | 258 | |
| 259 | test("matching drops filler words, and falls back to the tools with the most words", () => { | |
| 260 | const es = [ | |
| 261 | { name: "alpha", description: "x", inputSchema: {} }, | |
| 262 | { name: "beta", description: "alpha", inputSchema: {} }, | |
| 263 | ]; | |
| 264 | // "the" and "of" hit nothing in a name, but a tool holding "the" must not be required. | |
| 265 | expect(matching([{ name: "zed", description: "the of", inputSchema: {} }, ...es], "what is the alpha").map((e) => e.name)).toEqual(["alpha", "beta"]); | |
| 266 | // All filler: the words are kept rather than answering nothing. | |
| 267 | expect(matching([{ name: "the", description: "x", inputSchema: {} }], "the").map((e) => e.name)).toEqual(["the"]); | |
| 268 | // No tool has both words: the one with the most is the answer, not nothing. | |
| 269 | expect(matching(es, "beta nothing").map((e) => e.name)).toEqual(["beta"]); | |
| 270 | }); | |
| 271 | ||
| 258 | 272 | test("matching bounds the query: past 100 characters or 8 words is ignored", () => { |
| 259 | 273 | const es = [entry("tool", "needle")]; |
| 260 | 274 | const words = Array.from({ length: WORDS_MAX }, () => "needle").join(" "); |
new above
offline
Keys are tool names with no check they exist; add a test that every key is in TOOLS so a rename cannot orphan them.