Public: anyone can read this; the crew can change it.

find_tool answers twenty plain-words questions with the right tool first, and a test keeps it so #14714

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14702-find-tool-answers-twenty-plain-words-que → main 4 commits fork 2 threads · 2 unresolved

  • Merged. scout/14702-find-tool-answers-twenty-plain-words-que was merged into main as aefa81f.

Review

Sign in to review this change.
4 files changed +178 −9
src/http/mcp/catalog.ts +38 −7
@@ -37,6 +37,28 @@ export const WORDS_MAX = 8;
3737 /** How many matches one answer holds. */
3838 export const FIND_MAX = 8;
3939
40 /** Words that carry no tool: a plain question is read without them. */
41 const STOP = new Set(
42 "a an the of to for my me i is it this that on in at do does did can how what whats which who why where about with and or be so should now us we you our your am are has have".split(" "),
43 );
44
45 /**
46 * Plain words for what a tool does, matched like a description word but never sent to a client, so
47 * they cost the size ceiling nothing. A word here is a hit, weighed between a name and a
48 * description.
49 */
50 export const KEYWORDS: Readonly<Record<string, string>> = {
51 make_it_yours: "copy clone duplicate",

Keys are tool names with no check they exist; add a test that every key is in TOOLS so a rename cannot orphan them.

52 fork: "copy clone duplicate",
53 asks: "waiting inbox todo attention pending",
54 say: "message crew comment reply talk chat",
55 tell: "request change crew ask build",
56 preview: "failed failure check why",
57 now: "doing busy working right currently crew",
58 usage: "spent spend cost money budget",
59 run_log: "failed failure why log output",
60 };
61
4062 /** Entries matching every word of `query`, best first; a name weighs more than a description. */
4163 export function matching(entries: readonly ToolEntry[], query: string): ToolEntry[] {
4264 const words = query
@@ -44,23 +66,32 @@ export function matching(entries: readonly ToolEntry[], query: string): ToolEntr
4466 .toLowerCase()
4567 .split(/[^a-z0-9_]+/)
4668 .filter((w) => w !== "")
47 .slice(0, WORDS_MAX);
69 .slice(0, WORDS_MAX)
70 .filter((w, _, all) => !STOP.has(w) || all.every((x) => STOP.has(x)));
4871 if (words.length === 0) return [];
49 const scored: { e: ToolEntry; score: number }[] = [];
72 const scored: { e: ToolEntry; score: number; hits: number }[] = [];
5073 for (const e of entries) {
5174 const name = e.name.toLowerCase();
5275 const desc = e.description.toLowerCase();
76 const keys = KEYWORDS[e.name]?.split(" ") ?? [];
5377 let score = 0;
54 let all = true;
78 let hits = 0;
5579 for (const w of words) {
5680 const n = name.includes(w) ? (name === w ? 6 : 3) : 0;
5781 const d = desc.includes(w) ? 1 : 0;
58 if (n + d === 0) all = false;
59 score += n + d;
82 const k = keys.includes(w) ? 2 : 0;
83 if (n + d + k > 0) hits++;
84 score += n + d + k;
6085 }
61 if (all) scored.push({ e, score });
86 if (hits > 0) scored.push({ e, score, hits });
6287 }
63 return scored.sort((a, b) => b.score - a.score || a.e.name.localeCompare(b.e.name)).map((s) => s.e);
88 // Every word is the answer; when no tool has them all (a plain question carries words no tool
89 // uses), the tools with the most words are.
90 const best = Math.max(0, ...scored.map((s) => s.hits));
91 return scored
92 .filter((s) => s.hits === best)
93 .sort((a, b) => b.score - a.score || a.e.name.localeCompare(b.e.name))
94 .map((s) => s.e);
6495 }
6596
6697 /** Whether `args.all` asks for today's full list. */
src/http/mcp/tools/find_tool.ts +1 −1
@@ -3,7 +3,7 @@ import { FIND_MAX, matching, QUERY_MAX } from "../catalog.ts";
33
44 export const tool: Tool = {
55 name: "find_tool",
6 doc: "`{ query }`: the tools whose name or description match every word, with their one-line description and schema; a tool found is callable at once, listed or not. `tools/list` with `{\"all\": true}` is the full list",
6 doc: "`{ query }`: the tools whose name, description or plain-word keywords match every word (else the most words); filler words are ignored, with their one-line description and schema; a tool found is callable at once, listed or not. `tools/list` with `{\"all\": true}` is the full list",
77 description: "Find a tool by what it does; returns name, description and schema. Any tool is callable by name.",
88 inputSchema: {
99 type: "object",
test/mcp-find-tool-queries.test.ts added +124
@@ -0,0 +1,124 @@
1 /**
2 * An outside agent that does not know tool names asks in plain words, and the
3 * tool it needs comes first (#14702). A miss is fixed in the tool's description
4 * or its KEYWORDS in src/http/mcp/catalog.ts, never by a new tool.
5 */
6 import { expect, test } from "bun:test";
7 import { afterAll, beforeAll } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { createToken } from "../src/core/tokens.ts";
13 import { createUser } from "../src/core/users.ts";
14 import { serve, type Servers } from "../src/http/server.ts";
15 import { FIND_MAX, KEYWORDS } from "../src/http/mcp/catalog.ts";
16 import { TOOLS } from "../src/http/mcp/tools.ts";
17
18 export const QUESTIONS: readonly (readonly [query: string, tool: string])[] = [
19 ["make a copy of this app", "make_it_yours"],
20 ["what is waiting on me", "asks"],
21 ["say something to the crew", "say"],
22 ["see why a check failed", "preview"],
23 ["what is the crew doing right now", "now"],
24 ["tell the crew to change something", "tell"],
25 ["make a new app from an idea", "new_app"],
26 ["open a pull request", "open_pull"],
27 ["merge a pull", "merge"],
28 ["how much has the crew spent", "usage"],
29 ["read the log of a run", "run_log"],
30 ["search the code for a word", "search"],
31 ["list the issues in a repo", "issues"],
32 ["close an issue", "close"],
33 ["approve a pull", "review"],
34 ["stop a run that is going wrong", "stop"],
35 ["undo a change that shipped", "take_back"],
36 ["go back to the last release", "rollback"],
37 ["who am I and what can I do", "whoami_can"],
38 ["change an environment variable", "app_env"],
39 ["add a custom domain to an app", "app_domains"],
outdated

This runs matching(), not find_tool, though box 2 says find_tool. Call the real tool handler / tools/call so the test fails when find_tool itself breaks.

40 ["show me my home timeline", "feed"],
41 ];
42
43 let dataDir: string;
44 let app: App;
45 let servers: Servers;
46 let token: string;
47
48 beforeAll(async () => {
49 dataDir = await mkdtemp(join(tmpdir(), "forge-find-tool-"));
50 app = createApp({ dataDir, port: 0 });
51 servers = await serve(app);
52 const made = await createUser(app.db, {
53 origin: "seed",
54 trusted: true,
55 username: "alice",
56 email: "[email protected]",
57 password: "a-good-passphrase",
58 adminIfFirst: true,
59 });
60 if (!made.ok) throw new Error(made.error);
61 token = createToken(app.db, made.value.id, "admin", "admin").token;
62 }, 30_000);
63
64 afterAll(async () => {
65 servers.stop();
66 app.close();
67 await rm(dataDir, { recursive: true, force: true });
68 });
69
70 /** The real tools/call path: visibility, the FIND_MAX cut and the output shape included. */
71 async function ask(query: string): Promise<{ tools: { name: string }[]; more?: number }> {
72 const res = (await (
73 await fetch(`${servers.url}/mcp`, {
74 method: "POST",
75 headers: {
76 "content-type": "application/json",
77 accept: "application/json, text/event-stream",
78 authorization: `Bearer ${token}`,
79 },
80 body: JSON.stringify({
81 jsonrpc: "2.0",
82 id: 1,
83 method: "tools/call",
84 params: { name: "find_tool", arguments: { query } },
85 }),
86 })
87 ).json()) as any;
88 return JSON.parse(res.result.content[0].text);
89 }
90
91 for (const [query, expected] of QUESTIONS) {
92 test(`find_tool "${query}" finds ${expected} first`, async () => {
93 const found = await ask(query);
94 expect(found.tools.length).toBeLessThanOrEqual(FIND_MAX);
95 const actual = found.tools[0]?.name;
96 expect(actual, `query "${query}": expected ${expected}, got ${actual}`).toBe(expected);
97 expect(typeof (found.tools[0] as any).description).toBe("string");
98 });
99 }
100
101 test("at least twenty questions, each naming a real tool", () => {
102 expect(QUESTIONS.length).toBeGreaterThanOrEqual(20);
103 const names = TOOLS.map((t) => t.name);
104 for (const [, tool] of QUESTIONS) expect(names).toContain(tool);
105 });
106
107 test("every KEYWORDS key is a real tool, and every keyword is a lowercase word", () => {
108 const names = TOOLS.map((t) => t.name);
109 for (const [key, words] of Object.entries(KEYWORDS)) {
110 expect(names, `KEYWORDS names "${key}", which is no tool`).toContain(key);
111 for (const w of words.split(" ")) expect(w).toMatch(/^[a-z0-9_]+$/);
112 }
113 });
114
115 test("every keyword is used: some question reaches its tool by it", () => {
116 // A keyword that no tool's name or description already holds is live only through KEYWORDS;
117 // each key must carry at least one such word, or the entry is dead weight.
118 for (const [key, words] of Object.entries(KEYWORDS)) {
119 const tool = TOOLS.find((t) => t.name === key)!;
120 const own = `${tool.name} ${tool.description}`.toLowerCase();
121 const live = words.split(" ").filter((w) => !own.includes(w));
122 expect(live.length, `KEYWORDS["${key}"] only repeats the name and description`).toBeGreaterThan(0);
123 }
124 });
test/mcp-size.test.ts +15 −1
@@ -249,12 +249,26 @@ test("matching: every word must hit, a name outranks a description, FIND_MAX cut
249249 expect(matching(es, "alpha").map((e) => e.name)).toEqual(["alpha_tool", "beta_tool", "gamma"].sort((a, b) =>
250250 a === "alpha_tool" ? -1 : b === "alpha_tool" ? 1 : a.localeCompare(b)));
251251 expect(matching(es, "alpha beta").map((e) => e.name)[2]).toBe("gamma");
252 expect(matching(es, "alpha zzz")).toEqual([]);
252 expect(matching(es, "alpha zzz").length).toBe(3); // no tool has both: the tools with most words
253 expect(matching(es, "zzz")).toEqual([]);
253254 expect(matching(es, " ")).toEqual([]);
254255 const many = Array.from({ length: FIND_MAX + 3 }, (_, i) => entry(`t${i}`, "same"));
255256 expect(matching(many, "same").length).toBe(FIND_MAX + 3);
256257 });
257258
259 test("matching drops filler words, and falls back to the tools with the most words", () => {
260 const es = [
261 { name: "alpha", description: "x", inputSchema: {} },
262 { name: "beta", description: "alpha", inputSchema: {} },
263 ];
264 // "the" and "of" hit nothing in a name, but a tool holding "the" must not be required.
265 expect(matching([{ name: "zed", description: "the of", inputSchema: {} }, ...es], "what is the alpha").map((e) => e.name)).toEqual(["alpha", "beta"]);
266 // All filler: the words are kept rather than answering nothing.
267 expect(matching([{ name: "the", description: "x", inputSchema: {} }], "the").map((e) => e.name)).toEqual(["the"]);
268 // No tool has both words: the one with the most is the answer, not nothing.
269 expect(matching(es, "beta nothing").map((e) => e.name)).toEqual(["beta"]);
270 });
271
258272 test("matching bounds the query: past 100 characters or 8 words is ignored", () => {
259273 const es = [entry("tool", "needle")];
260274 const words = Array.from({ length: WORDS_MAX }, () => "needle").join(" ");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs