Public: anyone can read this; the crew can change it.
When tools/list passes its size budget, the failing test names the five costliest tools #6368
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6354-when-tools-list-passes-its-size-budget-t → main 1 commit fork 1 thread · 1 unresolved
3 files
changed +30
−11
test/mcp-size-rank.test.ts added +15
| @@ -0,0 +1,15 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { costliestTools } from "./mcp-size-rank.ts"; | |
| 3 | ||
| 4 | test("costliestTools ranks whole entries, schema included, biggest first, five at most", () => { | |
| 5 | const tool = (name: string, pad: number) => ({ | |
| 6 | name, | |
| 7 | description: "d", | |
| 8 | inputSchema: { type: "object", properties: { a: { description: "x".repeat(pad) } } }, | |
| 9 | }); | |
| 10 | const tools = [tool("a", 10), tool("b", 500), tool("c", 50), tool("d", 300), tool("e", 20), tool("f", 100), tool("g", 5)]; | |
| 11 | const out = costliestTools(tools); | |
| 12 | expect(out.split(", ").map((s) => s.split(" ")[0])).toEqual(["b", "d", "f", "c", "e"]); | |
| 13 | expect(out).toMatch(/^b \(\d+ bytes\)/); | |
| 14 | expect(out).toContain(`b (${Buffer.byteLength(JSON.stringify(tools[1]))} bytes)`); | |
| 15 | }); |
test/mcp-size-rank.ts added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | /** Ranks `tools/list` entries by what the whole entry costs on the wire. */ | |
| 2 | export function costliestTools(tools: { name: string }[], n = 5): string { | |
| 3 | return tools | |
| 4 | .map((t) => ({ name: t.name, bytes: Buffer.byteLength(JSON.stringify(t)) })) | |
| 5 | .sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)) | |
| 6 | .slice(0, n) | |
| 7 | .map((t) => `${t.name} (${t.bytes} bytes)`) | |
| 8 | .join(", "); | |
| 9 | } |
test/mcp-size.test.ts +6 −11
| @@ -4,8 +4,8 @@ | ||
| 4 | 4 | * the next cut is obvious. |
| 5 | 5 | * |
| 6 | 6 | * Measure locally with `bun test test/mcp-size.test.ts`. A failure says how |
| 7 | * many bytes the answer is, how many it is over, and the longest | |
| 8 | * descriptions. Before adding a tool, extend an existing one with an | |
| 7 | * many bytes the answer is, how many it is over, and the five costliest | |
| 8 | * tools. Before adding a tool, extend an existing one with an | |
| 9 | 9 | * `action` or `kind` (docs/MCP.md, "Room for a new tool"). The ceilings are |
| 10 | 10 | * the owner's: do not raise them in a feature pull. |
| 11 | 11 | */ |
| @@ -16,6 +16,7 @@ import { join } from "node:path"; | ||
| 16 | 16 | import { createApp, type App } from "../src/app.ts"; |
| 17 | 17 | import { createToken } from "../src/core/tokens.ts"; |
| 18 | 18 | import { createUser } from "../src/core/users.ts"; |
| 19 | import { costliestTools } from "./mcp-size-rank.ts"; | |
| 19 | 20 | import { serve, type Servers } from "../src/http/server.ts"; |
| 20 | 21 | |
| 21 | 22 | const CEILING = 40_000; |
| @@ -95,20 +96,14 @@ test(`tools/list stays under 40 KB, with 3 KB of headroom (${HEADROOM_CEILING.to | ||
| 95 | 96 | // readOnlyHint defaults to false; spelling the default out on every write |
| 96 | 97 | // tool cost 2 KB, so only a read tool carries annotations. |
| 97 | 98 | expect(tools.filter((t) => t.annotations?.readOnlyHint === false).map((t) => t.name)).toEqual([]); |
| 98 | const longest = tools | |
| 99 | .map((t) => `${t.name} (${Buffer.byteLength(t.description)} bytes)`) | |
| 100 | .sort( | |
| 101 | (a, b) => | |
| 102 | Number(/\((\d+) /.exec(b)![1]) - Number(/\((\d+) /.exec(a)![1]), | |
| 103 | ) | |
| 104 | .slice(0, 5); | |
| 99 | const longest = costliestTools(tools); | |
| 105 | 100 | expect( |
| 106 | 101 | bytes, |
| 107 | `tools/list is ${bytes} bytes, ${bytes - CEILING} over the ${CEILING} ceiling; the five longest descriptions: ${longest.join(", ")}`, | |
| 102 | `tools/list is ${bytes} bytes, ${bytes - CEILING} over the ${CEILING} ceiling; the five costliest tools: ${longest}`, | |
| 108 | 103 | ).toBeLessThanOrEqual(CEILING); |
| 109 | 104 | expect( |
| 110 | 105 | bytes, |
| 111 | `tools/list is ${bytes} bytes, ${bytes - HEADROOM_CEILING} over the ${HEADROOM_CEILING} headroom (${CEILING - bytes} left under the ceiling); the five longest descriptions: ${longest.join(", ")}`, | |
| 106 | `tools/list is ${bytes} bytes, ${bytes - HEADROOM_CEILING} over the ${HEADROOM_CEILING} headroom (${CEILING - bytes} left under the ceiling); the five costliest tools: ${longest}`, | |
| 112 | 107 | ).toBeLessThanOrEqual(HEADROOM_CEILING); |
| 113 | 108 | expect( |
| 114 | 109 | tools |
new above
offline
Nit: keep the imports ordered. Put this after the ../src/http/server.ts import.