Public: anyone can read this; the crew can change it.

An outsider's first MCP call is timed: start, contribute and find_tool each answer under 50 ms warm, held by a test #14716

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14703-an-outsider-s-first-mcp-call-is-timed-st → main 1 commit fork

  • Merged. scout/14703-an-outsider-s-first-mcp-call-is-timed-st was merged into main as 532ee62.

Review

Sign in to review this change.
1 file changed +82 −0
test/mcp-first-calls-timing.test.ts added +82
@@ -0,0 +1,82 @@
1 /**
2 * An outsider's first MCP calls are instant (#14703): contribute() and find_tool() answer warm,
3 * ten times each, with a median under MEDIAN_MS and the worst under WORST_MS, and each response
4 * names its time in Server-Timing (set on every response by the server, so
5 * nothing new is added; src/http/measure.ts is the page-view beacon, not this).
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { createToken } from "../src/core/tokens.ts";
13 import { createUser } from "../src/core/users.ts";
14 import { serve, type Servers } from "../src/http/server.ts";
15 import { quietLogs } from "./helpers/globals.ts";
16
17 quietLogs();
18
19 const MEDIAN_MS = 50;
20 const WORST_MS = 250; // a loaded machine may stall one call; a slow query shows in the median.
21 const RUNS = 10;
22
23 let dir: string;
24 let app: App;
25 let servers: Servers;
26 let token: string;
27
28 const call = (name: string, args: unknown, bearer?: string) =>
29 fetch(`${servers.url}/mcp`, {
30 method: "POST",
31 headers: {
32 "content-type": "application/json",
33 accept: "application/json, text/event-stream",
34 ...(bearer ? { authorization: `Bearer ${bearer}` } : {}),
35 },
36 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }),
37 });
38
39 beforeAll(async () => {
40 dir = await mkdtemp(join(tmpdir(), "forge-first-calls-"));
41 app = createApp({ dataDir: dir, port: 0 });
42 servers = await serve(app);
43 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "ada", email: "[email protected]", password: "a-good-passphrase" });
44 if (!u.ok) throw new Error("user");
45 token = createToken(app.db, u.value.id, "ro", "ro").token;
46 }, 30_000);
47
48 afterAll(async () => {
49 servers.stop();
50 app.db.close();
51 await rm(dir, { recursive: true, force: true });
52 });
53
54 async function timed(name: string, args: unknown, bearer?: string): Promise<void> {
55 const text = async () => {
56 const res = await call(name, args, bearer);
57 const body = (await res.json()) as any;
58 expect(res.status, name).toBe(200);
59 expect(body.result.isError, name).not.toBe(true);
60 return res;
61 };
62 const warm = await text();
63 expect(warm.headers.get("server-timing"), name).toMatch(/dur=\d/);
64 const ms: number[] = [];
65 for (let i = 0; i < RUNS; i++) {
66 const t = performance.now();
67 await text();
68 ms.push(performance.now() - t);
69 }
70 ms.sort((a, b) => a - b);
71 const median = (ms[RUNS / 2 - 1]! + ms[RUNS / 2]!) / 2;
72 expect(median, `${name} median ${median.toFixed(1)} ms`).toBeLessThan(MEDIAN_MS);
73 expect(ms[RUNS - 1]!, `${name} worst ${ms[RUNS - 1]!.toFixed(1)} ms`).toBeLessThan(WORST_MS);
74 }
75
76 test("contribute() answers warm under the ceiling", async () => {
77 await timed("contribute", {});
78 });
79
80 test("find_tool() answers warm under the ceiling", async () => {
81 await timed("find_tool", { query: "issue" }, token);
82 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs