Public: anyone can read this; the crew can change it.

The MCP `explore` and `search` tools answer with no token, like their HTTP doors #12429

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12406-the-mcp-explore-and-search-tools-answer → main 11 commits fork 1 thread · 1 unresolved

  • Merged. torque/12406-the-mcp-explore-and-search-tools-answer was merged into main as 3d367f5.

Review

Sign in to review this change.
20 files changed +368 −128
docs/MCP.md +11 −1
@@ -242,6 +242,16 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
242242 | `start` | `contribute` | Start building with no account: an account, a repo, a live app and a token in one call | 387 |
243243 <!-- mcp-tools:end -->
244244
245 Four tools need no token, for the same reach as their HTTP doors: `report`,
246 `start`, `explore` and `search` (`TOKEN_FREE_TOOLS` in `src/core/mcp_card.ts`).
247 Called with no token, `explore` and `search` run the functions the HTTP doors
248 call with no account, so they read only what is public (a private repository
249 is absent), and a stranger's code search takes the same per-address slot as
250 `/api/v1/search`. Called with a token they are unchanged and see what the
251 account sees. `tools/list` still needs a token (its 401 is what starts a
252 client's OAuth sign-in); the open tools are named in `/llms.txt`, the server
253 card (`tokenFreeTools`) and `/api/v1/here`.
254
245255 The complete list, with schemas, is MCP `tools/list` called with a token.
246256 `/llms.txt`, under "## MCP tools", lists only the tools open without a token (one line each, from its `doc`) and counts the rest, pointing on to `tools/list`; signed in, it lists every tool.
247257
@@ -406,7 +416,7 @@ longer sentences:
406416 The MCP is the forge's control surface; repo work is git's (clone and push).
407417 `tools/list` opens with the tools that direct and watch a forge: `tell`,
408418 `say`, `next_up`, `now`, `runs`, `knobs`, `usage`, `apps`, `app_deploy`,
409 `forges`, `find_tool`, plus `report` and `start` (`DEFAULT_TOOLS` in
419 `forges`, `find_tool`, and the open tools `report` and `start` are added to it (`DEFAULT_TOOLS` in
410420 `src/http/mcp/catalog.ts`; `api` joins when it exists). `read_file` and
411421 `write_file` are not in it: they are the fallback for an agent with no shell
412422 ("No shell? ..."), reached by `find_tool`, and no diff, blame, tree or
src/actions/here.ts +2
@@ -305,12 +305,14 @@ function forgeActions(
305305 do: "explore apps and repositories",
306306 http: "GET /api/v1/explore?q=",
307307 mcp: "explore",
308 ...tokenFree("explore"),
308309 mcpArgs: { q: "" },
309310 },
310311 {
311312 do: "search issues, pulls, repositories and code",
312313 http: "GET /api/v1/search?q=",
313314 mcp: "search",
315 ...tokenFree("search"),
314316 mcpArgs: { q: "" },
315317 },
316318 { do: "list public repositories", http: "GET /api/v1/repos" },
src/core/mcp_card.ts +2 −2
@@ -58,7 +58,7 @@ export function apiCatalog(origin: string) {
5858 * `http/mcp.ts` answers these and refuses the rest with 401. The card and
5959 * `/api/v1/here` both read this list, so they cannot drift.
6060 */
61 export const TOKEN_FREE_TOOLS: readonly string[] = ["report", "start"];
61 export const TOKEN_FREE_TOOLS: readonly string[] = ["report", "start", "explore", "search"];
6262
6363 /** A thin plugin the forge ships for a harness: where its source is and its one install line. */
6464 export interface HarnessPlugin {
@@ -173,7 +173,7 @@ export function agentCard(origin: string, now: Running = running()) {
173173 capabilities: { streaming: false, pushNotifications: false },
174174 securitySchemes: { oauth: { type: "openIdConnect", openIdConnectUrl: `${origin}/.well-known/openid-configuration` } },
175175 security: [{ oauth: [] }],
176 mcp: { endpoint: url, card: `${origin}${DISCOVERY.card}`, protectedResourceMetadata: oauth },
176 mcp: { endpoint: url, card: `${origin}${DISCOVERY.card}`, protectedResourceMetadata: oauth, tokenFreeTools: [...TOKEN_FREE_TOOLS] },
177177 defaultInputModes: ["application/json"],
178178 defaultOutputModes: ["application/json"],
179179 skills: cardTools.map((t) => ({ id: t.name, name: t.name, description: t.description, tags: ["mcp"] })),
src/http/mcp.ts +58 −7
@@ -31,6 +31,10 @@ import { html } from "../web/html.ts";
3131 import type { Router } from "./router.ts";
3232 import { clientBucket } from "./peer.ts";
3333 import { TOKEN_FREE_TOOLS } from "../core/mcp_card.ts";
34 import { exploreRun, searchRun } from "./mcp/reads.ts";
35 import { greppable, searchKind } from "../core/search.ts";
36 import { strangerCodeSearch } from "./strangers.ts";
37 import { retryAfterOf } from "./respond.ts";
3438 import { proofOf } from "./report_filer.ts";
3539 import { fileReport } from "../actions/reports.ts";
3640 import { START_HINT, START_WORDS, startBuilding } from "../actions/start.ts";
@@ -327,10 +331,55 @@ export const START_TOOL = {
327331 type FreeHandler = (ctx: Ctx, id: unknown, args: Args) => unknown;
328332
329333 /** The gate picks its handler here, so a name listed in TOKEN_FREE_TOOLS without one is caught by a test. */
330 export const TOKEN_FREE_HANDLERS: Readonly<Record<string, FreeHandler>> = { report: reportAnswer, start: startAnswer };
334 export const TOKEN_FREE_HANDLERS: Readonly<Record<string, FreeHandler>> = {
335 report: reportAnswer,
336 start: startAnswer,
337 explore: (ctx, id, args) => readAnswer(ctx, id, "explore", args),
338 search: (ctx, id, args) => readAnswer(ctx, id, "search", args),
339 };
331340
332 /** The JSON-RPC answer to a token-free tool call, or undefined for any other message. */
333 function reportCall(ctx: Ctx, message: unknown): unknown | undefined {
341 /**
342 * The token-free tools that are also ordinary tools (`report` and `start` are
343 * not in TOOLS): a stranger's call goes through the handler (as no one, so
344 * only what is public), a signed-in call goes the ordinary way and sees what
345 * its account sees. Derived from TOKEN_FREE_TOOLS, so there is one list.
346 */
347 const STRANGER_READS: readonly string[] = TOKEN_FREE_TOOLS.filter((n) => TOOLS.some((t) => t.name === n));
348
349 /**
350 * `explore` and `search` for a stranger: the same functions the HTTP doors
351 * call, with no user. A search that greps code takes the same stranger slot
352 * as `/api/v1/search` first.
353 */
354 async function readAnswer(ctx: Ctx, id: unknown, name: string, args: Args): Promise<unknown> {
355 const text = (t: string, isError = false, extra: Record<string, unknown> = {}) => ({
356 jsonrpc: "2.0",
357 id,
358 result: { content: [{ type: "text", text: t }], ...(isError ? { isError } : {}), ...extra },
359 });
360 let held: ReturnType<typeof strangerCodeSearch> | undefined;
361 if (name === "search" && typeof args["q"] === "string") {
362 const kind = searchKind(typeof args["kind"] === "string" ? args["kind"] : null);
363 if ((kind === "code" || kind === "all") && greppable(args["q"])) {
364 held = strangerCodeSearch(ctx.app, ctx.request);
365 if (!held.allowed)
366 return text(`rate_limited: ${held.why}; sign in, or retry in ${retryAfterOf(held.retryAfter)}s`, true);
367 }
368 }
369 try {
370 const answer = await (name === "explore" ? exploreRun : searchRun)(ctx, undefined, args);
371 if (!answer.ok) return text(`${answer.error}: ${answer.message}`, true);
372 return text(JSON.stringify(answer.value), false, cardResult(name, answer.value, ctx.url.origin, answer.card));
373 } finally {
374 if (held?.allowed) held.release();
375 }
376 }
377
378 /**
379 * The JSON-RPC answer to a call of a token-free tool (report, start, explore, search), or
380 * undefined for any other message.
381 */
382 function openToolCall(ctx: Ctx, message: unknown, signedIn: boolean): unknown | undefined {
334383 const m =
335384 typeof message === "object" && message !== null
336385 ? (message as Record<string, unknown>)
@@ -341,7 +390,9 @@ function reportCall(ctx: Ctx, message: unknown): unknown | undefined {
341390 : {};
342391 const name = typeof params["name"] === "string" ? params["name"] : "";
343392 const handler =
344 TOKEN_FREE_TOOLS.includes(name) && Object.hasOwn(TOKEN_FREE_HANDLERS, name)
393 TOKEN_FREE_TOOLS.includes(name) &&
394 !(signedIn && STRANGER_READS.includes(name)) &&
395 Object.hasOwn(TOKEN_FREE_HANDLERS, name)
345396 ? TOKEN_FREE_HANDLERS[name]
346397 : undefined;
347398 if (
@@ -453,13 +504,13 @@ export function mcpRoutes(router: Router<Handler>): void {
453504 (ctx.auth.via === "token" || ctx.auth.via === "password")
454505 ) {
455506 const message: unknown = await body(ctx.request);
456 const reported = reportCall(ctx, message);
507 const reported = openToolCall(ctx, message, true);
457508 if (reported !== undefined) return json(await reported);
458509 const answer = await rpcOne(ctx, user, message);
459510 if (answer === undefined) return new Response(null, { status: 202 });
460511 return json(answer);
461512 }
462 // A stranger may call `report` and nothing else, so their body is read
513 // A stranger may call the token-free tools and nothing else, so their body is read
463514 // to a hard byte limit and no further.
464515 const message = await boundedBody(ctx.request, REQUEST_MAX);
465516 if (message === "too_large")
@@ -467,7 +518,7 @@ export function mcpRoutes(router: Router<Handler>): void {
467518 { error: "too_large", message: `at most ${REQUEST_MAX} bytes without a token` },
468519 413,
469520 );
470 const reported = reportCall(ctx, message);
521 const reported = openToolCall(ctx, message, false);
471522 if (reported !== undefined) return json(await reported);
472523 // The refusal keeps the OAuth challenge a client starts from, and names
473524 // the one thing a stranger can do without signing in.
src/http/mcp/reads.ts added +104
@@ -0,0 +1,104 @@
1 /**
2 * What `explore` and `search` answer, for a signed-in caller and for a
3 * stranger alike: `user` is undefined for a stranger, who reads only what is
4 * public, as `/api/v1/explore` and `/api/v1/search` do (the same functions).
5 */
6 import type { Database } from "bun:sqlite";
7 import { getRepo, type Repo } from "../../core/repos.ts";
8 import { getIssue, type Issue } from "../../core/issues.ts";
9 import { markReaders } from "../../crew/direct.ts";
10 import { SEARCH_ROWS as CARD_HITS } from "../../web/studio/cards.ts";
11 import { search, searchRefusal } from "../../actions/search.ts";
12 import { CODE_BUDGET_MS, searchKind, searchNote, type SearchResults } from "../../core/search.ts";
13 import { explorePage, exploreLine, exploreCount, exploreFiltered, exploreLineFrom, exploreNote, exploreSuggestion, readExploreLine, rowsWithWhy } from "../../actions/explore.ts";
14 import { forkTreeRows } from "../../actions/fork_tree_rows.ts";
15 import type { User } from "../../core/users.ts";
16 import type { Ctx } from "../context.ts";
17 import { type Answer, type Args, bad, good, str, repoArg, isAnswer } from "./shared.ts";
18
19 export async function exploreRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> {
20 if (str(args["kind"]) === "tree") {
21 const repo = repoArg(ctx, user, { repo: args["app"] });
22 return isAnswer(repo) ? repo : good(await forkTreeRows(ctx.app, repo, user));
23 }
24 // The stranger path skips the tool schema, so the lengths are checked here for both.
25 for (const key of ["q", "query", "cursor"])
26 if (args[key] !== undefined && (typeof args[key] !== "string" || (args[key] as string).length > 200))
27 return bad("invalid", `\`${key}\` is a string of at most 200 characters`);
28 const q = str(args["q"]);
29 const line = exploreLineFrom((name) => (name === "q" ? (q !== "" ? q : str(args["query"])) : str(args[name])));
30 const { query, unknown } = readExploreLine(line);
31 const suggest = exploreSuggestion(line);
32 if (unknown.length > 0) return bad("unknown_filter", exploreNote(unknown) + (suggest === undefined ? "" : `; did you mean "${suggest.line}"?`));
33 const cursor = str(args["cursor"]);
34 const page = await explorePage(ctx.app, user, query, cursor === "" ? null : cursor);
35 if (page === null) return bad("invalid_cursor", "cursor is not the next_cursor of a page before");
36 return good({
37 query: exploreLine(query),
38 total: page.total,
39 line: exploreCount(query, page.total),
40 ...(exploreFiltered(query) ? { clear: { tool: "explore", arguments: {} } } : {}),
outdated

Stray blank line before the closing brace. Also validate q and cursor length here, since the stranger path does not go through the tool schema.

41 items: rowsWithWhy(ctx.app, query, page.items),
42 next_cursor: page.nextCursor,
43 });
44 }
45
46 export async function searchRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> {
47 const q = args["q"];
48 if (typeof q !== "string" || q.length > 200) return bad("invalid", "`q` is a string of at most 200 characters");
49 const kind = args["kind"];
50 if (kind !== undefined && typeof kind !== "string") return bad("invalid_kind", "kind is all, repos, issues or code");
51 const refusal = searchRefusal(kind, args["limit"] === undefined || typeof args["limit"] === "number" ? (args["limit"] as number | undefined) : NaN);
52 if (refusal) return bad(refusal.code, refusal.message);
53 const all = args["all"];
54 if (all !== undefined && typeof all !== "boolean") return bad("invalid", "`all` is true or false");
55 let only: Repo | undefined;
56 if (args["repo"] !== undefined) {
57 if (typeof args["repo"] !== "string") return bad("invalid", "`repo` is owner/name");
58 const repo = repoArg(ctx, user, args);
59 if (isAnswer(repo)) return repo;
60 only = repo;
61 }
62 const note = searchNote(q, searchKind(kind ?? null));
63 const found = await search(ctx.app, user, {
64 q,
65 ...(kind ? { kind: searchKind(kind) } : {}),
66 ...(typeof args["limit"] === "number" ? { limit: args["limit"] } : {}),
67 ...(only ? { repo: only } : {}),
68 ...(all === true ? {} : { codeBudgetMs: ctx.app.config.codeBudgetMs ?? CODE_BUDGET_MS }),
69 });
70 const answer = { ok: true as const, value: note === undefined ? found : { ...found, note } };
71 return ctx.auth.through === undefined ? { ...answer, card: { q, ...cardMarks(ctx.app.db, found) } } : answer;
72 }
73
74 /**
75 * What the card needs beyond the hits: which local issue and pull hits an outsider wrote
76 * (`untrusted`, as "owner/name#number", by the same `markReaders` rule `issues` uses) and the
77 * default branch of each repository with a code hit, so a code row links to its file.
78 */
79 function cardMarks(db: Database, found: SearchResults): Record<string, unknown> {
80 const repos = new Map<string, Repo | undefined>();
81 const repoOf = (owner: string, name: string): Repo | undefined => {
82 const key = `${owner}/${name}`;
83 if (!repos.has(key)) repos.set(key, getRepo(db, owner, name));
84 return repos.get(key);
85 };
86 const byRepo = new Map<Repo, Issue[]>();
87 for (const h of found.issues.slice(0, CARD_HITS)) {
88 if (h.source !== "local") continue;
89 const repo = repoOf(h.repo.owner, h.repo.name);
90 const issue = repo === undefined ? undefined : getIssue(db, repo.id, h.number);
91 if (repo === undefined || issue === undefined) continue;
92 byRepo.set(repo, [...(byRepo.get(repo) ?? []), issue]);
93 }
94 const untrusted: string[] = [];
95 for (const [repo, issues] of byRepo)
96 for (const m of markReaders(db, repo, issues))
97 if (m.untrusted !== undefined) untrusted.push(`${repo.ownerName}/${repo.name}#${m.number}`);
98 const branches: Record<string, string> = {};
99 for (const c of found.code.slice(0, CARD_HITS)) {
100 const repo = repoOf(c.repo.owner, c.repo.name);
101 if (repo !== undefined) branches[`${c.repo.owner}/${c.repo.name}`] = repo.defaultBranch;
102 }
103 return { untrusted, branches };
104 }
src/http/mcp/shared.ts +1 −1
@@ -259,7 +259,7 @@ export const str = (v: unknown): string => (typeof v === "string" ? v.trim() : "
259259 export const REPO_ARG = { type: "string" } as const;
260260
261261 /** A repository the caller may read, by `owner/name`; a wrong one is answered with that form. */
262 export function repoArg(ctx: Ctx, user: User, args: Args): Repo | Answer {
262 export function repoArg(ctx: Ctx, user: User | undefined, args: Args): Repo | Answer {
263263 const [owner, name, ...rest] = str(args["repo"]).split("/");
264264 const repo =
265265 owner && name && rest.length === 0
src/http/mcp/tools/explore.ts +5 −25
@@ -1,6 +1,6 @@
1 import { explorePage, exploreLine, exploreCount, exploreFiltered, exploreLineFrom, EXPLORE_PARAMS, exploreNote, exploreSuggestion, readExploreLine, rowsWithWhy } from "../../../actions/explore.ts";
2 import { forkTreeRows } from "../../../actions/fork_tree_rows.ts";
3 import { bad, good, type Tool, str, repoArg, isAnswer } from "../shared.ts";
1 import { EXPLORE_PARAMS } from "../../../actions/explore.ts";
2 import type { Tool } from "../shared.ts";
3 import { exploreRun } from "../reads.ts";
44
55 export const tool: Tool = {
66 name: "explore",
@@ -25,30 +25,10 @@ export const tool: Tool = {
2525 app: { type: "string", maxLength: 200, description: "With kind: tree" },
2626 cursor: {
2727 type: "string",
28 maxLength: 200,
2829 },
2930 },
3031 },
3132 writes: false,
32 async run(ctx, user, args) {
33 if (str(args["kind"]) === "tree") {
34 const repo = repoArg(ctx, user, { repo: args["app"] });
35 return isAnswer(repo) ? repo : good(await forkTreeRows(ctx.app, repo, user));
36 }
37 const q = str(args["q"]);
38 const line = exploreLineFrom((name) => (name === "q" ? (q !== "" ? q : str(args["query"])) : str(args[name])));
39 const { query, unknown } = readExploreLine(line);
40 const suggest = exploreSuggestion(line);
41 if (unknown.length > 0) return bad("unknown_filter", exploreNote(unknown) + (suggest === undefined ? "" : `; did you mean "${suggest.line}"?`));
42 const cursor = str(args["cursor"]);
43 const page = await explorePage(ctx.app, user, query, cursor === "" ? null : cursor);
44 if (page === null) return bad("invalid_cursor", "cursor is not the next_cursor of a page before");
45 return good({
46 query: exploreLine(query),
47 total: page.total,
48 line: exploreCount(query, page.total),
49 ...(exploreFiltered(query) ? { clear: { tool: "explore", arguments: {} } } : {}),
50 items: rowsWithWhy(ctx.app, query, page.items),
51 next_cursor: page.nextCursor,
52 });
53 },
33 run: (ctx, user, args) => exploreRun(ctx, user, args),
5434 };
src/http/mcp/tools/search.ts +3 −67
@@ -1,11 +1,5 @@
1 import type { Database } from "bun:sqlite";
2 import { getRepo, type Repo } from "../../../core/repos.ts";
3 import { getIssue, type Issue } from "../../../core/issues.ts";
4 import { markReaders } from "../../../crew/direct.ts";
5 import { SEARCH_ROWS as CARD_HITS } from "../../../web/studio/cards.ts";
6 import { search, searchRefusal } from "../../../actions/search.ts";
7 import { CODE_BUDGET_MS, searchKind, searchNote, type SearchResults } from "../../../core/search.ts";
8 import { good, bad, type Tool, repoArg, isAnswer } from "../shared.ts";
1 import type { Tool } from "../shared.ts";
2 import { searchRun } from "../reads.ts";
93
104 export const tool: Tool = {
115 name: "search",
@@ -24,63 +18,5 @@ export const tool: Tool = {
2418 required: ["q"],
2519 },
2620 writes: false,
27 async run(ctx, user, args) {
28 const q = args["q"];
29 if (typeof q !== "string" || q.length > 200) return bad("invalid", "`q` is a string of at most 200 characters");
30 const kind = args["kind"];
31 if (kind !== undefined && typeof kind !== "string") return bad("invalid_kind", "kind is all, repos, issues or code");
32 const refusal = searchRefusal(kind, args["limit"] === undefined || typeof args["limit"] === "number" ? (args["limit"] as number | undefined) : NaN);
33 if (refusal) return bad(refusal.code, refusal.message);
34 const all = args["all"];
35 if (all !== undefined && typeof all !== "boolean") return bad("invalid", "`all` is true or false");
36 let only: Repo | undefined;
37 if (args["repo"] !== undefined) {
38 if (typeof args["repo"] !== "string") return bad("invalid", "`repo` is owner/name");
39 const repo = repoArg(ctx, user, args);
40 if (isAnswer(repo)) return repo;
41 only = repo;
42 }
43 const note = searchNote(q, searchKind(kind ?? null));
44 const found = await search(ctx.app, user, {
45 q,
46 ...(kind ? { kind: searchKind(kind) } : {}),
47 ...(typeof args["limit"] === "number" ? { limit: args["limit"] } : {}),
48 ...(only ? { repo: only } : {}),
49 ...(all === true ? {} : { codeBudgetMs: ctx.app.config.codeBudgetMs ?? CODE_BUDGET_MS }),
50 });
51 const answer = good(note === undefined ? found : { ...found, note });
52 return ctx.auth.through === undefined ? { ...answer, card: { q, ...cardMarks(ctx.app.db, found) } } : answer;
53 },
21 run: (ctx, user, args) => searchRun(ctx, user, args),
5422 };
55
56 /**
57 * What the card needs beyond the hits: which local issue and pull hits an outsider wrote
58 * (`untrusted`, as "owner/name#number", by the same `markReaders` rule `issues` uses) and the
59 * default branch of each repository with a code hit, so a code row links to its file.
60 */
61 function cardMarks(db: Database, found: SearchResults): Record<string, unknown> {
62 const repos = new Map<string, Repo | undefined>();
63 const repoOf = (owner: string, name: string): Repo | undefined => {
64 const key = `${owner}/${name}`;
65 if (!repos.has(key)) repos.set(key, getRepo(db, owner, name));
66 return repos.get(key);
67 };
68 const byRepo = new Map<Repo, Issue[]>();
69 for (const h of found.issues.slice(0, CARD_HITS)) {
70 if (h.source !== "local") continue;
71 const repo = repoOf(h.repo.owner, h.repo.name);
72 const issue = repo === undefined ? undefined : getIssue(db, repo.id, h.number);
73 if (repo === undefined || issue === undefined) continue;
74 byRepo.set(repo, [...(byRepo.get(repo) ?? []), issue]);
75 }
76 const untrusted: string[] = [];
77 for (const [repo, issues] of byRepo)
78 for (const m of markReaders(db, repo, issues))
79 if (m.untrusted !== undefined) untrusted.push(`${repo.ownerName}/${repo.name}#${m.number}`);
80 const branches: Record<string, string> = {};
81 for (const c of found.code.slice(0, CARD_HITS)) {
82 const repo = repoOf(c.repo.owner, c.repo.name);
83 if (repo !== undefined) branches[`${c.repo.owner}/${c.repo.name}`] = repo.defaultBranch;
84 }
85 return { untrusted, branches };
86 }
src/web/llms_tools.ts +8 −3
@@ -36,7 +36,9 @@ export const toolsSection = (tools: readonly Listed[]): string[] => [
3636 export const anonymousToolsSection = (open: readonly Listed[], more: number, origin: string): string[] => [
3737 "## MCP tools",
3838 "",
39 ...open.map((t) => toolLine(t)),
39 // A read (explore, search) is told by its short description here: the anonymous text
40 // stays small, and the signed-in one has the full `doc`.
41 ...open.map((t) => toolLine(t.writes ? t : { ...t, doc: t.description })),
4042 `- ${more} more tools with an account: call tools/list on ${origin}/mcp once signed in, or read ${origin}/contract/openapi.json.`,
4143 "",
4244 ];
@@ -47,8 +49,11 @@ let toolCount = 0;
4749 let open: readonly Listed[] = [];
4850 let section: readonly string[] = [];
4951 export const setMcpTools = (all: readonly Listed[], tokenFree: readonly Listed[] = []): void => {
50 toolCount = all.length;
51 open = tokenFree;
52 // The open tools in TOOLS (explore, search) are listed with `report` and `start`,
53 // not counted among the rest.
54 const free = all.filter((t) => TOKEN_FREE_TOOLS.includes(t.name));
55 toolCount = all.length - free.length;
56 open = [...tokenFree, ...free];
5257 section = toolsSection([...all, ...tokenFree]);
5358 };
5459 /** The signed-in reader's section is the whole index; a stranger's is folded. */
src/web/standards.ts +2 −2
@@ -511,8 +511,8 @@ function renderLlmsTxt(ctx: Ctx, o: string, tell: string | undefined, running: s
511511 ]),
512512 "## Find things (no account)",
513513 "",
514 `- Explore repositories: GET ${o}/api/v1/explore?q=kind:app+sort:forks+garden. (MCP "explore": with an account.) In q, kind: is app, person or forge; sort: is score, forks, ships, likes, use, fresh or name; updated: is 1d, 7d or 30d; owner:<name> is one account; archived: is yes or no; forks:N and ships:N are at least N (0-1000); live:yes is awake in 30 days; any other word is free text matched against name, owner and description.`,
515 `- Search issues, pulls, repositories and code: GET ${o}/api/v1/search?q=websocket+timeout. (MCP "search": with an account.)`,
514 `- Explore repositories: GET ${o}/api/v1/explore?q=kind:app+sort:forks+garden. (MCP "explore": no account.) In q, kind: is app, person or forge; sort: is score, forks, ships, likes, use, fresh or name; updated: is 1d, 7d or 30d; owner:<name> is one account; archived: is yes or no; forks:N and ships:N are at least N (0-1000); live:yes is awake in 30 days; any other word is free text matched against name, owner and description.`,
515 `- Search issues, pulls, repositories and code: GET ${o}/api/v1/search?q=websocket+timeout. (MCP "search": no account.)`,
516516 "- Both read only what is public to you.",
517517 `- A pull's check as text: GET ${o}/{owner}/{name}/pulls/{number}/check.txt (pending, green or red; head: <oid>).`,
518518 "",
test/deadends.test.ts +1 −1
@@ -209,7 +209,7 @@ test("the findings are the known ones, and every known one still happens", async
209209 const KNOWN_WORDS: readonly string[] = [
210210 // The scan reads "the MCP tool spawn" and "the one tool open without a token" as a tool called "the" and "one".
211211 "tool the at src/core/roles.ts:53",
212 "tool one at src/http/mcp.ts:279",
212 "tool one at src/http/mcp.ts:283",
213213 // Dead: a preview's sample app runs a file its repository does not hold.
214214 "script src/server.ts at src/spawn/preview-seed.ts:816",
215215 ];
test/fuzz-mcp.test.ts +8 −1
@@ -245,7 +245,7 @@ describe("MCP fuzz: hostile arguments get a named refusal", () => {
245245
246246 test("anonymous reaches exactly the token-free tools, and each refuses hostile arguments by name, leaving nothing behind", async () => {
247247 const free = Object.keys(TOKEN_FREE_HANDLERS).sort();
248 expect(free).toEqual(["report", "start"]);
248 expect(free).toEqual(["explore", "report", "search", "start"]);
249249 const reached: string[] = [];
250250 for (const name of toolNames()) {
251251 const a = await call(undefined, name, { repo: "../../etc", title: BIG, __proto__: { admin: true } });
@@ -271,6 +271,13 @@ describe("MCP fuzz: hostile arguments get a named refusal", () => {
271271 const label = `${name} ${JSON.stringify(args)?.slice(0, 160)}`;
272272 const before = counts();
273273 const a = await call(undefined, name, args);
274 // A body past the stranger cap (65536 bytes; explore and search take many
275 // string arguments, each hostile one 8 KiB) is refused by the door, by name.
276 if (a.status === 413) {
277 expect([label, a.body.error]).toEqual([label, "too_large"]);
278 expect([label, counts()]).toEqual([label, before]);
279 continue;
280 }
274281 expect([label, flaw(a)]).toEqual([label, undefined]);
275282 const refused = a.body.error !== undefined || a.body.result?.isError === true;
276283 if (refused) expect([label, counts()]).toEqual([label, before]);
test/here-manifest.test.ts +1 −1
@@ -347,7 +347,7 @@ describe("the manifest: what you can do here, as who you are", () => {
347347 );
348348 for (const a of f.can)
349349 if (a.mcp === "explore" || a.mcp === "search")
350 expect(a.account).toBeUndefined();
350 expect(a.account).toBe(false);
351351 }
352352 const q = (await manifest("/search?q=a%20b%26c")).body;
353353 expect(q.can[0].http).toBe("GET /api/v1/search?q=a%20b%26c");
test/llms-anonymous.test.ts +13 −9
@@ -43,36 +43,40 @@ const section = (body: string): string[] => {
4343 return lines.slice(start + 1, end).filter((l) => l.startsWith("- "));
4444 };
4545
46 test("the anonymous text is small, varies by reader, and lists report and start plus one line", async () => {
46 test("the anonymous text is small, varies by reader, and lists the open tools plus one line", async () => {
4747 const res = await fetch(`${servers.url}/llms.txt`);
4848 const body = await res.text();
4949 // 6.5 KiB until the three doors (#12015, about 440 bytes) opened the text,
5050 // then the guide line (#12069, about 100 bytes); nothing else was cut to make room.
51 // 7.5 KiB since the check.txt door's line.
52 expect(Buffer.byteLength(body)).toBeLessThan(7.5 * 1024);
51 // 7.5 KiB since the check.txt door's line, and since explore and search (#12406)
52 // moved from counted to listed: 8 KiB.
53 expect(Buffer.byteLength(body)).toBeLessThan(8 * 1024);
5354 expect(res.headers.get("vary")).toContain("Cookie");
5455 expect(res.headers.get("vary")).toContain("Authorization");
5556 const listed = section(body);
56 expect(listed).toHaveLength(3);
57 expect(listed).toHaveLength(5);
5758 expect(listed[0]).toStartWith("- report: ");
5859 expect(listed[1]).toStartWith("- start: ");
59 expect(listed[2]).toContain(`${TOOLS.length} more tools with an account`);
60 expect(listed[2]).toContain("tools/list");
61 expect(listed[2]).toContain("/contract/openapi.json");
60 expect(listed[2]).toStartWith("- explore: ");
61 expect(listed[3]).toStartWith("- search: ");
62 // The open tools in the index are listed, so the rest is counted without them.
63 expect(listed[4]).toContain(`${TOOLS.length - 2} more tools with an account`);
64 expect(listed[4]).toContain("tools/list");
65 expect(listed[4]).toContain("/contract/openapi.json");
6266 });
6367
6468 test("every tool the anonymous text names is one tools/list returns", async () => {
6569 const names = section(await fetch(`${servers.url}/llms.txt`).then((r) => r.text()))
6670 .map((l) => /^- ([a-z_]+): /.exec(l)?.[1])
6771 .filter((n): n is string => n !== undefined);
68 expect(names).toEqual(["report", "start"]);
72 expect(names).toEqual(["report", "start", "explore", "search"]);
6973 const returned: string[] = [];
7074 let cursor: string | undefined;
7175 do {
7276 const list = await fetch(`${servers.url}/mcp`, {
7377 method: "POST",
7478 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
75 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", ...(cursor === undefined ? {} : { params: { cursor } }) }),
79 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: { all: true, ...(cursor === undefined ? {} : { cursor }) } }),
7680 }).then((r) => r.json() as Promise<any>);
7781 returned.push(...list.result.tools.map((t: { name: string }) => t.name));
7882 cursor = list.result.nextCursor;
test/llms-find.test.ts +2 −2
@@ -15,8 +15,8 @@ test("llms.txt names explore and search, and an empty forge says None yet.", asy
1515 expect(body).toContain("## Find things (no account)");
1616 expect(body).toContain("/api/v1/explore?q=kind:app+sort:forks+garden");
1717 expect(body).toContain("/api/v1/search?q=");
18 expect(body).toContain('MCP "explore": with an account');
19 expect(body).toContain('MCP "search": with an account');
18 expect(body).toContain('MCP "explore": no account');
19 expect(body).toContain('MCP "search": no account');
2020 expect(body).toMatch(/## Public repositories\n\nNone yet\.\n/);
2121 } finally {
2222 servers.stop();
test/llms-tools.test.ts +2 −1
@@ -1,4 +1,5 @@
11 /** /llms.txt's `## MCP tools` section is the generated tool index and `report`, line for line, each by its `doc`. */
2 import { TOKEN_FREE_TOOLS } from "../src/core/mcp_card.ts";
23 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
34 import { mkdtemp, rm } from "node:fs/promises";
45 import { tmpdir } from "node:os";
@@ -47,7 +48,7 @@ describe("/llms.txt MCP tools", () => {
4748 const listed = lines.slice(start + 1, end).filter((l) => l.startsWith("- "));
4849 const expected = [...TOOLS, REPORT_TOOL, START_TOOL].map((t: Listed) => {
4950 const admin = t.adminOnly === true || t.needsAdminScope === true ? " (admin)" : "";
50 return `- ${t.name}: ${(t.doc ?? t.description).replace(/\s+/g, " ").trim()}${admin}${t.writes ? " (writes)" : ""}${t.name === "report" || t.name === "start" ? " (no account)" : ""}`;
51 return `- ${t.name}: ${(t.doc ?? t.description).replace(/\s+/g, " ").trim()}${admin}${t.writes ? " (writes)" : ""}${TOKEN_FREE_TOOLS.includes(t.name) ? " (no account)" : ""}`;
5152 });
5253 expect(listed).toEqual(expected);
5354 expect(TOOLS.some((t) => t.writes)).toBe(true);
test/mcp-card.test.ts +1 −1
@@ -42,7 +42,7 @@ describe("the MCP server card", () => {
4242 expect(card.version).toBe(running().version);
4343 expect(card.transport).toBe("streamable-http");
4444 expect(card.endpoint).toBe(`${base}/mcp`);
45 expect(card.tokenFreeTools).toEqual(["report", "start"]);
45 expect(card.tokenFreeTools).toEqual(["report", "start", "explore", "search"]);
4646
4747 // Without a token the endpoint refuses everything but `report`.
4848 const report = await fetch(card.endpoint, {
test/mcp-open-explore.test.ts added +140
@@ -0,0 +1,140 @@
1 /**
2 * `explore` and `search` over MCP answer with no token, as their HTTP doors
3 * do, reading only what is public; with a token the answer is unchanged.
4 */
5 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createRepo } from "../src/actions/repos.ts";
11 import { createIssue } from "../src/core/issues.ts";
12 import { createToken } from "../src/core/tokens.ts";
13 import { createApp as insertApp } from "../src/core/apps.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { TOKEN_FREE_TOOLS } from "../src/core/mcp_card.ts";
16 import { serve, type Servers } from "../src/http/server.ts";
17 import { strangerCodeSearch } from "../src/http/strangers.ts";
18
19 describe("open explore and search over MCP", () => {
20 let dataDir: string;
21 let app: App;
22 let servers: Servers;
23 let aliceRw: string;
24 let id = 1;
25
26 const call = async (token: string | null, name: string, args: unknown) => {
27 const res = await fetch(`${servers.url}/mcp`, {
28 method: "POST",
29 headers: {
30 "content-type": "application/json",
31 ...(token === null ? {} : { authorization: `Bearer ${token}` }),
32 },
33 body: JSON.stringify({ jsonrpc: "2.0", id: id++, method: "tools/call", params: { name, arguments: args } }),
34 });
35 const body: any = await res.json();
36 const text: string = body.result.content[0].text;
37 return { status: res.status, ...(body.result.isError === true ? { error: text } : { value: JSON.parse(text) }) } as any;
38 };
39
40 beforeAll(async () => {
41 dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-open-explore-"));
42 app = createApp({ dataDir, port: 0 });
43 servers = await serve(app);
44 const u = await createUser(app.db, {
45 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
46 password: "a-good-passphrase", adminIfFirst: true,
47 });
48 if (!u.ok) throw new Error(u.error);
49 aliceRw = createToken(app.db, u.value.id, "rw", "rw").token;
50 for (const [name, visibility] of [["openpebble", "public"], ["hiddenpebble", "private"]] as const) {
51 const made = await createRepo(app, u.value, { name, visibility });
52 if (!made.ok) throw new Error(String(made.error));
53 if (!insertApp(app.db, { repoId: made.value.id, slug: `alice-${name}`, access: visibility === "public" ? "public" : "private" }).ok) throw new Error("app");
54 createIssue(app.db, { repoId: made.value.id, authorId: u.value.id, title: `Polish the ${name} glow` });
55 }
56 }, 30_000);
57
58 afterAll(async () => {
59 servers.stop();
60 app.close();
61 await rm(dataDir, { recursive: true, force: true });
62 });
63
64 test("the open list names them, and the card prints it", async () => {
65 expect(TOKEN_FREE_TOOLS).toContain("explore");
66 expect(TOKEN_FREE_TOOLS).toContain("search");
67 const card: any = await (await fetch(`${servers.url}/.well-known/mcp.json`)).json();
68 expect(card.tokenFreeTools).toContain("explore");
69 expect(card.tokenFreeTools).toContain("search");
70 const agent: any = await (await fetch(`${servers.url}/.well-known/agent-card.json`)).json();
71 expect(agent.mcp.tokenFreeTools).toEqual(card.tokenFreeTools);
72 });
73
74 test("explore with no token lists the public app and not the private one", async () => {
75 const r = await call(null, "explore", { q: "pebble" });
76 const names = JSON.stringify(r.value.items);
77 expect(names).toContain("openpebble");
78 expect(names).not.toContain("hiddenpebble");
79 });
80
81 test("search with no token finds the public issue and not the private one", async () => {
82 const r = await call(null, "search", { q: "glow", kind: "issues" });
83 expect(r.value.issues.map((i: any) => i.repo.name)).toEqual(["openpebble"]);
84 const named = await call(null, "search", { q: "glow", repo: "alice/hiddenpebble" });
85 expect(named.error).toContain("no such app or repository");
86 });
87
88 test("a stranger's bad arguments stop at the boundary", async () => {
89 expect((await call(null, "search", { q: "x".repeat(201) })).error).toContain("invalid");
90 expect((await call(null, "explore", { q: "sort:sideways" })).error).toContain("unknown_filter");
91 });
92
93 test("signed in, the answers are unchanged: the private repository is there", async () => {
94 const s = await call(aliceRw, "search", { q: "glow", kind: "issues" });
95 expect(s.value.issues.map((i: any) => i.repo.name).sort()).toEqual(["hiddenpebble", "openpebble"]);
96 const e = await call(aliceRw, "explore", { q: "pebble" });
97 expect(JSON.stringify(e.value.items)).toContain("hiddenpebble");
98 });
99
100 test("llms.txt lists them in full to a stranger and counts only the rest", async () => {
101 const text = await (await fetch(`${servers.url}/llms.txt`)).text();
102 expect(text).toMatch(/- explore: .*\(no account\)/);
103 expect(text).toMatch(/- search: .*\(no account\)/);
104 expect(text).toMatch(/- \d+ more tools with an account/);
105 });
106
107 test("tools other than the open ones still answer 401", async () => {
108 const res = await fetch(`${servers.url}/mcp`, {
109 method: "POST",
110 headers: { "content-type": "application/json" },
111 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "tell", arguments: {} } }),
112 });
113 expect(res.status).toBe(401);
114 });
115
116 test("explore refuses a q, query or cursor over 200 characters the same way with and without a token", async () => {
117 for (const key of ["q", "query", "cursor"]) {
118 for (const token of [null, aliceRw]) {
119 const r = await call(token, "explore", { [key]: "x".repeat(201) });
120 expect(r.error).toContain(`\`${key}\` is a string of at most 200 characters`);
121 }
122 }
123 });
124
125 test("a stranger's code search takes the stranger slot and is refused while it is held; a signed-in one is not", async () => {
126 const held = strangerCodeSearch(app, new Request(`${servers.url}/x`));
127 expect(held.allowed).toBe(true);
128 try {
129 const refused = await call(null, "search", { q: "glow", kind: "code" });
130 expect(refused.error).toContain("rate_limited");
131 const other = await call(null, "search", { q: "glow", kind: "issues" });
132 expect(other.error).toBeUndefined();
133 const signed = await call(aliceRw, "search", { q: "glow", kind: "code" });
134 expect(signed.error).toBeUndefined();
135 } finally {
136 if (held.allowed) held.release();
137 }
138 expect((await call(null, "search", { q: "glow", kind: "code" })).error).toBeUndefined();
139 });
140 });
test/standards.test.ts +3 −3
@@ -259,15 +259,15 @@ describe("what the open web expects", () => {
259259 expect(res.status).not.toBe(401);
260260 }
261261 const find = body.split("## Find things (no account)")[1]!.split("\n## ")[0]!;
262 expect(find).toContain("with an account");
262 expect(find).not.toContain("with an account");
263263 expect(find).not.toContain("MCP: the tool");
264 expect(find).toContain('MCP "search": with an account');
264 expect(find).toContain('MCP "search": no account');
265265 const probe = await fetch(`${base}/mcp`, {
266266 method: "POST",
267267 headers: { "content-type": "application/json" },
268268 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "explore", arguments: { q: "" } } }),
269269 });
270 expect(probe.status).toBe(401);
270 expect(probe.status).not.toBe(401);
271271 });
272272
273273 test("llms.txt says how to use the forge from a harness, with the endpoint built from the URL", async () => {
test/token-free-tools.test.ts +1 −1
@@ -72,7 +72,7 @@ describe("token-free MCP tools", () => {
7272 }
7373 }
7474 expect(free).toContain(REPORT_TOOL.name);
75 expect(gated.sort()).toEqual(["explore", "here", "search"]);
75 expect(gated.sort()).toEqual(["here"]);
7676 for (const name of free) {
7777 expect({ name, status: (await call(name)).status }).not.toEqual({ name, status: 401 });
7878 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs