Public: anyone can read this; the crew can change it.
The Muse test is a test: an outside agent on a public app finds the report route from the app's own headers and files a report with no token #10408
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10383-the-muse-test-is-a-test-an-outside-agent → main 2 commits fork 3 threads · 3 unresolved
2 files
changed +305
−0
docs/features/muse-test.md added +22
| @@ -0,0 +1,22 @@ | ||
| 1 | # The Muse test: how an outsider reports a problem | |
| 2 | ||
| 3 | An agent or person with no account, looking at a public app, needs no teaching. | |
| 4 | ||
| 5 | 1. **The door says it.** Every HTML page of a public app carries | |
| 6 | `Link: </__forge/report>; rel="help"` and a `rel="alternate"` to the forge's | |
| 7 | manifest for the repository (`/api/v1/here?path=/owner/app`). A private app | |
| 8 | carries neither and names nothing. | |
| 9 | 2. **Report a problem, no token.** | |
| 10 | - Page: follow `rel=help`; `/__forge/report?from=/page` sends you to | |
| 11 | `/owner/app/report`, told which page you came from. | |
| 12 | - HTTP: `POST /api/v1/repos/{owner}/{name}/reports` with | |
| 13 | `{"title", "body"?, "page"?, "agent"?}` answers `202`. | |
| 14 | - MCP: call `report` on `/mcp` with `{repo, title, body?, page?, agent?}`. | |
| 15 | `/.well-known/mcp.json` lists it in `tokenFreeTools`; every other tool needs a token. | |
| 16 | 3. **A person decides.** The report waits, marked `untrusted`, `from: "outside"`, | |
| 17 | with its page. A person who writes the repository accepts it as an issue | |
| 18 | (label `outside`, the stranger's words fenced, "Sent from: <page>") or dismisses it. | |
| 19 | The door is open, so its limits hold: 3 reports a minute per source, 20 per | |
| 20 | repository, 60 for the forge, 100 waiting per repository, and a request body | |
| 21 | of at most 64 KiB (`413`). `test/muse-journey.test.ts` walks all of this | |
| 22 | end to end and asserts the limits. |
test/muse-journey.test.ts added +283
| @@ -0,0 +1,283 @@ | ||
| 1 | /** | |
| 2 | * The Muse test, end to end (#10383): a stranger agent opens a public app's | |
| 3 | * door, reads only what the door says (its `Link` headers, the forge's | |
| 4 | * `/.well-known/mcp.json` and `/llms.txt`), and files a report with no | |
| 5 | * account, over HTTP and then over the MCP `report` tool. A person who | |
| 6 | * writes the repository finds both marked as a stranger's, with the page | |
| 7 | * they came from. A private app's door offers nothing and names nothing. | |
| 8 | */ | |
| 9 | import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; | |
| 10 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { newApp } from "../src/actions/apps.ts"; | |
| 15 | import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts"; | |
| 16 | import { OUTSIDE_LABEL } from "../src/core/reports.ts"; | |
| 17 | import { createToken } from "../src/core/tokens.ts"; | |
| 18 | import { createUser } from "../src/core/users.ts"; | |
| 19 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 20 | import { quietLogs } from "./helpers/globals.ts"; | |
| 21 | ||
| 22 | quietLogs(); | |
| 23 | ||
| 24 | const DOMAIN = "apps.test"; | |
| 25 | ||
| 26 | /** The target of the first link relation `rel` in a `Link` header. */ | |
| 27 | const linkTo = (res: Response, rel: string): string | undefined => | |
| 28 | (res.headers.get("link") ?? "") | |
| 29 | .split(",") | |
| 30 | .map((part) => /<([^>]+)>\s*;\s*rel="([^"]+)"/.exec(part.trim())) | |
| 31 | .find((m) => m?.[2] === rel)?.[1]; | |
| 32 | ||
| 33 | describe("the Muse test: a stranger agent needs no teaching", () => { | |
| 34 | let dataDir: string; | |
| 35 | let app: App; | |
| 36 | let servers: Servers; | |
| 37 | let runtime: AppsRuntime; | |
| 38 | let backend: ReturnType<typeof Bun.serve>; | |
| 39 | let status: ReturnType<typeof spyOn>; | |
| 40 | let token = ""; | |
| 41 | let source = 0; | |
| 42 | const from = () => ({ "x-forwarded-for": `198.51.100.${++source}` }); | |
| 43 | ||
| 44 | beforeAll(async () => { | |
| 45 | backend = Bun.serve({ | |
| 46 | port: 0, | |
| 47 | fetch: () => | |
| 48 | new Response("<html><body>hi</body></html>", { | |
| 49 | headers: { "content-type": "text/html; charset=utf-8" }, | |
| 50 | }), | |
| 51 | }); | |
| 52 | dataDir = await mkdtemp(join(tmpdir(), "forge-muse-")); | |
| 53 | app = createApp({ dataDir, port: 0, host: "forge.test" }); | |
| 54 | servers = await serve(app); | |
| 55 | runtime = new AppsRuntime(app, { | |
| 56 | domain: DOMAIN, | |
| 57 | scheme: "http", | |
| 58 | publicPort: servers.port, | |
| 59 | forgeUrl: servers.url, | |
| 60 | concurrency: 1, | |
| 61 | }); | |
| 62 | await runtime.start(); | |
| 63 | status = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation( | |
| 64 | () => | |
| 65 | ({ | |
| 66 | env: "production", | |
| 67 | state: "running", | |
| 68 | port: backend.port, | |
| 69 | commitOid: "a".repeat(40), | |
| 70 | }) as EnvStatus, | |
| 71 | ); | |
| 72 | const a = await createUser(app.db, { | |
| 73 | origin: "seed", | |
| 74 | trusted: true, | |
| 75 | username: "alice", | |
| 76 | email: "[email protected]", | |
| 77 | password: "a-good-passphrase", | |
| 78 | adminIfFirst: true, | |
| 79 | }); | |
| 80 | if (!a.ok) throw new Error(a.error); | |
| 81 | token = createToken(app.db, a.value.id, "t", "rw").token; | |
| 82 | for (const [name, visibility] of [ | |
| 83 | ["notes", "public"], | |
| 84 | ["diary", "private"], | |
| 85 | ] as const) { | |
| 86 | const made = await newApp(app, a.value, { name, description: "Words", visibility }); | |
| 87 | if (!made.ok) throw new Error(String(made.error)); | |
| 88 | } | |
| 89 | }, 120_000); | |
| 90 | ||
| 91 | afterAll(async () => { | |
| 92 | status.mockRestore(); | |
| 93 | await runtime.stop(); | |
| 94 | servers.stop(); | |
| 95 | backend.stop(true); | |
| 96 | app.close(); | |
| 97 | await rm(dataDir, { recursive: true, force: true }); | |
| 98 | }); | |
| 99 | ||
| 100 | const door = (slug: string, path: string) => | |
| 101 | fetch(`${servers.url}${path}`, { | |
| 102 | headers: { host: `${slug}.${DOMAIN}`, accept: "text/html" }, | |
| 103 | redirect: "manual", | |
| 104 | }); | |
| 105 | const owner = (path: string, init: RequestInit = {}) => | |
| 106 | fetch(`${servers.url}${path}`, { | |
| 107 | ...init, | |
| 108 | headers: { | |
| 109 | authorization: `Bearer ${token}`, | |
| 110 | "content-type": "application/json", | |
| 111 | }, | |
| 112 | }); | |
| 113 | const rpc = (payload: unknown, headers: Record<string, string> = from()) => | |
| 114 | fetch(`${servers.url}/mcp`, { | |
| 115 | method: "POST", | |
| 116 | headers: { "content-type": "application/json", ...headers }, | |
| 117 | body: JSON.stringify(payload), | |
| 118 | }); | |
| 119 | const pending = (): number => | |
| 120 | (app.db.query("SELECT count(*) AS n FROM report WHERE state = 'pending'").get() as { n: number }).n; | |
| 121 | ||
| 122 | test("reading only the door, the forge's card and llms.txt, a stranger files over HTTP and over the MCP", async () => { | |
| 123 | const page = "http://notes.apps.test/leaderboard"; | |
| 124 | ||
| 125 | // 1. The door's own headers. | |
| 126 | const first = await door("notes", "/leaderboard"); | |
| 127 | const help = linkTo(first, "help"); | |
| 128 | expect(help).toBe("/__forge/report"); | |
| 129 | const manifest = linkTo(first, "alternate")!; | |
| 130 | expect(manifest).toStartWith(`${servers.url}/api/v1/here`); | |
| 131 | ||
| 132 | // 2. The manifest names the report action, HTTP and MCP, as needing no account. | |
| 133 | const here = (await (await fetch(manifest)).json()) as any; | |
| 134 | const act = here.actions.find((x: any) => x.do === "report a problem"); | |
| 135 | expect(act.http).toBe("POST /api/v1/repos/alice/notes/reports"); | |
| 136 | expect(act.mcp).toBe("report"); | |
| 137 | expect(act.account).toBe(false); | |
| 138 | ||
| 139 | // 3. The server card lists the tool as open, and says where the MCP is. | |
| 140 | const card = (await (await fetch(`${servers.url}/.well-known/mcp.json`)).json()) as any; | |
| 141 | expect(card.tokenFreeTools).toContain("report"); | |
| 142 | const tool = card.tokenFreeTools.find((t: string) => t === "report"); | |
| 143 | expect(card.documentation).toBe(`${servers.url}/llms.txt`); | |
| 144 | const llms = await (await fetch(card.documentation)).text(); | |
| 145 | expect(llms).toContain("POST " + servers.url + "/api/v1/repos/{owner}/{name}/reports"); | |
| 146 | expect(llms).toContain("report"); | |
| 147 | ||
| 148 | // 4. rel=help leads to the form, told which page the visitor came from. | |
| 149 | const helped = await door("notes", `${help}?from=/leaderboard`); | |
| 150 | expect(helped.status).toBe(303); | |
| 151 | const form = new URL(helped.headers.get("location")!); | |
| 152 | expect(form.pathname).toBe("/alice/notes/report"); | |
| 153 | expect(form.searchParams.get("from")).toStartWith("http://notes.apps.test"); | |
| 154 | expect((await fetch(form)).status).toBe(200); | |
| 155 | ||
| 156 | // 5. HTTP, no token. | |
| 157 | const before = pending(); | |
| 158 | const http = await fetch(`${servers.url}/api/v1/repos/alice/notes/reports`, { | |
| 159 | method: "POST", | |
| 160 | headers: { "content-type": "application/json", ...from() }, | |
| 161 | body: JSON.stringify({ title: "Leaderboard is empty", body: "HTTP report", page }), | |
outdated
Use the tool name 'report' found in tokenFreeTools, not index 0. |
||
| 162 | }); | |
| 163 | expect(http.status).toBe(202); | |
| 164 | const httpId = ((await http.json()) as any).id as string; | |
| 165 | ||
| 166 | // 6. MCP, no token; `report` is the one tool the card lists as open. | |
| 167 | const call = await rpc({ | |
| 168 | jsonrpc: "2.0", | |
| 169 | id: 1, | |
| 170 | method: "tools/call", | |
| 171 | params: { | |
| 172 | name: tool, | |
| 173 | arguments: { repo: "alice/notes", title: "Leaderboard is slow", body: "MCP report", page }, | |
| 174 | }, | |
| 175 | }); | |
| 176 | expect(call.status).toBe(200); | |
| 177 | const mcpId = JSON.parse(((await call.json()) as any).result.content[0].text).id as string; | |
| 178 | expect(pending()).toBe(before + 2); | |
| 179 | ||
| 180 | // 7. Both wait as a stranger's words, with the page, until a person decides. | |
| 181 | const listed = (await (await owner("/api/v1/repos/alice/notes/reports")).json()) as any; | |
| 182 | const items: any[] = listed.items ?? listed; | |
| 183 | for (const id of [httpId, mcpId]) { | |
| 184 | const r = items.find((x) => x.id === id); | |
outdated
rpc() sets a new x-forwarded-for per call, so this is a different source and is not limited. Reuse the burst's source and assert the rate-limit error, otherwise the comment is false. |
||
| 185 | expect(r.untrusted).toBe(true); | |
| 186 | expect(r.from).toBe("outside"); | |
| 187 | expect(r.page).toBe(page); | |
| 188 | } | |
| 189 | ||
| 190 | // 8. Accepted, each is an issue behind the outside marker, naming the page. | |
| 191 | for (const id of [httpId, mcpId]) { | |
| 192 | const accepted = await owner(`/api/v1/repos/alice/notes/reports/${id}/accept`, { | |
| 193 | method: "POST", | |
| 194 | body: "{}", | |
| 195 | }); | |
| 196 | expect(accepted.status).toBe(201); | |
| 197 | const issue = (await accepted.json()) as any; | |
| 198 | expect(issue.title).toContain("Leaderboard"); | |
| 199 | expect(issue.body).toContain(`Sent from: ${page}`); | |
| 200 | expect(issue.body).toContain("without an account"); | |
| 201 | expect(JSON.stringify(issue.labels ?? [])).toContain(OUTSIDE_LABEL); | |
| 202 | } | |
| 203 | }); | |
| 204 | ||
| 205 | test("the open door keeps its rate limit and its body cap", async () => { | |
| 206 | const before = pending(); | |
| 207 | const one = { "x-forwarded-for": "203.0.113.77" }; | |
| 208 | const statuses: number[] = []; | |
| 209 | for (let i = 0; i < 5; i++) { | |
| 210 | const res = await fetch(`${servers.url}/api/v1/repos/alice/notes/reports`, { | |
| 211 | method: "POST", | |
| 212 | headers: { "content-type": "application/json", ...one }, | |
| 213 | body: JSON.stringify({ title: `t${i}` }), | |
| 214 | }); | |
| 215 | statuses.push(res.status); | |
| 216 | } | |
| 217 | expect(statuses).toEqual([202, 202, 202, 429, 429]); | |
| 218 | // The MCP tool is the same door and the same source, so the same limit. | |
| 219 | const limited = (await ( | |
| 220 | await rpc( | |
| 221 | { | |
| 222 | jsonrpc: "2.0", | |
| 223 | id: 2, | |
| 224 | method: "tools/call", | |
| 225 | params: { name: "report", arguments: { repo: "alice/notes", title: "x" } }, | |
| 226 | }, | |
| 227 | one, | |
| 228 | ) | |
| 229 | ).json()) as any; | |
| 230 | expect(limited.result.isError).toBe(true); | |
| 231 | expect(limited.result.content[0].text.toLowerCase()).toMatch(/rate|limit|too_many|many/); | |
| 232 | const huge = await fetch(`${servers.url}/api/v1/repos/alice/notes/reports`, { | |
| 233 | method: "POST", | |
| 234 | headers: { "content-type": "application/json", ...from() }, | |
| 235 | body: JSON.stringify({ title: "t", body: "x".repeat(70_000) }), | |
| 236 | }); | |
| 237 | expect(huge.status).toBe(413); | |
| 238 | const hugeMcp = await rpc({ | |
| 239 | jsonrpc: "2.0", | |
| 240 | id: 3, | |
| 241 | method: "tools/call", | |
| 242 | params: { name: "report", arguments: { repo: "alice/notes", title: "t", body: "x".repeat(70_000) } }, | |
| 243 | }); | |
| 244 | expect(hugeMcp.status).toBe(413); | |
| 245 | expect(pending()).toBe(before + 3); | |
| 246 | }); | |
| 247 | ||
| 248 | test("a private app's door answers neither route and leaks no repository name", async () => { | |
| 249 | const res = await door("diary", "/"); | |
| 250 | const seen = [res.headers.get("link") ?? "", await res.text()].join("\n"); | |
| 251 | expect(seen).not.toContain("__forge/report"); | |
| 252 | expect(seen).not.toContain("alice/diary"); | |
| 253 | expect(linkTo(res, "help")).toBeUndefined(); | |
| 254 | ||
| 255 | const route = await door("diary", "/__forge/report"); | |
| 256 | expect(route.headers.get("location") ?? "").not.toContain("/report"); | |
| 257 | expect(await route.text()).not.toContain("alice/diary"); | |
| 258 | ||
| 259 | const before = pending(); | |
| 260 | const api = await fetch(`${servers.url}/api/v1/repos/alice/diary/reports`, { | |
| 261 | method: "POST", | |
| 262 | headers: { "content-type": "application/json", ...from() }, | |
| 263 | body: JSON.stringify({ title: "hello" }), | |
| 264 | }); | |
| 265 | expect(api.status).toBe(404); | |
| 266 | const viaMcp = (await ( | |
| 267 | await rpc({ | |
| 268 | jsonrpc: "2.0", | |
| 269 | id: 4, | |
| 270 | method: "tools/call", | |
| 271 | params: { name: "report", arguments: { repo: "alice/diary", title: "hello" } }, | |
| 272 | }) | |
| 273 | ).json()) as any; | |
| 274 | expect(viaMcp.result.isError).toBe(true); | |
| 275 | expect(viaMcp.result.content[0].text).toStartWith("not_found"); | |
| 276 | expect(pending()).toBe(before); | |
| 277 | ||
| 278 | const llms = await (await fetch(`${servers.url}/llms.txt`)).text(); | |
| 279 | expect(llms).not.toContain("diary"); | |
| 280 | const card = await (await fetch(`${servers.url}/.well-known/mcp.json`)).text(); | |
| 281 | expect(card).not.toContain("diary"); | |
| 282 | }); | |
| 283 | }); |
The fork-and-propose path is described but not covered by the test. Test it or trim the claim.