Public: anyone can read this; the crew can change it.
A search query has a length limit at the boundary: /explore?q= with 5,000 characters answers a 31 KB page today #14400
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14389-a-search-query-has-a-length-limit-at-the → main 5 commits fork 1 thread · 1 unresolved
9 files
changed +144
−15
docs/features/explore.md +15
| @@ -101,3 +101,18 @@ In the terminal the same sentence sits in the `Read as:` line under the search b | ||
| 101 | 101 | A row shows the app's name, what it does in plain words, and the one signal that explains its place (the other numbers are left off the page; the API and the MCP still carry them all). The filter line is the search box, the kind tabs and, signed in, "Mine"; "Sort and time" opens the sorts, the time windows and "Show archived", and stays open while one of them is set. The ranking order is unchanged. |
| 102 | 102 | |
| 103 | 103 | The terminal's explore screen deliberately keeps the stacked numbers (forks, ships, likes, use) that the page leaves off: a terminal row has the width for them and no second tap to reach them. The ranking is the same on both. |
| 104 | ||
| 105 | ## The longest query (#13846, #14389) | |
| 106 | ||
| 107 | A search query is at most 200 characters, named once as `MAX_QUERY_CHARS` in | |
| 108 | `src/core/query_limit.ts` and read by every door that takes one: `/explore`, | |
| 109 | `/search` and `/docs` as pages, `GET /api/v1/explore` and `GET /api/v1/search`, | |
| 110 | and the MCP `explore` and `search` tools, and the agent actions a page lists | |
| 111 | (`src/actions/here.ts`). Every door counts characters as code points after | |
| 112 | trimming (`queryTooLong`, `cappedQuery`), so 200 emoji are one query at the | |
| 113 | limit at each door, not 400 UTF-16 units refused by one. Over it, the API answers 422 with | |
| 114 | `{ "field": "q", "message": "q is at most 200 characters" }` and the MCP tools | |
| 115 | answer `invalid` naming the argument; a page searches the first 200 characters, | |
| 116 | keeps them in the box and says so in one line. No link a page builds back to | |
| 117 | itself (the sign-in and sign-up `next`, the palette's Sign in) carries more | |
| 118 | than that, so a 5,000-character paste does not bloat the page. |
src/actions/here.ts +4 −3
| @@ -31,6 +31,7 @@ import { | ||
| 31 | 31 | type Repo, |
| 32 | 32 | } from "../core/repos.ts"; |
| 33 | 33 | import { getUserByName, hiddenFrom } from "../core/users.ts"; |
| 34 | import { cappedQuery } from "../core/query_limit.ts"; | |
| 34 | 35 | import { SYSTEM_NAMES } from "../core/reserved.ts"; |
| 35 | 36 | import { scopeAllows, type TokenScope } from "../core/tokens.ts"; |
| 36 | 37 | import type { User } from "../core/users.ts"; |
| @@ -321,10 +322,10 @@ function pageRead(page: "explore" | "search", path: string): Action { | ||
| 321 | 322 | q === "" ? "" : `?q=${encodeURIComponent(q).replace(/%3A/gi, ":")}`; |
| 322 | 323 | const http = `GET /api/v1/${page}${query}`; |
| 323 | 324 | return { |
| 324 | do: `${page === "explore" ? "browse the forge" : "search the forge"}${q === "" ? "" : `: ${q.slice(0, 200)}`}`, | |
| 325 | do: `${page === "explore" ? "browse the forge" : "search the forge"}${q === "" ? "" : `: ${cappedQuery(q)}`}`, | |
| 325 | 326 | http, |
| 326 | 327 | mcp: page, |
| 327 | mcpArgs: { q: q.slice(0, 200) }, | |
| 328 | mcpArgs: { q: cappedQuery(q) }, | |
| 328 | 329 | }; |
| 329 | 330 | } |
| 330 | 331 | |
| @@ -358,7 +359,7 @@ function keptActions( | ||
| 358 | 359 | case "docs": |
| 359 | 360 | return [ |
| 360 | 361 | { |
| 361 | do: `search the guide${q === "" ? "" : `: ${q.slice(0, 200)}`}`, | |
| 362 | do: `search the guide${q === "" ? "" : `: ${cappedQuery(q)}`}`, | |
| 362 | 363 | http: `GET /docs?q=${encodeURIComponent(q.slice(0, 80))}`, |
| 363 | 364 | account: false, |
| 364 | 365 | }, |
src/http/mcp/reads.ts +6 −4
| @@ -15,16 +15,18 @@ import { forkTreeRows } from "../../actions/fork_tree_rows.ts"; | ||
| 15 | 15 | import type { User } from "../../core/users.ts"; |
| 16 | 16 | import type { Ctx } from "../context.ts"; |
| 17 | 17 | import { type Answer, type Args, bad, good, str, repoArg, isAnswer } from "./shared.ts"; |
| 18 | import { MAX_QUERY_CHARS, queryTooLong } from "../../core/query_limit.ts"; | |
| 18 | 19 | |
| 19 | 20 | export async function exploreRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> { |
| 20 | 21 | if (str(args["kind"]) === "tree") { |
| 21 | 22 | const repo = repoArg(ctx, user, { repo: args["app"] }); |
| 22 | 23 | return isAnswer(repo) ? repo : good(await forkTreeRows(ctx.app, repo, user)); |
| 23 | 24 | } |
| 24 | // The stranger path skips the tool schema, so the lengths are checked here for both. | |
| 25 | // The stranger path skips the tool schema, so the lengths are checked here for both, in code | |
| 26 | // points like the page and the API (`queryTooLong`). | |
| 25 | 27 | for (const key of ["q", "query", "cursor"]) |
| 26 | if (args[key] !== undefined && (typeof args[key] !== "string" || (args[key] as string).length > 200)) | |
| 27 | return bad("invalid", `\`${key}\` is a string of at most 200 characters`); | |
| 28 | if (args[key] !== undefined && (typeof args[key] !== "string" || queryTooLong(args[key] as string))) | |
| 29 | return bad("invalid", `\`${key}\` is a string of at most ${MAX_QUERY_CHARS} characters`); | |
| 28 | 30 | const q = str(args["q"]); |
| 29 | 31 | const line = exploreLineFrom((name) => (name === "q" ? (q !== "" ? q : str(args["query"])) : str(args[name]))); |
| 30 | 32 | const { query, unknown } = readExploreLine(line); |
| @@ -45,7 +47,7 @@ export async function exploreRun(ctx: Ctx, user: User | undefined, args: Args): | ||
| 45 | 47 | |
| 46 | 48 | export async function searchRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> { |
| 47 | 49 | const q = args["q"]; |
| 48 | if (typeof q !== "string" || q.length > 200) return bad("invalid", "`q` is a string of at most 200 characters"); | |
| 50 | if (typeof q !== "string" || queryTooLong(q)) return bad("invalid", `\`q\` is a string of at most ${MAX_QUERY_CHARS} characters`); | |
| 49 | 51 | const kind = args["kind"]; |
| 50 | 52 | if (kind !== undefined && typeof kind !== "string") return bad("invalid_kind", "kind is all, repos, issues or code"); |
| 51 | 53 | const refusal = searchRefusal(kind, args["limit"] === undefined || typeof args["limit"] === "number" ? (args["limit"] as number | undefined) : NaN); |
src/http/mcp/tools/explore.ts +3 −2
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | import { EXPLORE_PARAMS } from "../../../actions/explore.ts"; |
| 2 | 2 | import type { Tool } from "../shared.ts"; |
| 3 | 3 | import { exploreRun } from "../reads.ts"; |
| 4 | import { MAX_QUERY_CHARS } from "../../../core/query_limit.ts"; | |
| 4 | 5 | |
| 5 | 6 | export const tool: Tool = { |
| 6 | 7 | name: "explore", |
| @@ -12,12 +13,12 @@ export const tool: Tool = { | ||
| 12 | 13 | properties: { |
| 13 | 14 | q: { |
| 14 | 15 | type: "string", |
| 15 | maxLength: 200, | |
| 16 | maxLength: MAX_QUERY_CHARS, | |
| 16 | 17 | description: "kind: sort: updated: owner: archived:. Unknown word: error", |
| 17 | 18 | }, |
| 18 | 19 | query: { |
| 19 | 20 | type: "string", |
| 20 | maxLength: 200, | |
| 21 | maxLength: MAX_QUERY_CHARS, | |
| 21 | 22 | description: "Alias of q", |
| 22 | 23 | }, |
| 23 | 24 | ...Object.fromEntries(EXPLORE_PARAMS.map((k) => [k, { type: "string", maxLength: 100 }])), |
src/http/mcp/tools/search.ts +2 −1
| @@ -1,5 +1,6 @@ | ||
| 1 | 1 | import type { Tool } from "../shared.ts"; |
| 2 | 2 | import { searchRun } from "../reads.ts"; |
| 3 | import { MAX_QUERY_CHARS } from "../../../core/query_limit.ts"; | |
| 3 | 4 | |
| 4 | 5 | export const tool: Tool = { |
| 5 | 6 | name: "search", |
| @@ -9,7 +10,7 @@ export const tool: Tool = { | ||
| 9 | 10 | inputSchema: { |
| 10 | 11 | type: "object", |
| 11 | 12 | properties: { |
| 12 | q: { type: "string", maxLength: 200 }, | |
| 13 | q: { type: "string", maxLength: MAX_QUERY_CHARS }, | |
| 13 | 14 | kind: { type: "string", enum: ["all", "issues", "repos", "code"] }, |
| 14 | 15 | limit: { type: "integer" }, |
| 15 | 16 | repo: { type: "string", maxLength: 140 }, |
src/web/chrome.ts +2 −2
| @@ -8,7 +8,7 @@ | ||
| 8 | 8 | * `ui.ts` may not. That is the line between the two files. |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | import { returnTo, withReturn } from "./return_to.ts"; | |
| 11 | import { hereAddress, returnTo, withReturn } from "./return_to.ts"; | |
| 12 | 12 | import { MACHINE_PREFIX, typedMiss } from "../http/asset_miss.ts"; |
| 13 | 13 | import { audienceFacts, audienceOf } from "../core/audience.ts"; |
| 14 | 14 | import { activeEdgeToken, edgeLoginUrl, edgeStrangerLogin } from "../http/edge.ts"; |
| @@ -183,7 +183,7 @@ export function signInFirst(ctx: Ctx): Response { | ||
| 183 | 183 | |
| 184 | 184 | /** Where signing in starts, coming back to this page after. */ |
| 185 | 185 | export function signInHref(ctx: Ctx): string { |
| 186 | const next = ctx.url.pathname + ctx.url.search; | |
| 186 | const next = hereAddress(ctx); | |
| 187 | 187 | // Behind a platform edge the edge is how people sign in; an account it |
| 188 | 188 | // made has no password here, so the password form would only refuse them. |
| 189 | 189 | if (edgeStrangerLogin(ctx.request, activeEdgeToken(ctx.app), ctx.auth.user) !== undefined) |
src/web/layout.ts +2 −2
| @@ -8,7 +8,7 @@ | ||
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | 10 | import { sshOn } from "../settings/ssh.ts"; |
| 11 | import { returnTo, withReturn } from "./return_to.ts"; | |
| 11 | import { hereAddress, returnTo, withReturn } from "./return_to.ts"; | |
| 12 | 12 | import { waitingCount } from "../actions/conversation_confirms.ts"; |
| 13 | 13 | import { goLevel, goPaletteRows } from "./go_pages.ts"; |
| 14 | 14 | import { NEW_ISSUE_KEY } from "./script_keys.ts"; |
| @@ -655,7 +655,7 @@ function flashToast(flash: Flash): Html { | ||
| 655 | 655 | export function signInHref(ctx: Ctx): string { |
| 656 | 656 | const token = activeEdgeToken(ctx.app); |
| 657 | 657 | if (edgeStrangerLogin(ctx.request, token, ctx.auth.user) !== undefined) return "/signup"; |
| 658 | return token === undefined ? withReturn("/signin", returnTo(ctx)) : edgeLoginUrl(ctx.url.pathname + ctx.url.search); | |
| 658 | return token === undefined ? withReturn("/signin", returnTo(ctx)) : edgeLoginUrl(hereAddress(ctx)); | |
| 659 | 659 | } |
| 660 | 660 | |
| 661 | 661 | /** |
src/web/return_to.ts +15 −1
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | * link is built here, and `test/deadends.test.ts` walks them. |
| 7 | 7 | */ |
| 8 | 8 | import type { Ctx } from "../http/context.ts"; |
| 9 | import { cappedQuery, queryTooLong } from "../core/query_limit.ts"; | |
| 9 | 10 | |
| 10 | 11 | /** |
| 11 | 12 | * A `?next=` destination is a path on this forge or nothing at all. A |
| @@ -41,6 +42,19 @@ const DOORS = /^\/(?:signin|signup|recover|claim|first-admin)$/; | ||
| 41 | 42 | */ |
| 42 | 43 | const NO_RETURN = /^\/(?:(?:go|claim|first-admin|verify|signin\/access|signin\/parent|recover|broker)(?:\/|$)|[^/]+\/[^/]+\/toolbar$)/; |
| 43 | 44 | |
| 45 | /** | |
| 46 | * This page's own address, as a link back to it carries it: a search `q` past the longest query is | |
| 47 | * cut to the part the page searched, so a pasted page of text is not repeated in every sign-in link | |
| 48 | * (#14389). Any other address is returned as it came. | |
| 49 | */ | |
| 50 | export function hereAddress(ctx: Ctx): string { | |
| 51 | const here = ctx.url.pathname + ctx.url.search; | |
| 52 | if (!queryTooLong(ctx.url.searchParams.get("q"))) return here; | |
| 53 | const params = new URLSearchParams(ctx.url.search); | |
| 54 | params.set("q", cappedQuery(params.get("q"))); | |
| 55 | return `${ctx.url.pathname}?${params}`; | |
| 56 | } | |
| 57 | ||
| 44 | 58 | /** |
| 45 | 59 | * The address a link into the sign-in door from this page should carry: the page itself, or on a |
| 46 | 60 | * door page the address it was entered with. |
| @@ -48,7 +62,7 @@ const NO_RETURN = /^\/(?:(?:go|claim|first-admin|verify|signin\/access|signin\/p | ||
| 48 | 62 | export function returnTo(ctx: Ctx): string | undefined { |
| 49 | 63 | if (DOORS.test(ctx.url.pathname)) return safeNext(ctx.url.searchParams.get("next")); |
| 50 | 64 | if (NO_RETURN.test(ctx.url.pathname)) return undefined; |
| 51 | const here = ctx.url.pathname + ctx.url.search; | |
| 65 | const here = hereAddress(ctx); | |
| 52 | 66 | return here === "/" ? undefined : safeNext(here); |
| 53 | 67 | } |
| 54 | 68 |
test/explore-q-limit.test.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | /** | |
| 2 | * Every search door holds `q` to the one longest query, and no page repeats more of it (#14389). | |
| 3 | */ | |
| 4 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 5 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 6 | import { tmpdir } from "node:os"; | |
| 7 | import { join } from "node:path"; | |
| 8 | import { createApp, type App } from "../src/app.ts"; | |
| 9 | import { MAX_QUERY_CHARS } from "../src/core/query_limit.ts"; | |
| 10 | import { createToken } from "../src/core/tokens.ts"; | |
| 11 | import { createUser } from "../src/core/users.ts"; | |
| 12 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: App; | |
| 16 | let servers: Servers; | |
| 17 | let token: string; | |
| 18 | let id = 1; | |
| 19 | const paste = "a".repeat(5000); | |
| 20 | const over = "a".repeat(MAX_QUERY_CHARS + 1); | |
| 21 | /** The longest run of the query's letter anywhere in `text`. */ | |
| 22 | const longestRun = (text: string) => Math.max(0, ...(text.match(/a{50,}/g) ?? []).map((m) => m.length)); | |
| 23 | ||
| 24 | const tool = async (name: string, args: Record<string, unknown>) => { | |
| 25 | const res = await fetch(`${servers.url}/mcp`, { | |
| 26 | method: "POST", | |
| 27 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, | |
| 28 | body: JSON.stringify({ jsonrpc: "2.0", id: id++, method: "tools/call", params: { name, arguments: args } }), | |
| 29 | }); | |
| 30 | const body: any = await res.json(); | |
| 31 | return { error: body.result.isError === true, text: String(body.result.content[0].text) }; | |
| 32 | }; | |
| 33 | ||
| 34 | beforeAll(async () => { | |
| 35 | dataDir = await mkdtemp(join(tmpdir(), "forge-explore-q-limit-")); | |
| 36 | app = createApp({ dataDir, port: 0 }); | |
| 37 | servers = await serve(app); | |
| 38 | const user = await createUser(app.db, { origin: "seed", trusted: true, username: "quinn", email: "[email protected]", password: "a-good-passphrase" }); | |
| 39 | if (!user.ok) throw new Error(user.error); | |
| 40 | token = createToken(app.db, user.value.id, "ro", "ro").token; | |
| 41 | }); | |
| 42 | afterAll(async () => { | |
| 43 | servers.stop(); | |
| 44 | await rm(dataDir, { recursive: true, force: true }); | |
| 45 | }); | |
| 46 | ||
| 47 | describe("the page", () => { | |
| 48 | test("/explore with a 5,000-character q repeats at most the capped query, in the box and every link", async () => { | |
| 49 | const res = await fetch(`${servers.url}/explore?q=${paste}`); | |
| 50 | expect(res.status).toBe(200); | |
| 51 | const text = await res.text(); | |
| 52 | expect(text).toContain("data-query-trimmed"); | |
| 53 | expect(text).toContain(`value="${"a".repeat(MAX_QUERY_CHARS)}"`); | |
| 54 | // The sign-in and sign-up `next` and the palette's Sign in carried the whole paste before. | |
| 55 | expect(longestRun(text)).toBe(MAX_QUERY_CHARS); | |
| 56 | expect(text).toContain(`/signin?next=${encodeURIComponent(`/explore?q=${"a".repeat(MAX_QUERY_CHARS)}`)}`); | |
| 57 | }); | |
| 58 | ||
| 59 | test("a query within the limit keeps its address as it came", async () => { | |
| 60 | const text = await (await fetch(`${servers.url}/explore?q=abc&sort=new`)).text(); | |
| 61 | expect(text).toContain(`/signin?next=${encodeURIComponent("/explore?q=abc&sort=new")}`); | |
| 62 | }); | |
| 63 | }); | |
| 64 | ||
| 65 | describe("the API", () => { | |
| 66 | for (const path of ["/api/v1/explore", "/api/v1/search"]) { | |
| 67 | test(`${path} answers 422 naming q`, async () => { | |
| 68 | const res = await fetch(`${servers.url}${path}?q=${over}`); | |
| 69 | expect(res.status).toBe(422); | |
| 70 | expect(((await res.json()) as { field: string }).field).toBe("q"); | |
| 71 | }); | |
| 72 | test(`${path} takes ${MAX_QUERY_CHARS} emoji, as the MCP tools do`, async () => { | |
| 73 | const res = await fetch(`${servers.url}${path}?q=${encodeURIComponent("😀".repeat(MAX_QUERY_CHARS))}`, { headers: { authorization: `Bearer ${token}` } }); | |
| 74 | expect(res.status).toBe(200); | |
| 75 | }); | |
| 76 | } | |
| 77 | }); | |
| 78 | ||
| 79 | describe("the MCP tools", () => { | |
| 80 | for (const name of ["explore", "search"]) { | |
| 81 | test(`${name} refuses a q over the limit, naming it`, async () => { | |
| 82 | const answer = await tool(name, { q: over }); | |
| 83 | expect(answer.error).toBe(true); | |
| 84 | expect(answer.text).toContain("`q`"); | |
| 85 | expect(answer.text).toContain(String(MAX_QUERY_CHARS)); | |
| 86 | }); | |
| 87 | test(`${name} takes a q at the limit`, async () => { | |
| 88 | expect((await tool(name, { q: "a".repeat(MAX_QUERY_CHARS) })).error).toBe(false); | |
| 89 | }); | |
| 90 | test(`${name} counts code points like the page and the API: ${MAX_QUERY_CHARS} emoji are at the limit`, async () => { | |
| 91 | expect((await tool(name, { q: "😀".repeat(MAX_QUERY_CHARS) })).error).toBe(false); | |
| 92 | expect((await tool(name, { q: "😀".repeat(MAX_QUERY_CHARS + 1) })).error).toBe(true); | |
| 93 | }); | |
| 94 | } | |
| 95 | }); |
new above
offline
Use queryTooLong(q) from query_limit.ts here and in the loop above.
.lengthcounts UTF-16 units and the page and API count code points, so the doors disagree on emoji.