Public: anyone can read this; the crew can change it.

A search query has a length limit at the boundary: /explore?q= with 5,000 characters answers a 31 KB page today #14400

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14389-a-search-query-has-a-length-limit-at-the → main 5 commits fork 1 thread · 1 unresolved

  • Merged. scout/14389-a-search-query-has-a-length-limit-at-the was merged into main as 6cf404e.

Review

Sign in to review this change.
9 files changed +144 −15
docs/features/explore.md +15
@@ -101,3 +101,18 @@ In the terminal the same sentence sits in the `Read as:` line under the search b
101101 A row shows the app's name, what it does in plain words, and the one signal that explains its place (the other numbers are left off the page; the API and the MCP still carry them all). The filter line is the search box, the kind tabs and, signed in, "Mine"; "Sort and time" opens the sorts, the time windows and "Show archived", and stays open while one of them is set. The ranking order is unchanged.
102102
103103 The terminal's explore screen deliberately keeps the stacked numbers (forks, ships, likes, use) that the page leaves off: a terminal row has the width for them and no second tap to reach them. The ranking is the same on both.
104
105 ## The longest query (#13846, #14389)
106
107 A search query is at most 200 characters, named once as `MAX_QUERY_CHARS` in
108 `src/core/query_limit.ts` and read by every door that takes one: `/explore`,
109 `/search` and `/docs` as pages, `GET /api/v1/explore` and `GET /api/v1/search`,
110 and the MCP `explore` and `search` tools, and the agent actions a page lists
111 (`src/actions/here.ts`). Every door counts characters as code points after
112 trimming (`queryTooLong`, `cappedQuery`), so 200 emoji are one query at the
113 limit at each door, not 400 UTF-16 units refused by one. Over it, the API answers 422 with
114 `{ "field": "q", "message": "q is at most 200 characters" }` and the MCP tools
115 answer `invalid` naming the argument; a page searches the first 200 characters,
116 keeps them in the box and says so in one line. No link a page builds back to
117 itself (the sign-in and sign-up `next`, the palette's Sign in) carries more
118 than that, so a 5,000-character paste does not bloat the page.
src/actions/here.ts +4 −3
@@ -31,6 +31,7 @@ import {
3131 type Repo,
3232 } from "../core/repos.ts";
3333 import { getUserByName, hiddenFrom } from "../core/users.ts";
34 import { cappedQuery } from "../core/query_limit.ts";
3435 import { SYSTEM_NAMES } from "../core/reserved.ts";
3536 import { scopeAllows, type TokenScope } from "../core/tokens.ts";
3637 import type { User } from "../core/users.ts";
@@ -321,10 +322,10 @@ function pageRead(page: "explore" | "search", path: string): Action {
321322 q === "" ? "" : `?q=${encodeURIComponent(q).replace(/%3A/gi, ":")}`;
322323 const http = `GET /api/v1/${page}${query}`;
323324 return {
324 do: `${page === "explore" ? "browse the forge" : "search the forge"}${q === "" ? "" : `: ${q.slice(0, 200)}`}`,
325 do: `${page === "explore" ? "browse the forge" : "search the forge"}${q === "" ? "" : `: ${cappedQuery(q)}`}`,
325326 http,
326327 mcp: page,
327 mcpArgs: { q: q.slice(0, 200) },
328 mcpArgs: { q: cappedQuery(q) },
328329 };
329330 }
330331
@@ -358,7 +359,7 @@ function keptActions(
358359 case "docs":
359360 return [
360361 {
361 do: `search the guide${q === "" ? "" : `: ${q.slice(0, 200)}`}`,
362 do: `search the guide${q === "" ? "" : `: ${cappedQuery(q)}`}`,
362363 http: `GET /docs?q=${encodeURIComponent(q.slice(0, 80))}`,
363364 account: false,
364365 },
src/http/mcp/reads.ts +6 −4
@@ -15,16 +15,18 @@ import { forkTreeRows } from "../../actions/fork_tree_rows.ts";
1515 import type { User } from "../../core/users.ts";
1616 import type { Ctx } from "../context.ts";
1717 import { type Answer, type Args, bad, good, str, repoArg, isAnswer } from "./shared.ts";
18 import { MAX_QUERY_CHARS, queryTooLong } from "../../core/query_limit.ts";
1819
1920 export async function exploreRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> {
2021 if (str(args["kind"]) === "tree") {
2122 const repo = repoArg(ctx, user, { repo: args["app"] });
2223 return isAnswer(repo) ? repo : good(await forkTreeRows(ctx.app, repo, user));
2324 }
24 // The stranger path skips the tool schema, so the lengths are checked here for both.
25 // The stranger path skips the tool schema, so the lengths are checked here for both, in code
26 // points like the page and the API (`queryTooLong`).
2527 for (const key of ["q", "query", "cursor"])
26 if (args[key] !== undefined && (typeof args[key] !== "string" || (args[key] as string).length > 200))
27 return bad("invalid", `\`${key}\` is a string of at most 200 characters`);
28 if (args[key] !== undefined && (typeof args[key] !== "string" || queryTooLong(args[key] as string)))
29 return bad("invalid", `\`${key}\` is a string of at most ${MAX_QUERY_CHARS} characters`);
2830 const q = str(args["q"]);
2931 const line = exploreLineFrom((name) => (name === "q" ? (q !== "" ? q : str(args["query"])) : str(args[name])));
3032 const { query, unknown } = readExploreLine(line);
@@ -45,7 +47,7 @@ export async function exploreRun(ctx: Ctx, user: User | undefined, args: Args):
4547
4648 export async function searchRun(ctx: Ctx, user: User | undefined, args: Args): Promise<Answer> {
4749 const q = args["q"];
outdated

Use queryTooLong(q) from query_limit.ts here and in the loop above. .length counts UTF-16 units and the page and API count code points, so the doors disagree on emoji.

48 if (typeof q !== "string" || q.length > 200) return bad("invalid", "`q` is a string of at most 200 characters");
50 if (typeof q !== "string" || queryTooLong(q)) return bad("invalid", `\`q\` is a string of at most ${MAX_QUERY_CHARS} characters`);
4951 const kind = args["kind"];
5052 if (kind !== undefined && typeof kind !== "string") return bad("invalid_kind", "kind is all, repos, issues or code");
5153 const refusal = searchRefusal(kind, args["limit"] === undefined || typeof args["limit"] === "number" ? (args["limit"] as number | undefined) : NaN);
src/http/mcp/tools/explore.ts +3 −2
@@ -1,6 +1,7 @@
11 import { EXPLORE_PARAMS } from "../../../actions/explore.ts";
22 import type { Tool } from "../shared.ts";
33 import { exploreRun } from "../reads.ts";
4 import { MAX_QUERY_CHARS } from "../../../core/query_limit.ts";
45
56 export const tool: Tool = {
67 name: "explore",
@@ -12,12 +13,12 @@ export const tool: Tool = {
1213 properties: {
1314 q: {
1415 type: "string",
15 maxLength: 200,
16 maxLength: MAX_QUERY_CHARS,
1617 description: "kind: sort: updated: owner: archived:. Unknown word: error",
1718 },
1819 query: {
1920 type: "string",
20 maxLength: 200,
21 maxLength: MAX_QUERY_CHARS,
2122 description: "Alias of q",
2223 },
2324 ...Object.fromEntries(EXPLORE_PARAMS.map((k) => [k, { type: "string", maxLength: 100 }])),
src/http/mcp/tools/search.ts +2 −1
@@ -1,5 +1,6 @@
11 import type { Tool } from "../shared.ts";
22 import { searchRun } from "../reads.ts";
3 import { MAX_QUERY_CHARS } from "../../../core/query_limit.ts";
34
45 export const tool: Tool = {
56 name: "search",
@@ -9,7 +10,7 @@ export const tool: Tool = {
910 inputSchema: {
1011 type: "object",
1112 properties: {
12 q: { type: "string", maxLength: 200 },
13 q: { type: "string", maxLength: MAX_QUERY_CHARS },
1314 kind: { type: "string", enum: ["all", "issues", "repos", "code"] },
1415 limit: { type: "integer" },
1516 repo: { type: "string", maxLength: 140 },
src/web/chrome.ts +2 −2
@@ -8,7 +8,7 @@
88 * `ui.ts` may not. That is the line between the two files.
99 */
1010
11 import { returnTo, withReturn } from "./return_to.ts";
11 import { hereAddress, returnTo, withReturn } from "./return_to.ts";
1212 import { MACHINE_PREFIX, typedMiss } from "../http/asset_miss.ts";
1313 import { audienceFacts, audienceOf } from "../core/audience.ts";
1414 import { activeEdgeToken, edgeLoginUrl, edgeStrangerLogin } from "../http/edge.ts";
@@ -183,7 +183,7 @@ export function signInFirst(ctx: Ctx): Response {
183183
184184 /** Where signing in starts, coming back to this page after. */
185185 export function signInHref(ctx: Ctx): string {
186 const next = ctx.url.pathname + ctx.url.search;
186 const next = hereAddress(ctx);
187187 // Behind a platform edge the edge is how people sign in; an account it
188188 // made has no password here, so the password form would only refuse them.
189189 if (edgeStrangerLogin(ctx.request, activeEdgeToken(ctx.app), ctx.auth.user) !== undefined)
src/web/layout.ts +2 −2
@@ -8,7 +8,7 @@
88 */
99
1010 import { sshOn } from "../settings/ssh.ts";
11 import { returnTo, withReturn } from "./return_to.ts";
11 import { hereAddress, returnTo, withReturn } from "./return_to.ts";
1212 import { waitingCount } from "../actions/conversation_confirms.ts";
1313 import { goLevel, goPaletteRows } from "./go_pages.ts";
1414 import { NEW_ISSUE_KEY } from "./script_keys.ts";
@@ -655,7 +655,7 @@ function flashToast(flash: Flash): Html {
655655 export function signInHref(ctx: Ctx): string {
656656 const token = activeEdgeToken(ctx.app);
657657 if (edgeStrangerLogin(ctx.request, token, ctx.auth.user) !== undefined) return "/signup";
658 return token === undefined ? withReturn("/signin", returnTo(ctx)) : edgeLoginUrl(ctx.url.pathname + ctx.url.search);
658 return token === undefined ? withReturn("/signin", returnTo(ctx)) : edgeLoginUrl(hereAddress(ctx));
659659 }
660660
661661 /**
src/web/return_to.ts +15 −1
@@ -6,6 +6,7 @@
66 * link is built here, and `test/deadends.test.ts` walks them.
77 */
88 import type { Ctx } from "../http/context.ts";
9 import { cappedQuery, queryTooLong } from "../core/query_limit.ts";
910
1011 /**
1112 * A `?next=` destination is a path on this forge or nothing at all. A
@@ -41,6 +42,19 @@ const DOORS = /^\/(?:signin|signup|recover|claim|first-admin)$/;
4142 */
4243 const NO_RETURN = /^\/(?:(?:go|claim|first-admin|verify|signin\/access|signin\/parent|recover|broker)(?:\/|$)|[^/]+\/[^/]+\/toolbar$)/;
4344
45 /**
46 * This page's own address, as a link back to it carries it: a search `q` past the longest query is
47 * cut to the part the page searched, so a pasted page of text is not repeated in every sign-in link
48 * (#14389). Any other address is returned as it came.
49 */
50 export function hereAddress(ctx: Ctx): string {
51 const here = ctx.url.pathname + ctx.url.search;
52 if (!queryTooLong(ctx.url.searchParams.get("q"))) return here;
53 const params = new URLSearchParams(ctx.url.search);
54 params.set("q", cappedQuery(params.get("q")));
55 return `${ctx.url.pathname}?${params}`;
56 }
57
4458 /**
4559 * The address a link into the sign-in door from this page should carry: the page itself, or on a
4660 * door page the address it was entered with.
@@ -48,7 +62,7 @@ const NO_RETURN = /^\/(?:(?:go|claim|first-admin|verify|signin\/access|signin\/p
4862 export function returnTo(ctx: Ctx): string | undefined {
4963 if (DOORS.test(ctx.url.pathname)) return safeNext(ctx.url.searchParams.get("next"));
5064 if (NO_RETURN.test(ctx.url.pathname)) return undefined;
51 const here = ctx.url.pathname + ctx.url.search;
65 const here = hereAddress(ctx);
5266 return here === "/" ? undefined : safeNext(here);
5367 }
5468
test/explore-q-limit.test.ts added +95
@@ -0,0 +1,95 @@
1 /**
2 * Every search door holds `q` to the one longest query, and no page repeats more of it (#14389).
3 */
4 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
5 import { mkdtemp, rm } from "node:fs/promises";
6 import { tmpdir } from "node:os";
7 import { join } from "node:path";
8 import { createApp, type App } from "../src/app.ts";
9 import { MAX_QUERY_CHARS } from "../src/core/query_limit.ts";
10 import { createToken } from "../src/core/tokens.ts";
11 import { createUser } from "../src/core/users.ts";
12 import { serve, type Servers } from "../src/http/server.ts";
13
14 let dataDir: string;
15 let app: App;
16 let servers: Servers;
17 let token: string;
18 let id = 1;
19 const paste = "a".repeat(5000);
20 const over = "a".repeat(MAX_QUERY_CHARS + 1);
21 /** The longest run of the query's letter anywhere in `text`. */
22 const longestRun = (text: string) => Math.max(0, ...(text.match(/a{50,}/g) ?? []).map((m) => m.length));
23
24 const tool = async (name: string, args: Record<string, unknown>) => {
25 const res = await fetch(`${servers.url}/mcp`, {
26 method: "POST",
27 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
28 body: JSON.stringify({ jsonrpc: "2.0", id: id++, method: "tools/call", params: { name, arguments: args } }),
29 });
30 const body: any = await res.json();
31 return { error: body.result.isError === true, text: String(body.result.content[0].text) };
32 };
33
34 beforeAll(async () => {
35 dataDir = await mkdtemp(join(tmpdir(), "forge-explore-q-limit-"));
36 app = createApp({ dataDir, port: 0 });
37 servers = await serve(app);
38 const user = await createUser(app.db, { origin: "seed", trusted: true, username: "quinn", email: "[email protected]", password: "a-good-passphrase" });
39 if (!user.ok) throw new Error(user.error);
40 token = createToken(app.db, user.value.id, "ro", "ro").token;
41 });
42 afterAll(async () => {
43 servers.stop();
44 await rm(dataDir, { recursive: true, force: true });
45 });
46
47 describe("the page", () => {
48 test("/explore with a 5,000-character q repeats at most the capped query, in the box and every link", async () => {
49 const res = await fetch(`${servers.url}/explore?q=${paste}`);
50 expect(res.status).toBe(200);
51 const text = await res.text();
52 expect(text).toContain("data-query-trimmed");
53 expect(text).toContain(`value="${"a".repeat(MAX_QUERY_CHARS)}"`);
54 // The sign-in and sign-up `next` and the palette's Sign in carried the whole paste before.
55 expect(longestRun(text)).toBe(MAX_QUERY_CHARS);
56 expect(text).toContain(`/signin?next=${encodeURIComponent(`/explore?q=${"a".repeat(MAX_QUERY_CHARS)}`)}`);
57 });
58
59 test("a query within the limit keeps its address as it came", async () => {
60 const text = await (await fetch(`${servers.url}/explore?q=abc&sort=new`)).text();
61 expect(text).toContain(`/signin?next=${encodeURIComponent("/explore?q=abc&sort=new")}`);
62 });
63 });
64
65 describe("the API", () => {
66 for (const path of ["/api/v1/explore", "/api/v1/search"]) {
67 test(`${path} answers 422 naming q`, async () => {
68 const res = await fetch(`${servers.url}${path}?q=${over}`);
69 expect(res.status).toBe(422);
70 expect(((await res.json()) as { field: string }).field).toBe("q");
71 });
72 test(`${path} takes ${MAX_QUERY_CHARS} emoji, as the MCP tools do`, async () => {
73 const res = await fetch(`${servers.url}${path}?q=${encodeURIComponent("😀".repeat(MAX_QUERY_CHARS))}`, { headers: { authorization: `Bearer ${token}` } });
74 expect(res.status).toBe(200);
75 });
76 }
77 });
78
79 describe("the MCP tools", () => {
80 for (const name of ["explore", "search"]) {
81 test(`${name} refuses a q over the limit, naming it`, async () => {
82 const answer = await tool(name, { q: over });
83 expect(answer.error).toBe(true);
84 expect(answer.text).toContain("`q`");
85 expect(answer.text).toContain(String(MAX_QUERY_CHARS));
86 });
87 test(`${name} takes a q at the limit`, async () => {
88 expect((await tool(name, { q: "a".repeat(MAX_QUERY_CHARS) })).error).toBe(false);
89 });
90 test(`${name} counts code points like the page and the API: ${MAX_QUERY_CHARS} emoji are at the limit`, async () => {
91 expect((await tool(name, { q: "😀".repeat(MAX_QUERY_CHARS) })).error).toBe(false);
92 expect((await tool(name, { q: "😀".repeat(MAX_QUERY_CHARS + 1) })).error).toBe(true);
93 });
94 }
95 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs