Public: anyone can read this; the crew can change it.

GET /api/v1/explore pages with a cursor and a bounded limit, and refuses junk parameters with 422 #12333

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12317-get-api-v1-explore-pages-with-a-cursor-a → main 6 commits fork 2 threads · 2 unresolved

  • Merged. scout/12317-get-api-v1-explore-pages-with-a-cursor-a was merged into main as 5f71e3e.

Review

Sign in to review this change.
6 files changed +139 −7
docs/REFERENCE.md +13
@@ -366,10 +366,23 @@ does, by the same action, with each row's forks, merges, hours in use and
366366 freshness, and a "Read as:" line says how the line was understood. Enter on an
367367 app opens it; a person or forge row has no terminal page and says where the
368368 web has it. A plain line still matches your repositories by name.
369
369370 An unknown `kind:` or `sort:` word is dropped and named after "Read as:", with
370371 the words it takes, on the page, in the terminal and in a `note` field of
371372 `/api/v1/explore` and the MCP `explore` tool; results still show.
372373
374 `GET /api/v1/explore` takes `limit` (an integer 1 to 100, default 30) and
375 `cursor` (opaque: pass back the `next_cursor` of the page before; null on the
376 last page), and answers `{ query, items, next_cursor }`. Rows order by the
377 sort value, then kind and name, so tied scores never repeat or skip a row
378 across pages. A `sort`, `kind` or `limit` parameter that is unknown or not an
outdated

This splits the terminal explore paragraph (the "An unknown kind:" sentence below belongs with the text above). Move the API parameters into their own paragraph or section.

379 integer in range is a 422 `{ error: "invalid", field, message }` naming the
380 field (a word typed inside `q` is still a 400 `unknown_filter`); a cursor that
381 is not one is `invalid_cursor` 400. A cursor is a position in one ranking
382 snapshot, which the forge keeps for 60 seconds: scores that change between
383 pages can move a row across the position, so a client that must see a stable
384 list reads the pages in one minute.
385
373386 A signed-out visitor to `/explore` sees **Make it yours** on every public app (not on a private one, nor one only its writers may enter); it leads to sign-in and back to the plan of their copy, and nothing is copied until they confirm. Behind a platform edge the sign-in is the edge's; a newcomer it vouched for is sent to `/signup` and may need to come back. API and MCP rows say `copyable: true` for a guest too; making the copy needs a signed-in caller, and the terminal is always signed in and offers `Y` on the repository screen.
374387
375388 ## On your machine
src/actions/explore.ts +3 −2
@@ -450,6 +450,7 @@ export async function explorePage(
450450 cursor: string | null,
451451 visible?: (row: ExploreRow) => boolean,
452452 now?: number,
453 limit: number = ROWS,
453454 ): Promise<ExplorePage | null> {
454455 const after = cursor === null ? undefined : readCursor(cursor, query);
455456 if (after === null) return null;
@@ -458,10 +459,10 @@ export async function explorePage(
458459 const all = await everything(app, viewer, query, now);
459460 const rows = visible === undefined ? all : all.filter(visible);
460461 const rest = after === undefined ? rows : rows.filter((r) => compare(r, after, query) > 0);
461 const items = rest.slice(0, ROWS);
462 const items = rest.slice(0, limit);
462463 return {
463464 items,
464 nextCursor: rest.length > ROWS ? writeCursor(items[items.length - 1]!, query) : null,
465 nextCursor: rest.length > limit ? writeCursor(items[items.length - 1]!, query) : null,
465466 total: rows.length,
466467 };
467468 }
src/contract/openapi/routes/getApiV1Explore.ts +2 −2
@@ -1,3 +1,3 @@
1 import type { Route } from "../shared.ts";
1 import { limitTo, type Route } from "../shared.ts";
22
3 export const route: Route = { method: "get", path: "/api/v1/explore", summary: "Explore apps and repositories; `q` takes words and `kind:`/`owner:` filters; answers { query, items, next_cursor } (60 rows a page, next_cursor null on the last); cursor from next_cursor; a bad cursor is a 400; the answer carries an ETag (cache-control private, no-cache; vary authorization, cookie) that changes when anything listed changes, and a repeat sending it in If-None-Match gets 304 with no body", auth: false, role: "anonymous", alsoStatus: [304], query: { q: "string", cursor: "string" } };
3 export const route: Route = { method: "get", path: "/api/v1/explore", summary: "Explore apps and repositories; `q` takes words and `kind:`/`owner:` filters; answers { query, items, next_cursor } (`limit` 1 to 100 rows a page, default 30; next_cursor null on the last; a bad `limit`, `sort` or `kind` is a 422 naming the field); cursor from next_cursor; a bad cursor is a 400; the answer carries an ETag (cache-control private, no-cache; vary authorization, cookie) that changes when anything listed changes, and a repeat sending it in If-None-Match gets 304 with no body", auth: false, role: "anonymous", alsoStatus: [304, 422], query: { q: "string", cursor: "string", limit: limitTo(100), sort: "string", kind: "string" } };
src/http/api.ts +13 −2
@@ -3533,8 +3533,19 @@ export function apiRoutes(router: Router<Handler>): void {
35333533 // Explore (#607): the rows /explore shows, for the same one line.
35343534 router.add("GET", "/api/v1/explore", async (ctx) => {
35353535 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
3536 // The separate `sort`, `kind` and `limit` parameters are refused with 422
3537 // naming the field; a word typed in `q` keeps its 400 `unknown_filter`.
3538 const limitRaw = ctx.url.searchParams.get("limit");
3539 if (limitRaw !== null && !(/^\d{1,3}$/.test(limitRaw) && Number(limitRaw) >= 1 && Number(limitRaw) <= 100))
3540 return json({ error: "invalid", field: "limit", message: "limit is an integer from 1 to 100" }, 422);
3541 const limit = limitRaw === null ? 30 : Number(limitRaw);
35363542 const line = exploreLineFrom((name) => ctx.url.searchParams.get(name));
35373543 const { query, unknown } = readExploreLine(line);
3544 for (const field of ["sort", "kind"] as const) {
3545 const given = ctx.url.searchParams.get(field);
3546 if (given !== null && given.trim() !== "" && unknown.some((w) => w.toLowerCase().startsWith(`${field}:`)))
3547 return json({ error: "invalid", field, message: exploreNote(unknown) }, 422);
3548 }
35383549 const suggest = exploreSuggestion(line);
35393550 if (unknown.length > 0)
35403551 return json({ error: "unknown_filter", message: exploreNote(unknown), ...(suggest === undefined ? {} : { suggest: suggest.line }) }, 400);
@@ -3563,13 +3574,13 @@ export function apiRoutes(router: Router<Handler>): void {
35633574 const confined = mine === undefined ? "" : mine.map((r) => r.id).join(",");
35643575 const children = ctx.auth.user?.isAdmin === true ? JSON.stringify((await listChildren(ctx.app)).map((c) => [c.name, c.done])) : "";
35653576 const etag = weakEtag(
3566 [exploreLine(query), cursor ?? "", viewer, confined, stamp, children].join("\n"),
3577 [exploreLine(query), cursor ?? "", String(limit), viewer, confined, stamp, children].join("\n"),
35673578 );
35683579 headers = { etag, "cache-control": "private, no-cache", vary: "authorization, cookie" };
35693580 if (holds(ctx.request, etag)) return new Response(null, { status: 304, headers });
35703581 }
35713582 const page = await timePhaseAsync(ctx.request, "explore", () =>
3572 explorePage(ctx.app, ctx.auth.user, query, cursor, visible),
3583 explorePage(ctx.app, ctx.auth.user, query, cursor, visible, undefined, limit),
35733584 );
35743585 if (page === null)
35753586 return apiError("invalid_cursor", 400, "cursor is the next_cursor of a page before");
test/explore-api-paging.test.ts added +107
@@ -0,0 +1,107 @@
1 /**
2 * GET /api/v1/explore pages by `limit` and `cursor`, and refuses junk parameters
3 * with 422 naming the field (#12317).
4 */
5 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createRepo } from "../src/actions/repos.ts";
11 import { createApp as insertApp } from "../src/core/apps.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import { promote } from "./helpers/members.ts";
15
16 let dataDir: string;
17 let app: App;
18 let servers: Servers;
19 const N = 35;
20
21 interface Page { items: { name: string; kind: string }[]; next_cursor: string | null }
22 const get = (qs: string) => fetch(`${servers.url}/api/v1/explore?${qs}`);
23
24 beforeAll(async () => {
25 dataDir = await mkdtemp(join(tmpdir(), "forge-explore-paging-"));
26 app = createApp({ dataDir, port: 0 });
27 servers = await serve(app);
28 const r = await createUser(app.db, { origin: "seed", trusted: true, allowReserved: true, username: "root", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
29 if (!r.ok) throw new Error(r.error);
30 promote(app.db, r.value);
31 // Every app has the same (zero) score and forks, so only the name tiebreak orders them.
32 for (let i = 0; i < N; i++) {
33 const repo = await createRepo(app, r.value, { name: `tie-${i}`, visibility: "public" });
34 if (!repo.ok) throw new Error(String(repo.error));
35 const rec = insertApp(app.db, { repoId: repo.value.id, slug: `tie-${i}`, access: "public" });
36 if (!rec.ok) throw new Error(rec.error);
37 }
38 });
39 afterAll(async () => {
40 servers.stop();
41 app.close();
42 await rm(dataDir, { recursive: true, force: true });
43 });
44
45 describe("paging", () => {
46 test("the default page holds exactly 30 rows and a cursor to the rest", async () => {
outdated

This does not pin the default. With about 7 rows it passes for any default of 7 or more. Seed over 30 apps and assert exactly 30 rows plus a next_cursor.

47 const page = (await (await get("kind=app")).json()) as Page;
48 expect(page.items.length).toBe(30);
49 expect(typeof page.next_cursor).toBe("string");
50 });
51
52 for (const sort of ["score", "forks", "name"]) {
53 test(`paging by 3 under sort=${sort} yields every row once, ties included`, async () => {
54 const seen: string[] = [];
55 let cursor: string | null = null;
56 let pages = 0;
57 do {
58 const res = await get(`kind=app&sort=${sort}&limit=3${cursor === null ? "" : `&cursor=${cursor}`}`);
59 expect(res.status).toBe(200);
60 const page = (await res.json()) as Page;
61 expect(page.items.length).toBeLessThanOrEqual(3);
62 seen.push(...page.items.map((i) => i.name));
63 cursor = page.next_cursor;
64 pages++;
65 } while (cursor !== null && pages < 20);
66 expect(cursor).toBeNull();
67 expect(pages).toBeGreaterThan(1);
68 expect(new Set(seen).size).toBe(seen.length);
69 const whole = (await (await get(`kind=app&sort=${sort}&limit=100`)).json()) as Page;
70 expect(seen).toEqual(whole.items.map((i) => i.name));
71 expect(seen.length).toBeGreaterThanOrEqual(N);
72 });
73 }
74
75 test("limit=1 gives a one-row page with a cursor", async () => {
76 const page = (await (await get("kind=app&limit=1")).json()) as Page;
77 expect(page.items.length).toBe(1);
78 expect(typeof page.next_cursor).toBe("string");
79 });
80
81 test("a different limit is a different ETag", async () => {
82 const a = await get("kind=app&limit=2");
83 const b = await get("kind=app&limit=3");
84 await a.arrayBuffer();
85 await b.arrayBuffer();
86 expect(a.headers.get("etag")).not.toBe(b.headers.get("etag"));
87 });
88 });
89
90 describe("refusals", () => {
91 const refused = async (qs: string, field: string) => {
92 const res = await get(qs);
93 expect(res.status).toBe(422);
94 const body = (await res.json()) as { error: string; field: string };
95 expect(body.field).toBe(field);
96 };
97 test("a non-numeric limit", () => refused("limit=abc", "limit"));
98 test("a fractional limit", () => refused("limit=2.5", "limit"));
99 test("limit 0", () => refused("limit=0", "limit"));
100 test("limit 101", () => refused("limit=101", "limit"));
101 test("an empty limit", () => refused("limit=", "limit"));
102 test("an unknown sort", () => refused("sort=bogus", "sort"));
103 test("an unknown kind", () => refused("kind=nope", "kind"));
104 test("a cursor that is not one is still 400", async () => {
105 expect((await get("cursor=%25%25")).status).toBe(400);
106 });
107 });
test/explore.test.ts +1 −1
@@ -509,7 +509,7 @@ describe("explore past the first page", () => {
509509
510510 const get = async (q: string, cursor?: string) =>
511511 fetch(
512 `${servers.url}/api/v1/explore?q=${encodeURIComponent(q)}${cursor === undefined ? "" : `&cursor=${encodeURIComponent(cursor)}`}`,
512 `${servers.url}/api/v1/explore?q=${encodeURIComponent(q)}&limit=60${cursor === undefined ? "" : `&cursor=${encodeURIComponent(cursor)}`}`,
513513 );
514514
515515 beforeAll(async () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs