Public: anyone can read this; the crew can change it.
GET /api/v1/explore pages with a cursor and a bounded limit, and refuses junk parameters with 422 #12333
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12317-get-api-v1-explore-pages-with-a-cursor-a → main 6 commits fork 2 threads · 2 unresolved
6 files
changed +139
−7
docs/REFERENCE.md +13
| @@ -366,10 +366,23 @@ does, by the same action, with each row's forks, merges, hours in use and | ||
| 366 | 366 | freshness, and a "Read as:" line says how the line was understood. Enter on an |
| 367 | 367 | app opens it; a person or forge row has no terminal page and says where the |
| 368 | 368 | web has it. A plain line still matches your repositories by name. |
| 369 | ||
| 369 | 370 | An unknown `kind:` or `sort:` word is dropped and named after "Read as:", with |
| 370 | 371 | the words it takes, on the page, in the terminal and in a `note` field of |
| 371 | 372 | `/api/v1/explore` and the MCP `explore` tool; results still show. |
| 372 | 373 | |
| 374 | `GET /api/v1/explore` takes `limit` (an integer 1 to 100, default 30) and | |
| 375 | `cursor` (opaque: pass back the `next_cursor` of the page before; null on the | |
| 376 | last page), and answers `{ query, items, next_cursor }`. Rows order by the | |
| 377 | sort value, then kind and name, so tied scores never repeat or skip a row | |
| 378 | across pages. A `sort`, `kind` or `limit` parameter that is unknown or not an | |
| 379 | integer in range is a 422 `{ error: "invalid", field, message }` naming the | |
| 380 | field (a word typed inside `q` is still a 400 `unknown_filter`); a cursor that | |
| 381 | is not one is `invalid_cursor` 400. A cursor is a position in one ranking | |
| 382 | snapshot, which the forge keeps for 60 seconds: scores that change between | |
| 383 | pages can move a row across the position, so a client that must see a stable | |
| 384 | list reads the pages in one minute. | |
| 385 | ||
| 373 | 386 | A signed-out visitor to `/explore` sees **Make it yours** on every public app (not on a private one, nor one only its writers may enter); it leads to sign-in and back to the plan of their copy, and nothing is copied until they confirm. Behind a platform edge the sign-in is the edge's; a newcomer it vouched for is sent to `/signup` and may need to come back. API and MCP rows say `copyable: true` for a guest too; making the copy needs a signed-in caller, and the terminal is always signed in and offers `Y` on the repository screen. |
| 374 | 387 | |
| 375 | 388 | ## On your machine |
src/actions/explore.ts +3 −2
| @@ -450,6 +450,7 @@ export async function explorePage( | ||
| 450 | 450 | cursor: string | null, |
| 451 | 451 | visible?: (row: ExploreRow) => boolean, |
| 452 | 452 | now?: number, |
| 453 | limit: number = ROWS, | |
| 453 | 454 | ): Promise<ExplorePage | null> { |
| 454 | 455 | const after = cursor === null ? undefined : readCursor(cursor, query); |
| 455 | 456 | if (after === null) return null; |
| @@ -458,10 +459,10 @@ export async function explorePage( | ||
| 458 | 459 | const all = await everything(app, viewer, query, now); |
| 459 | 460 | const rows = visible === undefined ? all : all.filter(visible); |
| 460 | 461 | const rest = after === undefined ? rows : rows.filter((r) => compare(r, after, query) > 0); |
| 461 | const items = rest.slice(0, ROWS); | |
| 462 | const items = rest.slice(0, limit); | |
| 462 | 463 | return { |
| 463 | 464 | items, |
| 464 | nextCursor: rest.length > ROWS ? writeCursor(items[items.length - 1]!, query) : null, | |
| 465 | nextCursor: rest.length > limit ? writeCursor(items[items.length - 1]!, query) : null, | |
| 465 | 466 | total: rows.length, |
| 466 | 467 | }; |
| 467 | 468 | } |
src/contract/openapi/routes/getApiV1Explore.ts +2 −2
| @@ -1,3 +1,3 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 1 | import { limitTo, type Route } from "../shared.ts"; | |
| 2 | 2 | |
| 3 | export const route: Route = { method: "get", path: "/api/v1/explore", summary: "Explore apps and repositories; `q` takes words and `kind:`/`owner:` filters; answers { query, items, next_cursor } (60 rows a page, next_cursor null on the last); cursor from next_cursor; a bad cursor is a 400; the answer carries an ETag (cache-control private, no-cache; vary authorization, cookie) that changes when anything listed changes, and a repeat sending it in If-None-Match gets 304 with no body", auth: false, role: "anonymous", alsoStatus: [304], query: { q: "string", cursor: "string" } }; | |
| 3 | export const route: Route = { method: "get", path: "/api/v1/explore", summary: "Explore apps and repositories; `q` takes words and `kind:`/`owner:` filters; answers { query, items, next_cursor } (`limit` 1 to 100 rows a page, default 30; next_cursor null on the last; a bad `limit`, `sort` or `kind` is a 422 naming the field); cursor from next_cursor; a bad cursor is a 400; the answer carries an ETag (cache-control private, no-cache; vary authorization, cookie) that changes when anything listed changes, and a repeat sending it in If-None-Match gets 304 with no body", auth: false, role: "anonymous", alsoStatus: [304, 422], query: { q: "string", cursor: "string", limit: limitTo(100), sort: "string", kind: "string" } }; |
src/http/api.ts +13 −2
| @@ -3533,8 +3533,19 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 3533 | 3533 | // Explore (#607): the rows /explore shows, for the same one line. |
| 3534 | 3534 | router.add("GET", "/api/v1/explore", async (ctx) => { |
| 3535 | 3535 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 3536 | // The separate `sort`, `kind` and `limit` parameters are refused with 422 | |
| 3537 | // naming the field; a word typed in `q` keeps its 400 `unknown_filter`. | |
| 3538 | const limitRaw = ctx.url.searchParams.get("limit"); | |
| 3539 | if (limitRaw !== null && !(/^\d{1,3}$/.test(limitRaw) && Number(limitRaw) >= 1 && Number(limitRaw) <= 100)) | |
| 3540 | return json({ error: "invalid", field: "limit", message: "limit is an integer from 1 to 100" }, 422); | |
| 3541 | const limit = limitRaw === null ? 30 : Number(limitRaw); | |
| 3536 | 3542 | const line = exploreLineFrom((name) => ctx.url.searchParams.get(name)); |
| 3537 | 3543 | const { query, unknown } = readExploreLine(line); |
| 3544 | for (const field of ["sort", "kind"] as const) { | |
| 3545 | const given = ctx.url.searchParams.get(field); | |
| 3546 | if (given !== null && given.trim() !== "" && unknown.some((w) => w.toLowerCase().startsWith(`${field}:`))) | |
| 3547 | return json({ error: "invalid", field, message: exploreNote(unknown) }, 422); | |
| 3548 | } | |
| 3538 | 3549 | const suggest = exploreSuggestion(line); |
| 3539 | 3550 | if (unknown.length > 0) |
| 3540 | 3551 | return json({ error: "unknown_filter", message: exploreNote(unknown), ...(suggest === undefined ? {} : { suggest: suggest.line }) }, 400); |
| @@ -3563,13 +3574,13 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 3563 | 3574 | const confined = mine === undefined ? "" : mine.map((r) => r.id).join(","); |
| 3564 | 3575 | const children = ctx.auth.user?.isAdmin === true ? JSON.stringify((await listChildren(ctx.app)).map((c) => [c.name, c.done])) : ""; |
| 3565 | 3576 | const etag = weakEtag( |
| 3566 | [exploreLine(query), cursor ?? "", viewer, confined, stamp, children].join("\n"), | |
| 3577 | [exploreLine(query), cursor ?? "", String(limit), viewer, confined, stamp, children].join("\n"), | |
| 3567 | 3578 | ); |
| 3568 | 3579 | headers = { etag, "cache-control": "private, no-cache", vary: "authorization, cookie" }; |
| 3569 | 3580 | if (holds(ctx.request, etag)) return new Response(null, { status: 304, headers }); |
| 3570 | 3581 | } |
| 3571 | 3582 | const page = await timePhaseAsync(ctx.request, "explore", () => |
| 3572 | explorePage(ctx.app, ctx.auth.user, query, cursor, visible), | |
| 3583 | explorePage(ctx.app, ctx.auth.user, query, cursor, visible, undefined, limit), | |
| 3573 | 3584 | ); |
| 3574 | 3585 | if (page === null) |
| 3575 | 3586 | return apiError("invalid_cursor", 400, "cursor is the next_cursor of a page before"); |
test/explore-api-paging.test.ts added +107
| @@ -0,0 +1,107 @@ | ||
| 1 | /** | |
| 2 | * GET /api/v1/explore pages by `limit` and `cursor`, and refuses junk parameters | |
| 3 | * with 422 naming the field (#12317). | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createRepo } from "../src/actions/repos.ts"; | |
| 11 | import { createApp as insertApp } from "../src/core/apps.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 14 | import { promote } from "./helpers/members.ts"; | |
| 15 | ||
| 16 | let dataDir: string; | |
| 17 | let app: App; | |
| 18 | let servers: Servers; | |
| 19 | const N = 35; | |
| 20 | ||
| 21 | interface Page { items: { name: string; kind: string }[]; next_cursor: string | null } | |
| 22 | const get = (qs: string) => fetch(`${servers.url}/api/v1/explore?${qs}`); | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | dataDir = await mkdtemp(join(tmpdir(), "forge-explore-paging-")); | |
| 26 | app = createApp({ dataDir, port: 0 }); | |
| 27 | servers = await serve(app); | |
| 28 | const r = await createUser(app.db, { origin: "seed", trusted: true, allowReserved: true, username: "root", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 29 | if (!r.ok) throw new Error(r.error); | |
| 30 | promote(app.db, r.value); | |
| 31 | // Every app has the same (zero) score and forks, so only the name tiebreak orders them. | |
| 32 | for (let i = 0; i < N; i++) { | |
| 33 | const repo = await createRepo(app, r.value, { name: `tie-${i}`, visibility: "public" }); | |
| 34 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 35 | const rec = insertApp(app.db, { repoId: repo.value.id, slug: `tie-${i}`, access: "public" }); | |
| 36 | if (!rec.ok) throw new Error(rec.error); | |
| 37 | } | |
| 38 | }); | |
| 39 | afterAll(async () => { | |
| 40 | servers.stop(); | |
| 41 | app.close(); | |
| 42 | await rm(dataDir, { recursive: true, force: true }); | |
| 43 | }); | |
| 44 | ||
| 45 | describe("paging", () => { | |
| 46 | test("the default page holds exactly 30 rows and a cursor to the rest", async () => { | |
outdated
This does not pin the default. With about 7 rows it passes for any default of 7 or more. Seed over 30 apps and assert exactly 30 rows plus a next_cursor. |
||
| 47 | const page = (await (await get("kind=app")).json()) as Page; | |
| 48 | expect(page.items.length).toBe(30); | |
| 49 | expect(typeof page.next_cursor).toBe("string"); | |
| 50 | }); | |
| 51 | ||
| 52 | for (const sort of ["score", "forks", "name"]) { | |
| 53 | test(`paging by 3 under sort=${sort} yields every row once, ties included`, async () => { | |
| 54 | const seen: string[] = []; | |
| 55 | let cursor: string | null = null; | |
| 56 | let pages = 0; | |
| 57 | do { | |
| 58 | const res = await get(`kind=app&sort=${sort}&limit=3${cursor === null ? "" : `&cursor=${cursor}`}`); | |
| 59 | expect(res.status).toBe(200); | |
| 60 | const page = (await res.json()) as Page; | |
| 61 | expect(page.items.length).toBeLessThanOrEqual(3); | |
| 62 | seen.push(...page.items.map((i) => i.name)); | |
| 63 | cursor = page.next_cursor; | |
| 64 | pages++; | |
| 65 | } while (cursor !== null && pages < 20); | |
| 66 | expect(cursor).toBeNull(); | |
| 67 | expect(pages).toBeGreaterThan(1); | |
| 68 | expect(new Set(seen).size).toBe(seen.length); | |
| 69 | const whole = (await (await get(`kind=app&sort=${sort}&limit=100`)).json()) as Page; | |
| 70 | expect(seen).toEqual(whole.items.map((i) => i.name)); | |
| 71 | expect(seen.length).toBeGreaterThanOrEqual(N); | |
| 72 | }); | |
| 73 | } | |
| 74 | ||
| 75 | test("limit=1 gives a one-row page with a cursor", async () => { | |
| 76 | const page = (await (await get("kind=app&limit=1")).json()) as Page; | |
| 77 | expect(page.items.length).toBe(1); | |
| 78 | expect(typeof page.next_cursor).toBe("string"); | |
| 79 | }); | |
| 80 | ||
| 81 | test("a different limit is a different ETag", async () => { | |
| 82 | const a = await get("kind=app&limit=2"); | |
| 83 | const b = await get("kind=app&limit=3"); | |
| 84 | await a.arrayBuffer(); | |
| 85 | await b.arrayBuffer(); | |
| 86 | expect(a.headers.get("etag")).not.toBe(b.headers.get("etag")); | |
| 87 | }); | |
| 88 | }); | |
| 89 | ||
| 90 | describe("refusals", () => { | |
| 91 | const refused = async (qs: string, field: string) => { | |
| 92 | const res = await get(qs); | |
| 93 | expect(res.status).toBe(422); | |
| 94 | const body = (await res.json()) as { error: string; field: string }; | |
| 95 | expect(body.field).toBe(field); | |
| 96 | }; | |
| 97 | test("a non-numeric limit", () => refused("limit=abc", "limit")); | |
| 98 | test("a fractional limit", () => refused("limit=2.5", "limit")); | |
| 99 | test("limit 0", () => refused("limit=0", "limit")); | |
| 100 | test("limit 101", () => refused("limit=101", "limit")); | |
| 101 | test("an empty limit", () => refused("limit=", "limit")); | |
| 102 | test("an unknown sort", () => refused("sort=bogus", "sort")); | |
| 103 | test("an unknown kind", () => refused("kind=nope", "kind")); | |
| 104 | test("a cursor that is not one is still 400", async () => { | |
| 105 | expect((await get("cursor=%25%25")).status).toBe(400); | |
| 106 | }); | |
| 107 | }); |
test/explore.test.ts +1 −1
| @@ -509,7 +509,7 @@ describe("explore past the first page", () => { | ||
| 509 | 509 | |
| 510 | 510 | const get = async (q: string, cursor?: string) => |
| 511 | 511 | fetch( |
| 512 | `${servers.url}/api/v1/explore?q=${encodeURIComponent(q)}${cursor === undefined ? "" : `&cursor=${encodeURIComponent(cursor)}`}`, | |
| 512 | `${servers.url}/api/v1/explore?q=${encodeURIComponent(q)}&limit=60${cursor === undefined ? "" : `&cursor=${encodeURIComponent(cursor)}`}`, | |
| 513 | 513 | ); |
| 514 | 514 | |
| 515 | 515 | beforeAll(async () => { |
new above
offline
This splits the terminal explore paragraph (the "An unknown kind:" sentence below belongs with the text above). Move the API parameters into their own paragraph or section.