Public: anyone can read this; the crew can change it.
/explore's rank phase stays under 15 ms with 500 public repositories #4011
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3394-explore-s-rank-phase-stays-under-15-ms-w → main 4 commits fork
2 files
changed +265
−21
src/actions/explore.ts +80 −21
| @@ -15,6 +15,7 @@ | ||
| 15 | 15 | * - Forges are the children this forge spawned, for a site admin. |
| 16 | 16 | */ |
| 17 | 17 | |
| 18 | import type { Database } from "bun:sqlite"; | |
| 18 | 19 | import type { App } from "../app.ts"; |
| 19 | 20 | import { appsForRepos } from "../core/apps.ts"; |
| 20 | 21 | import { markOf, type Mark } from "../core/marks.ts"; |
| @@ -26,6 +27,8 @@ import { | ||
| 26 | 27 | } from "../core/front.ts"; |
| 27 | 28 | import { canEnterApp, listReposVisibleTo, type Repo } from "../core/repos.ts"; |
| 28 | 29 | import { getUsersByIds, isActive, type User } from "../core/users.ts"; |
| 30 | import { writeStamp } from "../core/write_clock.ts"; | |
| 31 | import { Stamped } from "../kernel/stamped.ts"; | |
| 29 | 32 | import { listChildren } from "./spawn.ts"; |
| 30 | 33 | |
| 31 | 34 | export type ExploreKind = "app" | "person" | "forge"; |
| @@ -136,12 +139,57 @@ export interface ExploreRow { | ||
| 136 | 139 | readonly copyable?: boolean; |
| 137 | 140 | } |
| 138 | 141 | |
| 142 | /** | |
| 143 | * The ranking as last computed, per viewer (id and admin standing) and line | |
| 144 | * (#3394). Valid while the forge's write clock has not moved — a new star, | |
| 145 | * push or merge moves it, so it shows on the next request — and for 5 seconds | |
| 146 | * at most, because a score ages with the clock. Admins' answers are kept too; | |
| 147 | * the spawned forges, which no write clock sees, are read fresh on every call | |
| 148 | * and added to a copy after the kept read. A call at a given `now` is never kept. | |
| 149 | */ | |
| 150 | const RANK_MAX_AGE_MS = 5_000; | |
| 151 | const ranked = new WeakMap<Database, Stamped<ExploreRow[]>>(); | |
| 152 | ||
| 139 | 153 | export async function explore( |
| 140 | 154 | app: App, |
| 141 | 155 | viewer: User | undefined, |
| 142 | 156 | query: ExploreQuery, |
| 143 | now = Date.now(), | |
| 157 | now?: number, | |
| 144 | 158 | ): Promise<ExploreRow[]> { |
| 159 | const forgeRows = await forges(app, viewer, query); | |
| 160 | const stamp = now === undefined ? writeStamp(app.db) : undefined; | |
| 161 | if (stamp === undefined) return place(rank(app, viewer, query, now ?? Date.now()), forgeRows, query); | |
| 162 | let cache = ranked.get(app.db); | |
| 163 | if (cache === undefined) { | |
| 164 | cache = new Stamped<ExploreRow[]>({ maxEntries: 64, maxAgeMs: RANK_MAX_AGE_MS }); | |
| 165 | ranked.set(app.db, cache); | |
| 166 | } | |
| 167 | const kept = cache.get(`${viewer?.id ?? ""}\n${viewer?.isAdmin === true ? "admin" : ""}\n${exploreLine(query)}`, stamp, () => | |
| 168 | rank(app, viewer, query, Date.now()), | |
| 169 | ); | |
| 170 | return place(kept, forgeRows, query); | |
| 171 | } | |
| 172 | ||
| 173 | /** The forges this forge spawned, for a site admin asking for them. */ | |
| 174 | async function forges(app: App, viewer: User | undefined, query: ExploreQuery): Promise<ExploreRow[]> { | |
| 175 | if (viewer?.isAdmin !== true || (query.kind !== "all" && query.kind !== "forge")) return []; | |
| 176 | return (await listChildren(app)).map((child) => ({ | |
| 177 | kind: "forge", | |
| 178 | name: child.name, | |
| 179 | url: child.url, | |
| 180 | description: child.done ? "spawned from this forge" : "being spawned", | |
| 181 | numbers: { forks: 0, ships: 0, awakeHours: 0, apps: 0 }, | |
| 182 | shippedAt: null, | |
| 183 | score: 0, | |
| 184 | })); | |
| 185 | } | |
| 186 | ||
| 187 | function rank( | |
| 188 | app: App, | |
| 189 | viewer: User | undefined, | |
| 190 | query: ExploreQuery, | |
| 191 | now: number, | |
| 192 | ): ExploreRow[] { | |
| 145 | 193 | const db = app.db; |
| 146 | 194 | const repos = listReposVisibleTo(db, viewer, { |
| 147 | 195 | limit: READ, |
| @@ -243,26 +291,24 @@ export async function explore( | ||
| 243 | 291 | }); |
| 244 | 292 | }); |
| 245 | 293 | |
| 246 | const forgeRows: ExploreRow[] = | |
| 247 | viewer?.isAdmin === true && (query.kind === "all" || query.kind === "forge") | |
| 248 | ? (await listChildren(app)).map((child) => ({ | |
| 249 | kind: "forge", | |
| 250 | name: child.name, | |
| 251 | url: child.url, | |
| 252 | description: child.done ? "spawned from this forge" : "being spawned", | |
| 253 | numbers: { forks: 0, ships: 0, awakeHours: 0, apps: 0 }, | |
| 254 | shippedAt: null, | |
| 255 | score: 0, | |
| 256 | })) | |
| 257 | : []; | |
| 258 | ||
| 259 | 294 | const text = query.text.toLowerCase(); |
| 260 | const rows = [...appRows, ...people.values(), ...forgeRows].filter( | |
| 261 | (row) => | |
| 262 | (query.kind === "all" || row.kind === query.kind) && | |
| 263 | (text === "" || | |
| 264 | `${row.name} ${row.description}`.toLowerCase().includes(text)), | |
| 295 | // Kept rows are shared by every request that reads them: none may change. | |
| 296 | const freeze = (row: ExploreRow): ExploreRow => { | |
| 297 | Object.freeze(row.numbers); | |
| 298 | return Object.freeze(row); | |
| 299 | }; | |
| 300 | return order( | |
| 301 | [...appRows, ...people.values()].map(freeze).filter( | |
| 302 | (row) => | |
| 303 | (query.kind === "all" || row.kind === query.kind) && | |
| 304 | (text === "" || `${row.name} ${row.description}`.toLowerCase().includes(text)), | |
| 305 | ), | |
| 306 | query, | |
| 265 | 307 | ); |
| 308 | } | |
| 309 | ||
| 310 | /** Rows sorted as the query says, in place. */ | |
| 311 | function order(rows: ExploreRow[], query: ExploreQuery): ExploreRow[] { | |
| 266 | 312 | const by: Record<ExploreSort, (r: ExploreRow) => number | string> = { |
| 267 | 313 | score: (r) => r.score, |
| 268 | 314 | forks: (r) => r.numbers.forks, |
| @@ -272,11 +318,24 @@ export async function explore( | ||
| 272 | 318 | name: (r) => r.name.toLowerCase(), |
| 273 | 319 | }; |
| 274 | 320 | const key = by[query.sort]; |
| 275 | rows.sort((a, b) => { | |
| 321 | return rows.sort((a, b) => { | |
| 276 | 322 | const x = key(a); |
| 277 | 323 | const y = key(b); |
| 278 | 324 | if (query.sort === "name") return String(x).localeCompare(String(y)); |
| 279 | 325 | return (y as number) - (x as number) || a.name.localeCompare(b.name); |
| 280 | 326 | }); |
| 281 | return rows.slice(0, ROWS); | |
| 327 | } | |
| 328 | ||
| 329 | /** | |
| 330 | * The answer: the ranked rows (kept, so never changed here) with the forge | |
| 331 | * rows among them, cut to a page. Only the forge rows lack a write clock, | |
| 332 | * so they are added after the kept read, to a copy. | |
| 333 | */ | |
| 334 | function place(ranked: readonly ExploreRow[], forgeRows: ExploreRow[], query: ExploreQuery): ExploreRow[] { | |
| 335 | const text = query.text.toLowerCase(); | |
| 336 | const own = forgeRows.filter( | |
| 337 | (row) => text === "" || `${row.name} ${row.description}`.toLowerCase().includes(text), | |
| 338 | ); | |
| 339 | if (own.length === 0) return ranked.slice(0, ROWS); | |
| 340 | return order([...ranked, ...own], query).slice(0, ROWS); | |
| 282 | 341 | } |
test/explore-rank.test.ts added +185
| @@ -0,0 +1,185 @@ | ||
| 1 | /** /explore's rank phase stays under 15 ms with 500 public repositories (#3394). */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdir, mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { explore, parseExploreQuery } from "../src/actions/explore.ts"; | |
| 8 | import { createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { createApp as insertApp } from "../src/core/apps.ts"; | |
| 10 | import { createIssue } from "../src/core/issues.ts"; | |
| 11 | import { createUser, type User } from "../src/core/users.ts"; | |
| 12 | import { countStatements } from "../bench/run.ts"; | |
| 13 | import { quietLogs } from "./helpers/globals.ts"; | |
| 14 | import { proveAll } from "./helpers/verified.ts"; | |
| 15 | import { promote } from "./helpers/members.ts"; | |
| 16 | ||
| 17 | quietLogs(); | |
| 18 | ||
| 19 | let dataDir: string; | |
| 20 | let app: App; | |
| 21 | let owner: User; | |
| 22 | const DAY = 86_400_000; | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | dataDir = await mkdtemp(join(tmpdir(), "forge-explore-rank-")); | |
| 26 | app = createApp({ dataDir, port: 0 }); | |
| 27 | const made = await createUser(app.db, { | |
| 28 | origin: "seed", | |
| 29 | username: "olive", | |
| 30 | email: "[email protected]", | |
| 31 | password: "a-good-passphrase", | |
| 32 | adminIfFirst: false, | |
| 33 | }); | |
| 34 | if (!made.ok) throw new Error(made.error); | |
| 35 | proveAll(app); | |
| 36 | promote(app.db, made.value); | |
| 37 | owner = made.value; | |
| 38 | const now = Date.now(); | |
| 39 | for (let i = 0; i < 500; i++) { | |
| 40 | const repo = await createRepo(app, owner, { | |
| 41 | name: `app-${i}`, | |
| 42 | visibility: i === 7 ? "private" : "public", | |
| 43 | }); | |
| 44 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 45 | const rec = insertApp(app.db, { | |
| 46 | repoId: repo.value.id, | |
| 47 | slug: `olive-app-${i}`, | |
| 48 | access: i === 7 ? "private" : "public", | |
| 49 | }); | |
| 50 | if (!rec.ok) throw new Error(rec.error); | |
| 51 | // A week of activity: a merged pull every few repositories, hours awake on others. | |
| 52 | if (i % 3 === 0) { | |
| 53 | const issue = createIssue(app.db, { | |
| 54 | repoId: repo.value.id, | |
| 55 | authorId: owner.id, | |
| 56 | title: "change", | |
| 57 | kind: "pull", | |
| 58 | }); | |
| 59 | if (!issue.ok) throw new Error(issue.error); | |
| 60 | app.db | |
| 61 | .query( | |
| 62 | `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) | |
| 63 | VALUES (?1, 'main', 'b', '', '', ?2)`, | |
| 64 | ) | |
| 65 | .run(issue.value.id, new Date(now - (i % 7) * DAY - 3_600_000).toISOString()); | |
| 66 | } | |
| 67 | if (i % 2 === 0) | |
| 68 | app.db | |
| 69 | .query( | |
| 70 | `INSERT INTO meter (repo_id, env, placement, started_at, beat_at, ended_at) | |
| 71 | VALUES (?1, 'production', 'owned', ?2, ?3, ?3)`, | |
| 72 | ) | |
| 73 | .run( | |
| 74 | repo.value.id, | |
| 75 | new Date(now - (i % 6) * DAY - 5 * 3_600_000).toISOString(), | |
| 76 | new Date(now - (i % 6) * DAY).toISOString(), | |
| 77 | ); | |
| 78 | } | |
| 79 | }, 300_000); | |
| 80 | afterAll(async () => { | |
| 81 | app.close(); | |
| 82 | await rm(dataDir, { recursive: true, force: true }); | |
| 83 | }); | |
| 84 | ||
| 85 | const query = parseExploreQuery(""); | |
| 86 | ||
| 87 | test("a warm rank of 500 public repositories is a fraction of a cold one and under 15 ms", async () => { | |
| 88 | const cold: number[] = []; | |
| 89 | const warm: number[] = []; | |
| 90 | await explore(app, undefined, query); | |
| 91 | for (let i = 0; i < 5; i++) { | |
| 92 | let t = performance.now(); | |
| 93 | await explore(app, undefined, query, Date.now()); | |
| 94 | cold.push(performance.now() - t); | |
| 95 | t = performance.now(); | |
| 96 | await explore(app, undefined, query); | |
| 97 | warm.push(performance.now() - t); | |
| 98 | } | |
| 99 | expect(Math.min(...warm)).toBeLessThan(15); | |
| 100 | expect(Math.min(...warm) * 5).toBeLessThan(Math.min(...cold)); | |
| 101 | }); | |
| 102 | ||
| 103 | test("a cold rank runs many statements, a warm one only the write clock", async () => { | |
| 104 | const cold = countStatements(app.db); | |
| 105 | await explore(app, undefined, query, Date.now()); | |
| 106 | const coldN = cold.count; | |
| 107 | cold.stop(); | |
| 108 | await explore(app, undefined, query); | |
| 109 | const warm = countStatements(app.db); | |
| 110 | await explore(app, undefined, query); | |
| 111 | const warmN = warm.count; | |
| 112 | warm.stop(); | |
| 113 | expect(coldN).toBeGreaterThan(3); | |
| 114 | expect(warmN).toBeLessThanOrEqual(2); | |
| 115 | }); | |
| 116 | ||
| 117 | test("a warm request reads nothing but the write clock", async () => { | |
| 118 | await explore(app, undefined, query); | |
| 119 | const sql = countStatements(app.db); | |
| 120 | await explore(app, undefined, query); | |
| 121 | const n = sql.count; | |
| 122 | const only = sql.top().map((t) => t.sql); | |
| 123 | sql.stop(); | |
| 124 | expect(only.length).toBe(1); | |
| 125 | expect(only[0]).toContain("total_changes()"); | |
| 126 | expect(n).toBeLessThanOrEqual(2); | |
| 127 | }); | |
| 128 | ||
| 129 | test("the cached order is the computed order, and a new write shows on the next request", async () => { | |
| 130 | const first = await explore(app, undefined, query); | |
| 131 | const again = await explore(app, undefined, query); | |
| 132 | expect(again).toEqual(first); | |
| 133 | // Asked at a given time it is ranked afresh: the same order. | |
| 134 | const fresh = await explore(app, undefined, query, Date.now()); | |
| 135 | expect(fresh.map((r) => r.name)).toEqual(first.map((r) => r.name)); | |
| 136 | expect(first.length).toBe(60); | |
| 137 | expect(first.some((r) => r.name === "olive/app-7")).toBe(false); | |
| 138 | // A fork by someone else moves the repository up at the next request. | |
| 139 | const target = first.filter((r) => r.kind === "app").at(-1)!; | |
| 140 | const parent = app.db | |
| 141 | .query<{ id: string }, [string]>( | |
| 142 | `SELECT repo.id FROM repo JOIN user ON user.id = repo.owner_id WHERE user.username || '/' || repo.name = ?1`, | |
| 143 | ) | |
| 144 | .get(target.name)!; | |
| 145 | app.db.query(`UPDATE repo SET fork_parent_id = ?1 WHERE name = 'app-1'`).run(parent.id); | |
| 146 | const after = await explore(app, undefined, query); | |
| 147 | expect(after.find((r) => r.name === target.name)!.numbers.forks).toBe(1); | |
| 148 | }); | |
| 149 | ||
| 150 | test("an admin's warm request reads the write clock and the spawned forges, and still lists them", async () => { | |
| 151 | await mkdir(join(dataDir, "spawn"), { recursive: true }); | |
| 152 | await Bun.write( | |
| 153 | join(dataDir, "spawn", "kid.json"), | |
| 154 | JSON.stringify({ name: "kid", bodyId: "b", url: "http://kid", planHash: "h", done: true }), | |
| 155 | ); | |
| 156 | const forgeQuery = parseExploreQuery("kind:forge"); | |
| 157 | const admin = { ...owner, isAdmin: true } as User; | |
| 158 | await explore(app, admin, forgeQuery); | |
| 159 | const sql = countStatements(app.db); | |
| 160 | const rows = await explore(app, admin, forgeQuery); | |
| 161 | const only = sql.top().map((t) => t.sql); | |
| 162 | sql.stop(); | |
| 163 | expect(only.length).toBe(1); | |
| 164 | expect(only[0]).toContain("total_changes()"); | |
| 165 | expect(rows.some((r) => r.kind === "forge" && r.name === "kid")).toBe(true); | |
| 166 | // Forges are read fresh: a stranger sees none, and the kept rows are not changed by the admin's. | |
| 167 | const stranger = await explore(app, undefined, query); | |
| 168 | expect(stranger.some((r) => r.kind === "forge")).toBe(false); | |
| 169 | expect(Object.isFrozen(stranger[0])).toBe(true); | |
| 170 | // With every kind asked, the same kept rows serve the admin and the forge rows sort among them. | |
| 171 | const all = await explore(app, admin, query); | |
| 172 | expect(all.length).toBeLessThanOrEqual(60); | |
| 173 | }); | |
| 174 | ||
| 175 | test("the same id with another admin standing is ranked on its own, not served the other's", async () => { | |
| 176 | const fresh = parseExploreQuery("sort:forks"); | |
| 177 | const plain = { ...owner, isAdmin: false } as User; | |
| 178 | const admin = { ...owner, isAdmin: true } as User; | |
| 179 | await explore(app, plain, fresh); | |
| 180 | const sql = countStatements(app.db); | |
| 181 | await explore(app, admin, fresh); | |
| 182 | const n = sql.count; | |
| 183 | sql.stop(); | |
| 184 | expect(n).toBeGreaterThan(3); | |
| 185 | }); |
new above
offline