Public: anyone can read this; the crew can change it.

/explore's rank phase stays under 15 ms with 500 public repositories #4011

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/3394-explore-s-rank-phase-stays-under-15-ms-w → main 4 commits fork

  • Merged. scout/3394-explore-s-rank-phase-stays-under-15-ms-w was merged into main as 71a83fc.

Review

Sign in to review this change.
2 files changed +265 −21
src/actions/explore.ts +80 −21
@@ -15,6 +15,7 @@
1515 * - Forges are the children this forge spawned, for a site admin.
1616 */
1717
18 import type { Database } from "bun:sqlite";
1819 import type { App } from "../app.ts";
1920 import { appsForRepos } from "../core/apps.ts";
2021 import { markOf, type Mark } from "../core/marks.ts";
@@ -26,6 +27,8 @@ import {
2627 } from "../core/front.ts";
2728 import { canEnterApp, listReposVisibleTo, type Repo } from "../core/repos.ts";
2829 import { getUsersByIds, isActive, type User } from "../core/users.ts";
30 import { writeStamp } from "../core/write_clock.ts";
31 import { Stamped } from "../kernel/stamped.ts";
2932 import { listChildren } from "./spawn.ts";
3033
3134 export type ExploreKind = "app" | "person" | "forge";
@@ -136,12 +139,57 @@ export interface ExploreRow {
136139 readonly copyable?: boolean;
137140 }
138141
142 /**
143 * The ranking as last computed, per viewer (id and admin standing) and line
144 * (#3394). Valid while the forge's write clock has not moved — a new star,
145 * push or merge moves it, so it shows on the next request — and for 5 seconds
146 * at most, because a score ages with the clock. Admins' answers are kept too;
147 * the spawned forges, which no write clock sees, are read fresh on every call
148 * and added to a copy after the kept read. A call at a given `now` is never kept.
149 */
150 const RANK_MAX_AGE_MS = 5_000;
151 const ranked = new WeakMap<Database, Stamped<ExploreRow[]>>();
152
139153 export async function explore(
140154 app: App,
141155 viewer: User | undefined,
142156 query: ExploreQuery,
143 now = Date.now(),
157 now?: number,
144158 ): Promise<ExploreRow[]> {
159 const forgeRows = await forges(app, viewer, query);
160 const stamp = now === undefined ? writeStamp(app.db) : undefined;
161 if (stamp === undefined) return place(rank(app, viewer, query, now ?? Date.now()), forgeRows, query);
162 let cache = ranked.get(app.db);
163 if (cache === undefined) {
164 cache = new Stamped<ExploreRow[]>({ maxEntries: 64, maxAgeMs: RANK_MAX_AGE_MS });
165 ranked.set(app.db, cache);
166 }
167 const kept = cache.get(`${viewer?.id ?? ""}\n${viewer?.isAdmin === true ? "admin" : ""}\n${exploreLine(query)}`, stamp, () =>
168 rank(app, viewer, query, Date.now()),
169 );
170 return place(kept, forgeRows, query);
171 }
172
173 /** The forges this forge spawned, for a site admin asking for them. */
174 async function forges(app: App, viewer: User | undefined, query: ExploreQuery): Promise<ExploreRow[]> {
175 if (viewer?.isAdmin !== true || (query.kind !== "all" && query.kind !== "forge")) return [];
176 return (await listChildren(app)).map((child) => ({
177 kind: "forge",
178 name: child.name,
179 url: child.url,
180 description: child.done ? "spawned from this forge" : "being spawned",
181 numbers: { forks: 0, ships: 0, awakeHours: 0, apps: 0 },
182 shippedAt: null,
183 score: 0,
184 }));
185 }
186
187 function rank(
188 app: App,
189 viewer: User | undefined,
190 query: ExploreQuery,
191 now: number,
192 ): ExploreRow[] {
145193 const db = app.db;
146194 const repos = listReposVisibleTo(db, viewer, {
147195 limit: READ,
@@ -243,26 +291,24 @@ export async function explore(
243291 });
244292 });
245293
246 const forgeRows: ExploreRow[] =
247 viewer?.isAdmin === true && (query.kind === "all" || query.kind === "forge")
248 ? (await listChildren(app)).map((child) => ({
249 kind: "forge",
250 name: child.name,
251 url: child.url,
252 description: child.done ? "spawned from this forge" : "being spawned",
253 numbers: { forks: 0, ships: 0, awakeHours: 0, apps: 0 },
254 shippedAt: null,
255 score: 0,
256 }))
257 : [];
258
259294 const text = query.text.toLowerCase();
260 const rows = [...appRows, ...people.values(), ...forgeRows].filter(
261 (row) =>
262 (query.kind === "all" || row.kind === query.kind) &&
263 (text === "" ||
264 `${row.name} ${row.description}`.toLowerCase().includes(text)),
295 // Kept rows are shared by every request that reads them: none may change.
296 const freeze = (row: ExploreRow): ExploreRow => {
297 Object.freeze(row.numbers);
298 return Object.freeze(row);
299 };
300 return order(
301 [...appRows, ...people.values()].map(freeze).filter(
302 (row) =>
303 (query.kind === "all" || row.kind === query.kind) &&
304 (text === "" || `${row.name} ${row.description}`.toLowerCase().includes(text)),
305 ),
306 query,
265307 );
308 }
309
310 /** Rows sorted as the query says, in place. */
311 function order(rows: ExploreRow[], query: ExploreQuery): ExploreRow[] {
266312 const by: Record<ExploreSort, (r: ExploreRow) => number | string> = {
267313 score: (r) => r.score,
268314 forks: (r) => r.numbers.forks,
@@ -272,11 +318,24 @@ export async function explore(
272318 name: (r) => r.name.toLowerCase(),
273319 };
274320 const key = by[query.sort];
275 rows.sort((a, b) => {
321 return rows.sort((a, b) => {
276322 const x = key(a);
277323 const y = key(b);
278324 if (query.sort === "name") return String(x).localeCompare(String(y));
279325 return (y as number) - (x as number) || a.name.localeCompare(b.name);
280326 });
281 return rows.slice(0, ROWS);
327 }
328
329 /**
330 * The answer: the ranked rows (kept, so never changed here) with the forge
331 * rows among them, cut to a page. Only the forge rows lack a write clock,
332 * so they are added after the kept read, to a copy.
333 */
334 function place(ranked: readonly ExploreRow[], forgeRows: ExploreRow[], query: ExploreQuery): ExploreRow[] {
335 const text = query.text.toLowerCase();
336 const own = forgeRows.filter(
337 (row) => text === "" || `${row.name} ${row.description}`.toLowerCase().includes(text),
338 );
339 if (own.length === 0) return ranked.slice(0, ROWS);
340 return order([...ranked, ...own], query).slice(0, ROWS);
282341 }
test/explore-rank.test.ts added +185
@@ -0,0 +1,185 @@
1 /** /explore's rank phase stays under 15 ms with 500 public repositories (#3394). */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdir, mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { explore, parseExploreQuery } from "../src/actions/explore.ts";
8 import { createRepo } from "../src/actions/repos.ts";
9 import { createApp as insertApp } from "../src/core/apps.ts";
10 import { createIssue } from "../src/core/issues.ts";
11 import { createUser, type User } from "../src/core/users.ts";
12 import { countStatements } from "../bench/run.ts";
13 import { quietLogs } from "./helpers/globals.ts";
14 import { proveAll } from "./helpers/verified.ts";
15 import { promote } from "./helpers/members.ts";
16
17 quietLogs();
18
19 let dataDir: string;
20 let app: App;
21 let owner: User;
22 const DAY = 86_400_000;
23
24 beforeAll(async () => {
25 dataDir = await mkdtemp(join(tmpdir(), "forge-explore-rank-"));
26 app = createApp({ dataDir, port: 0 });
27 const made = await createUser(app.db, {
28 origin: "seed",
29 username: "olive",
30 email: "[email protected]",
31 password: "a-good-passphrase",
32 adminIfFirst: false,
33 });
34 if (!made.ok) throw new Error(made.error);
35 proveAll(app);
36 promote(app.db, made.value);
37 owner = made.value;
38 const now = Date.now();
39 for (let i = 0; i < 500; i++) {
40 const repo = await createRepo(app, owner, {
41 name: `app-${i}`,
42 visibility: i === 7 ? "private" : "public",
43 });
44 if (!repo.ok) throw new Error(String(repo.error));
45 const rec = insertApp(app.db, {
46 repoId: repo.value.id,
47 slug: `olive-app-${i}`,
48 access: i === 7 ? "private" : "public",
49 });
50 if (!rec.ok) throw new Error(rec.error);
51 // A week of activity: a merged pull every few repositories, hours awake on others.
52 if (i % 3 === 0) {
53 const issue = createIssue(app.db, {
54 repoId: repo.value.id,
55 authorId: owner.id,
56 title: "change",
57 kind: "pull",
58 });
59 if (!issue.ok) throw new Error(issue.error);
60 app.db
61 .query(
62 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
63 VALUES (?1, 'main', 'b', '', '', ?2)`,
64 )
65 .run(issue.value.id, new Date(now - (i % 7) * DAY - 3_600_000).toISOString());
66 }
67 if (i % 2 === 0)
68 app.db
69 .query(
70 `INSERT INTO meter (repo_id, env, placement, started_at, beat_at, ended_at)
71 VALUES (?1, 'production', 'owned', ?2, ?3, ?3)`,
72 )
73 .run(
74 repo.value.id,
75 new Date(now - (i % 6) * DAY - 5 * 3_600_000).toISOString(),
76 new Date(now - (i % 6) * DAY).toISOString(),
77 );
78 }
79 }, 300_000);
80 afterAll(async () => {
81 app.close();
82 await rm(dataDir, { recursive: true, force: true });
83 });
84
85 const query = parseExploreQuery("");
86
87 test("a warm rank of 500 public repositories is a fraction of a cold one and under 15 ms", async () => {
88 const cold: number[] = [];
89 const warm: number[] = [];
90 await explore(app, undefined, query);
91 for (let i = 0; i < 5; i++) {
92 let t = performance.now();
93 await explore(app, undefined, query, Date.now());
94 cold.push(performance.now() - t);
95 t = performance.now();
96 await explore(app, undefined, query);
97 warm.push(performance.now() - t);
98 }
99 expect(Math.min(...warm)).toBeLessThan(15);
100 expect(Math.min(...warm) * 5).toBeLessThan(Math.min(...cold));
101 });
102
103 test("a cold rank runs many statements, a warm one only the write clock", async () => {
104 const cold = countStatements(app.db);
105 await explore(app, undefined, query, Date.now());
106 const coldN = cold.count;
107 cold.stop();
108 await explore(app, undefined, query);
109 const warm = countStatements(app.db);
110 await explore(app, undefined, query);
111 const warmN = warm.count;
112 warm.stop();
113 expect(coldN).toBeGreaterThan(3);
114 expect(warmN).toBeLessThanOrEqual(2);
115 });
116
117 test("a warm request reads nothing but the write clock", async () => {
118 await explore(app, undefined, query);
119 const sql = countStatements(app.db);
120 await explore(app, undefined, query);
121 const n = sql.count;
122 const only = sql.top().map((t) => t.sql);
123 sql.stop();
124 expect(only.length).toBe(1);
125 expect(only[0]).toContain("total_changes()");
126 expect(n).toBeLessThanOrEqual(2);
127 });
128
129 test("the cached order is the computed order, and a new write shows on the next request", async () => {
130 const first = await explore(app, undefined, query);
131 const again = await explore(app, undefined, query);
132 expect(again).toEqual(first);
133 // Asked at a given time it is ranked afresh: the same order.
134 const fresh = await explore(app, undefined, query, Date.now());
135 expect(fresh.map((r) => r.name)).toEqual(first.map((r) => r.name));
136 expect(first.length).toBe(60);
137 expect(first.some((r) => r.name === "olive/app-7")).toBe(false);
138 // A fork by someone else moves the repository up at the next request.
139 const target = first.filter((r) => r.kind === "app").at(-1)!;
140 const parent = app.db
141 .query<{ id: string }, [string]>(
142 `SELECT repo.id FROM repo JOIN user ON user.id = repo.owner_id WHERE user.username || '/' || repo.name = ?1`,
143 )
144 .get(target.name)!;
145 app.db.query(`UPDATE repo SET fork_parent_id = ?1 WHERE name = 'app-1'`).run(parent.id);
146 const after = await explore(app, undefined, query);
147 expect(after.find((r) => r.name === target.name)!.numbers.forks).toBe(1);
148 });
149
150 test("an admin's warm request reads the write clock and the spawned forges, and still lists them", async () => {
151 await mkdir(join(dataDir, "spawn"), { recursive: true });
152 await Bun.write(
153 join(dataDir, "spawn", "kid.json"),
154 JSON.stringify({ name: "kid", bodyId: "b", url: "http://kid", planHash: "h", done: true }),
155 );
156 const forgeQuery = parseExploreQuery("kind:forge");
157 const admin = { ...owner, isAdmin: true } as User;
158 await explore(app, admin, forgeQuery);
159 const sql = countStatements(app.db);
160 const rows = await explore(app, admin, forgeQuery);
161 const only = sql.top().map((t) => t.sql);
162 sql.stop();
163 expect(only.length).toBe(1);
164 expect(only[0]).toContain("total_changes()");
165 expect(rows.some((r) => r.kind === "forge" && r.name === "kid")).toBe(true);
166 // Forges are read fresh: a stranger sees none, and the kept rows are not changed by the admin's.
167 const stranger = await explore(app, undefined, query);
168 expect(stranger.some((r) => r.kind === "forge")).toBe(false);
169 expect(Object.isFrozen(stranger[0])).toBe(true);
170 // With every kind asked, the same kept rows serve the admin and the forge rows sort among them.
171 const all = await explore(app, admin, query);
172 expect(all.length).toBeLessThanOrEqual(60);
173 });
174
175 test("the same id with another admin standing is ranked on its own, not served the other's", async () => {
176 const fresh = parseExploreQuery("sort:forks");
177 const plain = { ...owner, isAdmin: false } as User;
178 const admin = { ...owner, isAdmin: true } as User;
179 await explore(app, plain, fresh);
180 const sql = countStatements(app.db);
181 await explore(app, admin, fresh);
182 const n = sql.count;
183 sql.stop();
184 expect(n).toBeGreaterThan(3);
185 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs