Public: anyone can read this; the crew can change it.

The cold front page build is slow: batch the per-repo canRead queries and the runtime status scan #6139

Parked opened this
Closed without merging

scout/forge:scout/6136-the-cold-front-page-build-is-slow-batch → main 4 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
5 files changed +117 −12
src/actions/front.ts +7 −3
@@ -42,7 +42,7 @@ import {
4242 type Look,
4343 } from "../core/looks.ts";
4444 import { keepLook } from "../looks/service.ts";
45 import { canRead, getRepo, getReposByIds, type Repo } from "../core/repos.ts";
45 import { getRepo, getReposByIds, strangerReadable, type Repo } from "../core/repos.ts";
4646 import { listActiveRuns } from "../core/runs.ts";
4747 import { getUsersByIds, isActive, type User } from "../core/users.ts";
4848 import { timePhase } from "../core/route_timing.ts";
@@ -205,9 +205,13 @@ function buildFrontPage(app: App, now: number): FrontPage {
205205 const owners = getUsersByIds(db, [
206206 ...new Set([...pinnedRepos.values(), ...repos.values()].map((r) => r.ownerId)),
207207 ]);
208 const strangers = strangerReadable(db, [
209 ...pinnedRepos.values(),
210 ...repos.values(),
211 ]);
208212 const readable = (repo: Repo | undefined): repo is Repo =>
209213 repo !== undefined &&
210 canRead(db, repo, undefined) &&
214 strangers.has(repo.id) &&
211215 (owners.get(repo.ownerId) === undefined || isActive(owners.get(repo.ownerId)!));
212216 const pinned = picks.pinned
213217 .map((id) => pinnedRepos.get(id))
@@ -238,7 +242,7 @@ function buildFrontPage(app: App, now: number): FrontPage {
238242 const production =
239243 record === undefined || runtime === undefined
240244 ? undefined
241 : runtime.status(record).find((env) => env.env === "production");
245 : runtime.envStatus(record, "production");
242246 // An app open only to signed-in readers shows a stranger a sign-in
243247 // page: there is no picture of it worth showing them.
244248 const thumb =
src/core/repos.ts +30
@@ -971,6 +971,36 @@ export function ownerInStanding(db: Database, repo: Repo): boolean {
971971 );
972972 }
973973
974 /**
975 * The repositories among `repos` that a stranger (nobody signed in) may
976 * read: `canRead(db, repo, undefined)` for each, with the owner-standing
977 * check asked once for all of them. Only a non-public repository of an
978 * owner in good standing needs the full check.
979 */
980 export function strangerReadable(
981 db: Database,
982 repos: readonly Repo[],
983 ): Set<string> {
984 const CHUNK = 400;
985 const standing = new Set<string>();
986 for (let i = 0; i < repos.length; i += CHUNK) {
987 const batch = repos.slice(i, i + CHUNK).map((r) => r.id);
988 const rows = db
989 .query<{ id: string }, string[]>(
990 `SELECT repo.id AS id FROM repo WHERE repo.id IN (${batch.map((_, n) => `?${n + 1}`).join(",")}) AND ${OWNER_STANDING}`,
991 )
992 .all(...batch);
993 for (const row of rows) standing.add(row.id);
994 }
995 const out = new Set<string>();
996 for (const repo of repos) {
997 if (!standing.has(repo.id)) continue;
998 if (repo.visibility === "public" || canRead(db, repo, undefined))
999 out.add(repo.id);
1000 }
1001 return out;
1002 }
1003
9741004 export function canRead(
9751005 db: Database,
9761006 repo: Repo,
test/front-speed.test.ts +7 −1
@@ -51,7 +51,13 @@ beforeAll(async () => {
5151 const status = spyOn(AppsRuntime.prototype, "status").mockImplementation(
5252 (): EnvStatus[] => [{ env: "production", state: "running", commitOid: HEAD } as EnvStatus],
5353 );
54 restore = () => status.mockRestore();
54 const envStatus = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation(
55 (): EnvStatus => ({ env: "production", state: "running", commitOid: HEAD }) as EnvStatus,
56 );
57 restore = () => {
58 status.mockRestore();
59 envStatus.mockRestore();
60 };
5561 for (let n = 0; n < APPS; n++) {
5662 const name = `speed${n}`;
5763 const repo = await createRepo(app, owner, { name, visibility: "public", description: name });
test/helpers/studio.ts +12 −8
@@ -33,18 +33,22 @@ export async function serving(
3333 publicPort: servers.port,
3434 forgeUrl: servers.url,
3535 });
36 const production = (record: AppRecord): EnvStatus =>
37 ({
38 env: "production",
39 state: stateOf(record),
40 // Serving the commit its newest picture is of.
41 commitOid: looksForEnv(app.db, record.id, "production")[0]?.commitOid ?? "",
42 }) as EnvStatus;
3643 const status = spyOn(AppsRuntime.prototype, "status").mockImplementation(
37 (record: AppRecord): EnvStatus[] => [
38 {
39 env: "production",
40 state: stateOf(record),
41 // Serving the commit its newest picture is of.
42 commitOid: looksForEnv(app.db, record.id, "production")[0]?.commitOid ?? "",
43 } as EnvStatus,
44 ],
44 (record: AppRecord): EnvStatus[] => [production(record)],
45 );
46 const envStatus = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation(
47 (record: AppRecord): EnvStatus | undefined => production(record),
4548 );
4649 return async () => {
4750 status.mockRestore();
51 envStatus.mockRestore();
4852 await runtime.stop();
4953 };
5054 }
test/stranger-readable.test.ts added +61
@@ -0,0 +1,61 @@
1 /** `strangerReadable` is `canRead` with nobody, asked once for many (#6136). */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { suspendAccount } from "../src/actions/accounts.ts";
8 import { createRepo } from "../src/actions/repos.ts";
9 import { canRead, strangerReadable, type Repo } from "../src/core/repos.ts";
10 import { createUser, type User } from "../src/core/users.ts";
11 import { quietLogs } from "./helpers/globals.ts";
12 import { promote } from "./helpers/members.ts";
13
14 quietLogs();
15
16 let dir = "";
17 let app: App;
18
19 beforeAll(async () => {
20 dir = await mkdtemp(join(tmpdir(), "forge-stranger-"));
21 app = createApp({ dataDir: dir, port: 0 });
22 });
23 afterAll(async () => {
24 app.close();
25 await rm(dir, { recursive: true, force: true });
26 });
27
28 test("agrees with canRead for a stranger: public, private, and a suspended owner's", async () => {
29 const mk = async (name: string): Promise<User> => {
30 const u = await createUser(app.db, {
31 origin: "seed",
32 trusted: true,
33 username: name,
34 email: `${name}@forge.local`,
35 password: "a-good-passphrase",
36 adminIfFirst: true,
37 });
38 if (!u.ok) throw new Error(String(u.error));
39 promote(app.db, u.value);
40 return u.value;
41 };
42 const admin = await mk("root-r");
43 const a = await mk("alpha-r");
44 const b = await mk("beta-r");
45 const repos: Repo[] = [];
46 for (const [owner, name, visibility] of [
47 [a, "open", "public"],
48 [a, "shut", "private"],
49 [b, "open", "public"],
50 ] as const) {
51 const made = await createRepo(app, owner, { name, visibility });
52 if (!made.ok) throw new Error(String(made.error));
53 repos.push(made.value);
54 }
55 expect((await suspendAccount(app, admin, "beta-r")).ok).toBe(true);
56 const got = strangerReadable(app.db, repos);
57 expect(got.size).toBe(1);
58 for (const r of repos)
59 expect(got.has(r.id)).toBe(canRead(app.db, r, undefined));
60 expect(strangerReadable(app.db, [])).toEqual(new Set());
61 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs