Public: anyone can read this; the crew can change it.
The cold front page build is slow: batch the per-repo canRead queries and the runtime status scan #6139
scout/forge:scout/6136-the-cold-front-page-build-is-slow-batch → main 4 commits fork
5 files
changed +117
−12
src/actions/front.ts +7 −3
| @@ -42,7 +42,7 @@ import { | ||
| 42 | 42 | type Look, |
| 43 | 43 | } from "../core/looks.ts"; |
| 44 | 44 | import { keepLook } from "../looks/service.ts"; |
| 45 | import { canRead, getRepo, getReposByIds, type Repo } from "../core/repos.ts"; | |
| 45 | import { getRepo, getReposByIds, strangerReadable, type Repo } from "../core/repos.ts"; | |
| 46 | 46 | import { listActiveRuns } from "../core/runs.ts"; |
| 47 | 47 | import { getUsersByIds, isActive, type User } from "../core/users.ts"; |
| 48 | 48 | import { timePhase } from "../core/route_timing.ts"; |
| @@ -205,9 +205,13 @@ function buildFrontPage(app: App, now: number): FrontPage { | ||
| 205 | 205 | const owners = getUsersByIds(db, [ |
| 206 | 206 | ...new Set([...pinnedRepos.values(), ...repos.values()].map((r) => r.ownerId)), |
| 207 | 207 | ]); |
| 208 | const strangers = strangerReadable(db, [ | |
| 209 | ...pinnedRepos.values(), | |
| 210 | ...repos.values(), | |
| 211 | ]); | |
| 208 | 212 | const readable = (repo: Repo | undefined): repo is Repo => |
| 209 | 213 | repo !== undefined && |
| 210 | canRead(db, repo, undefined) && | |
| 214 | strangers.has(repo.id) && | |
| 211 | 215 | (owners.get(repo.ownerId) === undefined || isActive(owners.get(repo.ownerId)!)); |
| 212 | 216 | const pinned = picks.pinned |
| 213 | 217 | .map((id) => pinnedRepos.get(id)) |
| @@ -238,7 +242,7 @@ function buildFrontPage(app: App, now: number): FrontPage { | ||
| 238 | 242 | const production = |
| 239 | 243 | record === undefined || runtime === undefined |
| 240 | 244 | ? undefined |
| 241 | : runtime.status(record).find((env) => env.env === "production"); | |
| 245 | : runtime.envStatus(record, "production"); | |
| 242 | 246 | // An app open only to signed-in readers shows a stranger a sign-in |
| 243 | 247 | // page: there is no picture of it worth showing them. |
| 244 | 248 | const thumb = |
src/core/repos.ts +30
| @@ -971,6 +971,36 @@ export function ownerInStanding(db: Database, repo: Repo): boolean { | ||
| 971 | 971 | ); |
| 972 | 972 | } |
| 973 | 973 | |
| 974 | /** | |
| 975 | * The repositories among `repos` that a stranger (nobody signed in) may | |
| 976 | * read: `canRead(db, repo, undefined)` for each, with the owner-standing | |
| 977 | * check asked once for all of them. Only a non-public repository of an | |
| 978 | * owner in good standing needs the full check. | |
| 979 | */ | |
| 980 | export function strangerReadable( | |
| 981 | db: Database, | |
| 982 | repos: readonly Repo[], | |
| 983 | ): Set<string> { | |
| 984 | const CHUNK = 400; | |
| 985 | const standing = new Set<string>(); | |
| 986 | for (let i = 0; i < repos.length; i += CHUNK) { | |
| 987 | const batch = repos.slice(i, i + CHUNK).map((r) => r.id); | |
| 988 | const rows = db | |
| 989 | .query<{ id: string }, string[]>( | |
| 990 | `SELECT repo.id AS id FROM repo WHERE repo.id IN (${batch.map((_, n) => `?${n + 1}`).join(",")}) AND ${OWNER_STANDING}`, | |
| 991 | ) | |
| 992 | .all(...batch); | |
| 993 | for (const row of rows) standing.add(row.id); | |
| 994 | } | |
| 995 | const out = new Set<string>(); | |
| 996 | for (const repo of repos) { | |
| 997 | if (!standing.has(repo.id)) continue; | |
| 998 | if (repo.visibility === "public" || canRead(db, repo, undefined)) | |
| 999 | out.add(repo.id); | |
| 1000 | } | |
| 1001 | return out; | |
| 1002 | } | |
| 1003 | ||
| 974 | 1004 | export function canRead( |
| 975 | 1005 | db: Database, |
| 976 | 1006 | repo: Repo, |
test/front-speed.test.ts +7 −1
| @@ -51,7 +51,13 @@ beforeAll(async () => { | ||
| 51 | 51 | const status = spyOn(AppsRuntime.prototype, "status").mockImplementation( |
| 52 | 52 | (): EnvStatus[] => [{ env: "production", state: "running", commitOid: HEAD } as EnvStatus], |
| 53 | 53 | ); |
| 54 | restore = () => status.mockRestore(); | |
| 54 | const envStatus = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation( | |
| 55 | (): EnvStatus => ({ env: "production", state: "running", commitOid: HEAD }) as EnvStatus, | |
| 56 | ); | |
| 57 | restore = () => { | |
| 58 | status.mockRestore(); | |
| 59 | envStatus.mockRestore(); | |
| 60 | }; | |
| 55 | 61 | for (let n = 0; n < APPS; n++) { |
| 56 | 62 | const name = `speed${n}`; |
| 57 | 63 | const repo = await createRepo(app, owner, { name, visibility: "public", description: name }); |
test/helpers/studio.ts +12 −8
| @@ -33,18 +33,22 @@ export async function serving( | ||
| 33 | 33 | publicPort: servers.port, |
| 34 | 34 | forgeUrl: servers.url, |
| 35 | 35 | }); |
| 36 | const production = (record: AppRecord): EnvStatus => | |
| 37 | ({ | |
| 38 | env: "production", | |
| 39 | state: stateOf(record), | |
| 40 | // Serving the commit its newest picture is of. | |
| 41 | commitOid: looksForEnv(app.db, record.id, "production")[0]?.commitOid ?? "", | |
| 42 | }) as EnvStatus; | |
| 36 | 43 | const status = spyOn(AppsRuntime.prototype, "status").mockImplementation( |
| 37 | (record: AppRecord): EnvStatus[] => [ | |
| 38 | { | |
| 39 | env: "production", | |
| 40 | state: stateOf(record), | |
| 41 | // Serving the commit its newest picture is of. | |
| 42 | commitOid: looksForEnv(app.db, record.id, "production")[0]?.commitOid ?? "", | |
| 43 | } as EnvStatus, | |
| 44 | ], | |
| 44 | (record: AppRecord): EnvStatus[] => [production(record)], | |
| 45 | ); | |
| 46 | const envStatus = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation( | |
| 47 | (record: AppRecord): EnvStatus | undefined => production(record), | |
| 45 | 48 | ); |
| 46 | 49 | return async () => { |
| 47 | 50 | status.mockRestore(); |
| 51 | envStatus.mockRestore(); | |
| 48 | 52 | await runtime.stop(); |
| 49 | 53 | }; |
| 50 | 54 | } |
test/stranger-readable.test.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | /** `strangerReadable` is `canRead` with nobody, asked once for many (#6136). */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { suspendAccount } from "../src/actions/accounts.ts"; | |
| 8 | import { createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { canRead, strangerReadable, type Repo } from "../src/core/repos.ts"; | |
| 10 | import { createUser, type User } from "../src/core/users.ts"; | |
| 11 | import { quietLogs } from "./helpers/globals.ts"; | |
| 12 | import { promote } from "./helpers/members.ts"; | |
| 13 | ||
| 14 | quietLogs(); | |
| 15 | ||
| 16 | let dir = ""; | |
| 17 | let app: App; | |
| 18 | ||
| 19 | beforeAll(async () => { | |
| 20 | dir = await mkdtemp(join(tmpdir(), "forge-stranger-")); | |
| 21 | app = createApp({ dataDir: dir, port: 0 }); | |
| 22 | }); | |
| 23 | afterAll(async () => { | |
| 24 | app.close(); | |
| 25 | await rm(dir, { recursive: true, force: true }); | |
| 26 | }); | |
| 27 | ||
| 28 | test("agrees with canRead for a stranger: public, private, and a suspended owner's", async () => { | |
| 29 | const mk = async (name: string): Promise<User> => { | |
| 30 | const u = await createUser(app.db, { | |
| 31 | origin: "seed", | |
| 32 | trusted: true, | |
| 33 | username: name, | |
| 34 | email: `${name}@forge.local`, | |
| 35 | password: "a-good-passphrase", | |
| 36 | adminIfFirst: true, | |
| 37 | }); | |
| 38 | if (!u.ok) throw new Error(String(u.error)); | |
| 39 | promote(app.db, u.value); | |
| 40 | return u.value; | |
| 41 | }; | |
| 42 | const admin = await mk("root-r"); | |
| 43 | const a = await mk("alpha-r"); | |
| 44 | const b = await mk("beta-r"); | |
| 45 | const repos: Repo[] = []; | |
| 46 | for (const [owner, name, visibility] of [ | |
| 47 | [a, "open", "public"], | |
| 48 | [a, "shut", "private"], | |
| 49 | [b, "open", "public"], | |
| 50 | ] as const) { | |
| 51 | const made = await createRepo(app, owner, { name, visibility }); | |
| 52 | if (!made.ok) throw new Error(String(made.error)); | |
| 53 | repos.push(made.value); | |
| 54 | } | |
| 55 | expect((await suspendAccount(app, admin, "beta-r")).ok).toBe(true); | |
| 56 | const got = strangerReadable(app.db, repos); | |
| 57 | expect(got.size).toBe(1); | |
| 58 | for (const r of repos) | |
| 59 | expect(got.has(r.id)).toBe(canRead(app.db, r, undefined)); | |
| 60 | expect(strangerReadable(app.db, [])).toEqual(new Set()); | |
| 61 | }); |
new above
offline