Public: anyone can read this; the crew can change it.

The signed-out front page's shelf stays under 10 ms on the live forge, not only in its test #6258

Closed opened this
Closed without merging

scout/forge:scout/6220-the-signed-out-front-page-s-shelf-stays → main 8 commits fork 4 threads · 4 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
4 files changed +161 −11
src/actions/front.ts +26 −10
@@ -38,6 +38,7 @@ import { getIssuesByIds } from "../core/issues.ts";
3838 import {
3939 DEFAULT_PAGE,
4040 looksForApps,
41 looksVersion,
4142 lookUrl,
4243 type Look,
4344 } from "../core/looks.ts";
@@ -157,19 +158,34 @@ export function forgetFrontScores(app: App): void {
157158 const PAGE_MS = 3_000;
158159
159160 /**
160 * The built page, kept for `PAGE_MS` while the database is untouched: any
161 * write on this connection, or by another, changes the key, so a repository
162 * made private, a pick, a picture or a suspension shows on the next request.
161 * The built page, kept for `PAGE_MS` while what the shelf reads is
162 * unchanged: the apps, their repositories and owners, the picks, the
163 * settings and the pictures taken. A session, a meter row or an event in
164 * another repository does not move the key, so a busy forge keeps its page;
165 * a repository made private, a pick, a picture or a suspension does.
163166 */
164167 const built = new WeakMap<App, { key: string; at: number; page: FrontPage }>();
165168
166 function dbKey(app: App): string {
167 const row = app.db
168 .query<{ c: number; v: number }, []>(
169 "SELECT total_changes() AS c, (SELECT data_version FROM pragma_data_version) AS v",
170 )
171 .get();
172 return `${row?.c}:${row?.v}`;
169 const SHELF_READS = `
170 SELECT
outdated

This concatenates every app row, including preview_data, on every request. The cost grows with the number of apps. Fingerprint only the fields the shelf renders, or measure it.

171 (SELECT group_concat(id || ',' || repo_id || ',' || slug || ',' || access, ';') FROM app),
172 (SELECT group_concat(json_array(id, owner_id, name, visibility, description, default_branch, fork_parent_id, sealed_for, archived_at), ';')
173 FROM repo WHERE id IN (SELECT repo_id FROM app UNION SELECT repo_id FROM front_pick)),
174 (SELECT group_concat(json_array(id, username, display_name, kind, suspended_at, removed_at), ';')
175 FROM user WHERE id IN (SELECT owner_id FROM repo WHERE id IN (SELECT repo_id FROM app UNION SELECT repo_id FROM front_pick))),
176 (SELECT group_concat(json_array(repo_id, place, position), ';') FROM front_pick),
177 (SELECT group_concat(key || '=' || value, ';') FROM front_setting)`;
outdated

This subselect concatenates every light look ever taken, across all envs and commits. It grows without bound and runs on every request, even inside the PAGE_MS window. Restrict it to env = 'production' AND kind = 'thumb', or use count(*), max(created_at) and max(id). Measure it on the live data.

outdated

Unindexed filter on look plus three aggregates on every request. This scans the table, which grows without bound. Use max(rowid) or drop the term and let PAGE_MS cover it.

178
179 /**
180 * What the shelf reads, as one string. Its length grows with the apps (about
181 * 1 ms for 300); past a few thousand, hash it in SQL or bump a version on
182 * writes. A look written by another process does not move it: `PAGE_MS`
183 * bounds how long that stays stale.
184 *
185 * It is the pictures by how often this process wrote them. */
186 export function dbKey(app: App): string {
187 const row = app.db.query<Record<string, string | null>, []>(SHELF_READS).get();

Broken doc comment: the last sentence is a fragment and the */ is on the same line. Fix the wording and the closing.

188 return JSON.stringify([Object.values(row ?? {}), looksVersion(app.db)]);

O(apps) text built per request. Without a measurement on the live data, the 10 ms claim in the title is unproven. Measure it, or hash in SQL.

173189 }
174190
175191 /** The front page as a stranger is shown it. */
src/core/looks.ts +9
@@ -172,8 +172,16 @@ export interface RecordLookInput {
172172 readonly note?: string | undefined;
173173 }
174174
175 const taken = new WeakMap<Database, number>();
176
177 /** How many times this connection's looks were written or dropped: a number a reader can compare, where scanning the table would cost. */
178 export function looksVersion(db: Database): number {
179 return taken.get(db) ?? 0;
180 }
181
175182 /** Record a picture, or the reason there is none. A retake replaces. */
176183 export function recordLook(db: Database, input: RecordLookInput): Look {
184 taken.set(db, looksVersion(db) + 1);
177185 const page = input.page ?? DEFAULT_PAGE;
178186 const theme = input.theme ?? DEFAULT_THEME;
179187 db.query(
@@ -281,6 +289,7 @@ export function deleteLooks(
281289 [string, string, string]
282290 >(`DELETE FROM look WHERE app_id = ?1 AND env = ?2 AND commit_oid != ?3 AND attached_by IS NULL RETURNING *`)
283291 .all(appId, env, keepCommit);
292 taken.set(db, looksVersion(db) + 1);
284293 return gone.map(toLook);
285294 }
286295
test/front-key.test.ts added +93
@@ -0,0 +1,93 @@
1 /** The front page's key reads what the shelf reads, at a cost that does not grow with the looks taken (#6220). */
2 import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
3 import { mkdirSync, writeFileSync } from "node:fs";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { dbKey } from "../src/actions/front.ts";
9 import { createRepo } from "../src/actions/repos.ts";
10 import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
11 import { createApp as insertApp, type AppRecord } from "../src/core/apps.ts";
12 import { recordLook } from "../src/core/looks.ts";
13 import { encodePng } from "../src/design/image/png.ts";
14 import { createUser, type User } from "../src/core/users.ts";
15 import { quietLogs } from "./helpers/globals.ts";
16 import { promote } from "./helpers/members.ts";
17
18 quietLogs();
19
20 const HEAD = "a".repeat(40);
21 const PNG = Buffer.from(encodePng(1, 1, new Uint8Array([255, 255, 255, 255])));
22 const APPS = Number(process.env.N ?? 300);
23
24 let dataDir = "";
25 let app: App;
26 let runtime: AppsRuntime;
27 let restore: () => void;
28 const repoIds: string[] = [];
29
30 beforeAll(async () => {
31 dataDir = await mkdtemp(join(tmpdir(), "forge-front-key-"));
32 app = createApp({ dataDir, port: 0 });
33 const made = await createUser(app.db, {
34 origin: "seed",
35 trusted: true,
36 username: "owner",
37 allowReserved: true,
38 email: "[email protected]",
39 password: "a-good-passphrase",
40 adminIfFirst: true,
41 });
42 if (!made.ok) throw new Error(made.error);
43 promote(app.db, made.value);
44 const owner: User = made.value;
45 runtime = new AppsRuntime(app, {
46 domain: "front.test",
47 scheme: "http",
48 publicPort: 0,
49 forgeUrl: "http://localhost",
50 });
51 const status = spyOn(AppsRuntime.prototype, "status").mockImplementation(
52 (): EnvStatus[] => [{ env: "production", state: "running", commitOid: HEAD } as EnvStatus],
53 );
54 restore = () => status.mockRestore();
55 for (let n = 0; n < APPS; n++) {
56 const name = `speed${n}`;
57 const repo = await createRepo(app, owner, { name, visibility: "public", description: name });
58 if (!repo.ok) throw new Error(String(repo.error));
59 const record = insertApp(app.db, { repoId: repo.value.id, slug: name, access: "public" });
60 if (!record.ok) throw new Error(record.error);
61 const id = (record.value as AppRecord).id;
62 mkdirSync(join(dataDir, "looks", id), { recursive: true });
63 writeFileSync(join(dataDir, "looks", id, "thumb.png"), PNG);
64 recordLook(app.db, {
65 appId: id,
66 env: "production",
67 commitOid: HEAD,
68 kind: "thumb",
69 path: `looks/${id}/thumb.png`,
70 });
71 repoIds.push(repo.value.id);
72 }
73 }, 60_000);
74
75 afterAll(async () => {
76 restore();
77 await runtime.stop();
78 app.close();
79 await rm(dataDir, { recursive: true, force: true });
80 });
81
82
83 test("the key does not read the look table: five thousand more looks leave it as it was (#6220)", () => {
84 const appIds = app.db.query<{ id: string }, []>("SELECT id FROM app").all().map((r) => r.id);
85 const before = dbKey(app);
86 const put = app.db.query(
87 "INSERT INTO look (id, app_id, env, commit_oid, page, kind, path, theme) VALUES (?1, ?2, 'production', ?3, '/', 'thumb', 'p', 'dark')",
88 );
89 app.db.transaction(() => {
90 for (let n = 0; n < 5000; n++) put.run(`bulk${n}`, appIds[n % appIds.length]!, n.toString(16).padStart(40, "0"));
91 })();
92 expect(dbKey(app)).toBe(before);
93 });
test/front-speed.test.ts +33 −1
@@ -9,7 +9,7 @@ import { forgetFrontScores, frontPage } from "../src/actions/front.ts";
99 import { createRepo } from "../src/actions/repos.ts";
1010 import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
1111 import { createApp as insertApp, type AppRecord } from "../src/core/apps.ts";
12 import { recordLook } from "../src/core/looks.ts";
12 import { deleteLooks, recordLook } from "../src/core/looks.ts";
1313 import { encodePng } from "../src/design/image/png.ts";
1414 import { createUser, type User } from "../src/core/users.ts";
1515 import { quietLogs } from "./helpers/globals.ts";
@@ -103,3 +103,35 @@ test("making one app private builds a new page without it on the next request",
103103 expect(after.total).toBe(APPS - 1);
104104 expect([after.featured, ...after.shelf].some((e) => e?.repo.id === gone.id)).toBe(false);
105105 });
106
107 test("writes the shelf does not read leave the kept page standing (#6220)", () => {
108 const first = frontPage(app);
109 const owner = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!;
110 app.db.query("UPDATE repo SET issue_seq = issue_seq + 1").run();
111 app.db
112 .query("INSERT INTO session (id, user_id, expires_at, created_at) VALUES ('s6220', ?1, 'x', 'x')")
113 .run(owner.id);
114 const second = frontPage(app);
115 expect(second).toBe(first);
116 });
117
118 test("archiving or unpublishing an app changes the shelf on the next request (#6220)", () => {
119 const before = frontPage(app);
120 const victim = before.shelf[0]!.repo;
121 app.db.query("UPDATE repo SET archived_at = '2026-10-26T00:00:00Z' WHERE id = ?1").run(victim.id);
122 expect(frontPage(app)).not.toBe(before);
123 const mid = frontPage(app);
124 app.db.query("UPDATE app SET access = 'private' WHERE repo_id = ?1").run(mid.shelf[0]!.repo.id);
125 expect(frontPage(app)).not.toBe(mid);
126 });
127
128 test("a picture retaken, or a look forgotten, builds a new page on the next request (#6220)", () => {
129 const before = frontPage(app);
130 const shown = before.shelf[0]!.app!.id;
131 recordLook(app.db, { appId: shown, env: "production", commitOid: HEAD, kind: "thumb", path: `looks/${shown}/thumb.png` });
132 const retaken = frontPage(app);
133 expect(retaken).not.toBe(before);
134 expect(frontPage(app)).toBe(retaken);
135 deleteLooks(app.db, shown, "production");
136 expect(frontPage(app)).not.toBe(retaken);
137 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs