Public: anyone can read this; the crew can change it.

The signed-in feed's first read walks an index, not a scan, and answers inside a stated budget with Server-Timing #13498

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13482-the-signed-in-feed-s-first-read-walks-an → main 2 commits fork

  • Merged. torque/13482-the-signed-in-feed-s-first-read-walks-an was merged into main as d3fe125.

Review

Sign in to review this change.
3 files changed +136 −0
src/core/migration-pins/20261115140000-event-actor-id-index.pin added +1
@@ -0,0 +1,1 @@
1 479efef0ff3b2ca6caaff326558f9cf007926d481cfc4ee147bf20681ad3b8dd
src/core/migrations/20261115140000-event-actor-id-index.ts added +7
@@ -0,0 +1,7 @@
1 // The feed ranks with what the viewer did lately (`seedOwnActs`, feed/affinity.ts): their newest
2 // events by id. No index led with the actor, so SQLite read every event and sorted the viewer's
3 // (#13482). `(actor_id, id)` answers by walking the actor's events newest first to the limit.
4 // Additive.
5 export default `
6 CREATE INDEX IF NOT EXISTS event_actor_id ON event(actor_id, id);
7 `;
test/feed-hot-read.test.ts added +128
@@ -0,0 +1,128 @@
1 /**
2 * The signed-in feed's first read walks an index and reads a bounded number of rows (#13482).
3 * Judged by the query plan and by rows read, never the clock.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtempSync, rmSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createRepo } from "../src/actions/repos.ts";
11 import { createUser, type User } from "../src/core/users.ts";
12 import { promote } from "./helpers/members.ts";
13 import { createSession } from "../src/core/sessions.ts";
14 import { serve } from "../src/http/server.ts";
15 import { readFeedPage } from "../src/feed/feed.ts";
16
17 /**
18 * Rows the first page of a 5,000-item feed may read in all, whatever the feed's depth. The event
19 * window (300) and the viewer's own acts (200) are the most it may read; measured at 310.
20 */
21 const ROW_BUDGET = 600;
22 const SEEDED = 5000;
23
24 let root = "";
25 let app: App;
26 let viewer: User;
27 let repoId = "";
28 let n = 0;
29
30 beforeAll(async () => {
31 root = mkdtempSync(join(tmpdir(), "forge-feed-hot-"));
32 app = createApp({ dataDir: root, port: 0 });
33 const root0 = await createUser(app.db, { origin: "seed", trusted: true, username: "root", allowReserved: true, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
34 if (!root0.ok) throw new Error(root0.error);
35 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", allowReserved: true, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: false });
36 if (!made.ok) throw new Error(made.error);
37 viewer = made.value;
38 promote(app.db, viewer);
39 const repo = await createRepo(app, viewer, { name: "open", visibility: "public" });
40 if (!repo.ok) throw new Error(JSON.stringify(repo));
41 repoId = repo.value.id;
42 app.db.transaction(() => {
43 for (let i = 0; i < SEEDED; i++)
44 app.bus.append({ kind: "run.finished", repoId: repo.value.id, actorId: viewer.id, payload: { kind: "agent", state: "ok", summary: `work ${i}`, runId: `r${i}` } });
45 })();
46 });
47 afterAll(() => {
48 app.close();
49 rmSync(root, { recursive: true, force: true });
50 });
51
52 type Probe = { sql: string; rows: number };
53
54 /** Every SELECT `work` runs, with the rows it returned. */
55 function probe(work: () => void): Probe[] {
56 const db = app.db;
57 const original = db.query.bind(db);
58 const seen: Probe[] = [];
59 (db as unknown as { query: unknown }).query = (sql: string) => {
60 const stmt = original(sql) as unknown as Record<string, (...a: unknown[]) => unknown>;
61 if (!/^\s*SELECT/i.test(sql)) return stmt;
62 return new Proxy(stmt, {
63 get(target, prop) {
64 const v = target[prop as string]!;
65 if (prop !== "all" && prop !== "get") return typeof v === "function" ? v.bind(target) : v;
66 return (...args: unknown[]) => {
67 const out = v.apply(target, args);
68 seen.push({ sql, rows: Array.isArray(out) ? out.length : out == null ? 0 : 1 });
69 return out;
70 };
71 },
72 });
73 };
74 try {
75 work();
76 } finally {
77 (db as unknown as { query: unknown }).query = original;
78 }
79 return seen;
80 }
81
82 /**
83 * A write first, so the candidate caches (stamped by the write clock) read afresh: this is the cold
84 * first read.
85 */
86 const firstPage = () => {
87 app.bus.append({ kind: "run.finished", repoId, actorId: viewer.id, payload: { kind: "agent", state: "ok", summary: `fresh ${n++}`, runId: `f${n}` } });
88 return readFeedPage(app.db, viewer, { limit: 20, withoutAsks: true, now: Date.now() });
89 };
90
91 test("the feed's reads of the event table walk an index: no scan, no temp b-tree", () => {
92 const reads = probe(() => void firstPage());
93 const bad: string[] = [];
94 // The feed's item query is its read of the event table (and the viewer's own acts in it); the
95 // small per-viewer tables (mutes, shares, watches) are bounded by the viewer, not by the forge.
96 const ofEvents = reads.filter(({ sql }) => /\b(FROM|JOIN) event\b/.test(sql));
97 expect(ofEvents.length).toBeGreaterThanOrEqual(2);
98 for (const { sql } of ofEvents) {
99 const n = Math.max(0, ...[...sql.matchAll(/\?(\d+)/g)].map((m) => Number(m[1])));
100 const args = Array.from({ length: n }, () => viewer.id);
101 for (const row of app.db.query(`EXPLAIN QUERY PLAN ${sql}`).all(...args)) {
102 const line = (row as { detail: string }).detail;
103 if (/^SCAN (?!.*USING (COVERING )?INDEX)/.test(line) || line.includes("TEMP B-TREE")) bad.push(`${line}\n in: ${sql.replace(/\s+/g, " ").slice(0, 160)}`);
104 }
105 }
106 expect(bad).toEqual([]);
107 });
108
109 test("the first page of a 5,000-item feed reads no more than a fixed number of rows", () => {
110 const reads = probe(() => void firstPage());
111 const total = reads.reduce((n, r) => n + r.rows, 0);
112 expect(total).toBeLessThanOrEqual(ROW_BUDGET);
113 });
114
115 test("the signed-in feed's response names the feed read in Server-Timing, on the page and on the API", async () => {
116 const servers = await serve(app);
117 try {
118 const cookie = `forge_session=${createSession(app.db, viewer.id).token}`;
119 for (const path of ["/feed", "/api/v1/feed"]) {
120 const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
121 expect(res.status).toBe(200);
122 await res.text();
123 expect(res.headers.get("server-timing") ?? "").toMatch(/(^|, )feed;dur=\d/);
124 }
125 } finally {
126 servers.stop();
127 }
128 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs