Public: anyone can read this; the crew can change it.
The signed-in feed's first read walks an index, not a scan, and answers inside a stated budget with Server-Timing #13498
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13482-the-signed-in-feed-s-first-read-walks-an → main 2 commits fork
3 files
changed +136
−0
src/core/migration-pins/20261115140000-event-actor-id-index.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 479efef0ff3b2ca6caaff326558f9cf007926d481cfc4ee147bf20681ad3b8dd |
src/core/migrations/20261115140000-event-actor-id-index.ts added +7
| @@ -0,0 +1,7 @@ | ||
| 1 | // The feed ranks with what the viewer did lately (`seedOwnActs`, feed/affinity.ts): their newest | |
| 2 | // events by id. No index led with the actor, so SQLite read every event and sorted the viewer's | |
| 3 | // (#13482). `(actor_id, id)` answers by walking the actor's events newest first to the limit. | |
| 4 | // Additive. | |
| 5 | export default ` | |
| 6 | CREATE INDEX IF NOT EXISTS event_actor_id ON event(actor_id, id); | |
| 7 | `; |
test/feed-hot-read.test.ts added +128
| @@ -0,0 +1,128 @@ | ||
| 1 | /** | |
| 2 | * The signed-in feed's first read walks an index and reads a bounded number of rows (#13482). | |
| 3 | * Judged by the query plan and by rows read, never the clock. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createRepo } from "../src/actions/repos.ts"; | |
| 11 | import { createUser, type User } from "../src/core/users.ts"; | |
| 12 | import { promote } from "./helpers/members.ts"; | |
| 13 | import { createSession } from "../src/core/sessions.ts"; | |
| 14 | import { serve } from "../src/http/server.ts"; | |
| 15 | import { readFeedPage } from "../src/feed/feed.ts"; | |
| 16 | ||
| 17 | /** | |
| 18 | * Rows the first page of a 5,000-item feed may read in all, whatever the feed's depth. The event | |
| 19 | * window (300) and the viewer's own acts (200) are the most it may read; measured at 310. | |
| 20 | */ | |
| 21 | const ROW_BUDGET = 600; | |
| 22 | const SEEDED = 5000; | |
| 23 | ||
| 24 | let root = ""; | |
| 25 | let app: App; | |
| 26 | let viewer: User; | |
| 27 | let repoId = ""; | |
| 28 | let n = 0; | |
| 29 | ||
| 30 | beforeAll(async () => { | |
| 31 | root = mkdtempSync(join(tmpdir(), "forge-feed-hot-")); | |
| 32 | app = createApp({ dataDir: root, port: 0 }); | |
| 33 | const root0 = await createUser(app.db, { origin: "seed", trusted: true, username: "root", allowReserved: true, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 34 | if (!root0.ok) throw new Error(root0.error); | |
| 35 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", allowReserved: true, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: false }); | |
| 36 | if (!made.ok) throw new Error(made.error); | |
| 37 | viewer = made.value; | |
| 38 | promote(app.db, viewer); | |
| 39 | const repo = await createRepo(app, viewer, { name: "open", visibility: "public" }); | |
| 40 | if (!repo.ok) throw new Error(JSON.stringify(repo)); | |
| 41 | repoId = repo.value.id; | |
| 42 | app.db.transaction(() => { | |
| 43 | for (let i = 0; i < SEEDED; i++) | |
| 44 | app.bus.append({ kind: "run.finished", repoId: repo.value.id, actorId: viewer.id, payload: { kind: "agent", state: "ok", summary: `work ${i}`, runId: `r${i}` } }); | |
| 45 | })(); | |
| 46 | }); | |
| 47 | afterAll(() => { | |
| 48 | app.close(); | |
| 49 | rmSync(root, { recursive: true, force: true }); | |
| 50 | }); | |
| 51 | ||
| 52 | type Probe = { sql: string; rows: number }; | |
| 53 | ||
| 54 | /** Every SELECT `work` runs, with the rows it returned. */ | |
| 55 | function probe(work: () => void): Probe[] { | |
| 56 | const db = app.db; | |
| 57 | const original = db.query.bind(db); | |
| 58 | const seen: Probe[] = []; | |
| 59 | (db as unknown as { query: unknown }).query = (sql: string) => { | |
| 60 | const stmt = original(sql) as unknown as Record<string, (...a: unknown[]) => unknown>; | |
| 61 | if (!/^\s*SELECT/i.test(sql)) return stmt; | |
| 62 | return new Proxy(stmt, { | |
| 63 | get(target, prop) { | |
| 64 | const v = target[prop as string]!; | |
| 65 | if (prop !== "all" && prop !== "get") return typeof v === "function" ? v.bind(target) : v; | |
| 66 | return (...args: unknown[]) => { | |
| 67 | const out = v.apply(target, args); | |
| 68 | seen.push({ sql, rows: Array.isArray(out) ? out.length : out == null ? 0 : 1 }); | |
| 69 | return out; | |
| 70 | }; | |
| 71 | }, | |
| 72 | }); | |
| 73 | }; | |
| 74 | try { | |
| 75 | work(); | |
| 76 | } finally { | |
| 77 | (db as unknown as { query: unknown }).query = original; | |
| 78 | } | |
| 79 | return seen; | |
| 80 | } | |
| 81 | ||
| 82 | /** | |
| 83 | * A write first, so the candidate caches (stamped by the write clock) read afresh: this is the cold | |
| 84 | * first read. | |
| 85 | */ | |
| 86 | const firstPage = () => { | |
| 87 | app.bus.append({ kind: "run.finished", repoId, actorId: viewer.id, payload: { kind: "agent", state: "ok", summary: `fresh ${n++}`, runId: `f${n}` } }); | |
| 88 | return readFeedPage(app.db, viewer, { limit: 20, withoutAsks: true, now: Date.now() }); | |
| 89 | }; | |
| 90 | ||
| 91 | test("the feed's reads of the event table walk an index: no scan, no temp b-tree", () => { | |
| 92 | const reads = probe(() => void firstPage()); | |
| 93 | const bad: string[] = []; | |
| 94 | // The feed's item query is its read of the event table (and the viewer's own acts in it); the | |
| 95 | // small per-viewer tables (mutes, shares, watches) are bounded by the viewer, not by the forge. | |
| 96 | const ofEvents = reads.filter(({ sql }) => /\b(FROM|JOIN) event\b/.test(sql)); | |
| 97 | expect(ofEvents.length).toBeGreaterThanOrEqual(2); | |
| 98 | for (const { sql } of ofEvents) { | |
| 99 | const n = Math.max(0, ...[...sql.matchAll(/\?(\d+)/g)].map((m) => Number(m[1]))); | |
| 100 | const args = Array.from({ length: n }, () => viewer.id); | |
| 101 | for (const row of app.db.query(`EXPLAIN QUERY PLAN ${sql}`).all(...args)) { | |
| 102 | const line = (row as { detail: string }).detail; | |
| 103 | if (/^SCAN (?!.*USING (COVERING )?INDEX)/.test(line) || line.includes("TEMP B-TREE")) bad.push(`${line}\n in: ${sql.replace(/\s+/g, " ").slice(0, 160)}`); | |
| 104 | } | |
| 105 | } | |
| 106 | expect(bad).toEqual([]); | |
| 107 | }); | |
| 108 | ||
| 109 | test("the first page of a 5,000-item feed reads no more than a fixed number of rows", () => { | |
| 110 | const reads = probe(() => void firstPage()); | |
| 111 | const total = reads.reduce((n, r) => n + r.rows, 0); | |
| 112 | expect(total).toBeLessThanOrEqual(ROW_BUDGET); | |
| 113 | }); | |
| 114 | ||
| 115 | test("the signed-in feed's response names the feed read in Server-Timing, on the page and on the API", async () => { | |
| 116 | const servers = await serve(app); | |
| 117 | try { | |
| 118 | const cookie = `forge_session=${createSession(app.db, viewer.id).token}`; | |
| 119 | for (const path of ["/feed", "/api/v1/feed"]) { | |
| 120 | const res = await fetch(`${servers.url}${path}`, { headers: { cookie } }); | |
| 121 | expect(res.status).toBe(200); | |
| 122 | await res.text(); | |
| 123 | expect(res.headers.get("server-timing") ?? "").toMatch(/(^|, )feed;dur=\d/); | |
| 124 | } | |
| 125 | } finally { | |
| 126 | servers.stop(); | |
| 127 | } | |
| 128 | }); |
new above
offline