Public: anyone can read this; the crew can change it.
A map's ask-to-live medians come from one batched read: `asktolive` under 50 ms for trevato/forge #6186
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6175-a-map-s-ask-to-live-medians-come-from-on → main 3 commits fork 4 threads · 4 unresolved
src/core/pillars.ts +24 −15
| @@ -258,6 +258,9 @@ export function mergedByPillar( | ||
| 258 | 258 | ]; |
| 259 | 259 | } |
| 260 | 260 | |
| 261 | /** How many ids one query binds; far under SQLite's variable limit. */ | |
| 262 | const CHUNK = 400; | |
| 263 | ||
| 261 | 264 | interface MergedPull { |
| 262 | 265 | readonly id: string; |
| 263 | 266 | readonly closes: readonly number[]; |
| @@ -308,7 +311,6 @@ function mergedPulls( | ||
| 308 | 311 | })); |
| 309 | 312 | const numbers = [...new Set(pulls.flatMap((pull) => pull.closes))]; |
| 310 | 313 | const issues = new Map<number, ClosedIssue>(); |
| 311 | const CHUNK = 400; | |
| 312 | 314 | for (let i = 0; i < numbers.length; i += CHUNK) { |
| 313 | 315 | const chunk = numbers.slice(i, i + CHUNK); |
| 314 | 316 | const found = db |
| @@ -364,7 +366,8 @@ export interface AskToLiveCount { | ||
| 364 | 366 | * the first production deploy that went live carrying the pull (the |
| 365 | 367 | * `deploy_pull` record, so no git runs here). A pull not yet live counts |
| 366 | 368 | * for nothing; an issue closed by several pulls counts once, at its first |
| 367 | * time live. Every name has a row, in order; two reads however many pulls. | |
| 369 | * time live. Every name has a row, in order; a fixed number of reads however | |
| 370 | * many pulls (the window's pulls and their issues, then the deploys carrying them). | |
| 368 | 371 | */ |
| 369 | 372 | export function askToLiveByPillar( |
| 370 | 373 | db: Database, |
| @@ -375,19 +378,26 @@ export function askToLiveByPillar( | ||
| 375 | 378 | scope: SealedScope = {}, |
| 376 | 379 | ): AskToLiveCount[] { |
| 377 | 380 | const { pulls, issues } = mergedPulls(db, repoId, since, scope); |
| 381 | // Read from the window's merged pulls up, by `deploy_pull_issue` and the run's key, | |
| 382 | // not from every run the repository ever made. | |
| 378 | 383 | const live = new Map<string, string[]>(); |
| 379 | for (const row of db | |
| 380 | .query<{ id: string; at: string }, [string, string]>( | |
| 381 | `SELECT dp.issue_id AS id, COALESCE(r.serving_at, r.finished_at) AS at | |
| 382 | FROM deploy_pull dp JOIN run r ON r.id = dp.run_id | |
| 383 | WHERE r.repo_id = ?1 AND r.kind = 'deploy' AND r.env = 'production' AND r.state = 'ok' | |
| 384 | AND COALESCE(r.serving_at, r.finished_at) >= ?2 | |
| 385 | ORDER BY COALESCE(r.serving_at, r.finished_at), r.id`, | |
| 386 | ) | |
| 387 | .all(repoId, since)) { | |
| 388 | const list = live.get(row.id) ?? []; | |
| 389 | list.push(row.at); | |
| 390 | live.set(row.id, list); | |
| 384 | const ids = pulls.map((pull) => pull.id); | |
| 385 | for (let i = 0; i < ids.length; i += CHUNK) { | |
| 386 | const chunk = ids.slice(i, i + CHUNK); | |
| 387 | for (const row of db | |
| 388 | .query<{ id: string; at: string }, string[]>( | |
| 389 | `SELECT dp.issue_id AS id, COALESCE(r.serving_at, r.finished_at) AS at | |
| 390 | FROM deploy_pull dp CROSS JOIN run r ON r.id = dp.run_id | |
| 391 | WHERE dp.issue_id IN (${chunk.map((_, n) => `?${n + 3}`).join(",")}) | |
| 392 | AND r.repo_id = ?1 AND r.kind = 'deploy' AND r.env = 'production' AND r.state = 'ok' | |
| 393 | AND COALESCE(r.serving_at, r.finished_at) >= ?2 | |
| 394 | ORDER BY COALESCE(r.serving_at, r.finished_at), r.id`, | |
| 395 | ) | |
| 396 | .all(repoId, since, ...chunk)) { | |
| 397 | const list = live.get(row.id) ?? []; | |
| 398 | list.push(row.at); | |
| 399 | live.set(row.id, list); | |
| 400 | } | |
| 391 | 401 | } |
| 392 | 402 | // One entry per asked issue: its first time live over every pull that closed it. |
| 393 | 403 | const first = new Map<number, { pillar: string; createdAt: string; at: string }>(); |
| @@ -435,7 +445,6 @@ export function reposWithUmbrellas( | ||
| 435 | 445 | repoIds: readonly string[], |
| 436 | 446 | ): Set<string> { |
| 437 | 447 | const out = new Set<string>(); |
| 438 | const CHUNK = 400; | |
| 439 | 448 | for (let i = 0; i < repoIds.length; i += CHUNK) { |
| 440 | 449 | const chunk = repoIds.slice(i, i + CHUNK); |
| 441 | 450 | const rows = db |
test/ask-to-live-batched.test.ts added +153
| @@ -0,0 +1,153 @@ | ||
| 1 | /** `askToLiveFor` reads the window's merged pulls in a fixed number of queries, answers as the per-pull read did, and keeps sealed issues from a viewer who cannot see them (#6186). */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { askToLiveOf, pillarWindowStart, type PillarView } from "../src/core/pillars.ts"; | |
| 8 | import { closesFromBody } from "../src/core/pulls.ts"; | |
| 9 | import { sealedScope } from "../src/core/sealed.ts"; | |
| 10 | import type { Repo } from "../src/core/repos.ts"; | |
| 11 | import { askToLiveFor } from "../src/web/ask_to_live.ts"; | |
| 12 | import { expectBudget } from "./helpers/budget.ts"; | |
| 13 | ||
| 14 | let root = ""; | |
| 15 | let app: App; | |
| 16 | let repo: Repo; | |
| 17 | let view: PillarView; | |
| 18 | const PILLARS = 8; | |
| 19 | const PULLS = 400; | |
| 20 | const T = Date.now(); | |
| 21 | const at = (hoursAgo: number) => new Date(T - hoursAgo * 3_600_000).toISOString(); | |
| 22 | ||
| 23 | beforeAll(() => { | |
| 24 | root = mkdtempSync(join(tmpdir(), "forge-ask-to-live-")); | |
| 25 | app = createApp({ dataDir: root, port: 0 }); | |
| 26 | const db = app.db; | |
| 27 | db.query( | |
| 28 | "INSERT INTO user (id, username, email, password_hash) VALUES ('u1','alice','[email protected]','x')", | |
| 29 | ).run(); | |
| 30 | db.query("INSERT INTO repo (id, owner_id, name) VALUES ('r1','u1','engine')").run(); | |
| 31 | repo = db.query("SELECT * FROM repo WHERE id = 'r1'").get() as never; | |
| 32 | const issue = db.query( | |
| 33 | `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at, sealed) | |
| 34 | VALUES (?1, 'r1', ?2, ?3, ?4, ?5, 'u1', 'closed', ?6, ?7)`, | |
| 35 | ); | |
| 36 | const pull = db.query( | |
| 37 | `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at, merged_oid) | |
| 38 | VALUES (?1, 'main', ?2, 'a', 'b', ?3, ?4)`, | |
| 39 | ); | |
| 40 | const run = db.query( | |
| 41 | `INSERT INTO run (id, repo_id, kind, env, state, serving_at, finished_at) | |
| 42 | VALUES (?1, 'r1', ?2, ?3, 'ok', ?4, ?4)`, | |
| 43 | ); | |
| 44 | const carry = db.query( | |
| 45 | "INSERT INTO deploy_pull (run_id, issue_id, pull_number, merged_oid) VALUES (?1, ?2, 0, 'o')", | |
| 46 | ); | |
| 47 | const pillars = Array.from({ length: PILLARS }, (_, p) => ({ | |
| 48 | short: `P${p}`, | |
| 49 | name: `P${p} Pillar`, | |
| 50 | umbrella: undefined as never, | |
| 51 | })); | |
| 52 | const of = new Map<string, string>(); | |
| 53 | db.transaction(() => { | |
| 54 | for (let i = 0; i < PULLS; i++) { | |
| 55 | const ask = `ask${i}`; | |
| 56 | // Every 25th ask is sealed; the rest are public. | |
| 57 | issue.run(ask, i + 1, "issue", `ask ${i}`, "", at(160 - (i % 7)), i % 25 === 0 ? 1 : 0); | |
| 58 | of.set(ask, pillars[i % PILLARS]!.name); | |
| 59 | issue.run(`pr${i}`, PULLS + i + 1, "pull", `Land ${i}`, `Closes #${i + 1}`, at(150), 0); | |
| 60 | pull.run(`pr${i}`, `b${i}`, at(100 - (i % 50)), `oid${i}`); | |
| 61 | // Most are live a few hours after the merge; every 10th never is. | |
| 62 | if (i % 10 !== 0) { | |
| 63 | run.run(`d${i}`, "deploy", "production", at(90 - (i % 60))); | |
| 64 | carry.run(`d${i}`, `pr${i}`); | |
| 65 | } | |
| 66 | } | |
| 67 | // Noise a real repository carries: thousands of agent runs and previews. | |
High: this noise is almost all |
||
| 68 | for (let n = 0; n < 5000; n++) | |
| 69 | run.run(`n${n}`, n % 2 ? "agent" : "deploy", n % 2 ? "" : "pull-1", at(n % 160)); | |
| 70 | // And the long history of successful production deploys, each carrying pulls merged before the window. | |
| 71 | for (let n = 0; n < 40000; n++) { | |
| 72 | run.run(`h${n}`, "deploy", "production", at(n % 160)); | |
| 73 | carry.run(`h${n}`, `old${n}`); | |
| 74 | } | |
| 75 | })(); | |
| 76 | view = { set: { pillars, gate: undefined, later: undefined }, names: [], of, merged: [] }; | |
| 77 | }); | |
| 78 | ||
| 79 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 80 | ||
| 81 | /** The answer as it was read one pull at a time. */ | |
| 82 | function reference(sealedOk: boolean) { | |
| 83 | const db = app.db; | |
| 84 | const since = pillarWindowStart(); | |
| 85 | const first = new Map<number, { pillar: string; hours: number }>(); | |
| 86 | const pulls = db | |
| 87 | .query<{ id: string; body: string; at: string }, [string]>( | |
| 88 | `SELECT i.id, i.body, p.merged_at AS at FROM pull p JOIN issue i ON i.id = p.issue_id | |
| 89 | WHERE p.merged_at >= ?1`, | |
| 90 | ) | |
| 91 | .all(since); | |
| 92 | for (const p of pulls) { | |
| 93 | for (const n of closesFromBody(p.body)) { | |
| 94 | const ask = db | |
| 95 | .query<{ id: string; created_at: string; sealed: number }, [number]>( | |
| 96 | "SELECT id, created_at, sealed FROM issue WHERE number = ?1 AND kind = 'issue'", | |
| 97 | ) | |
| 98 | .get(n); | |
| 99 | const pillar = ask && view.of.get(ask.id); | |
| 100 | if (!ask || !pillar || (ask.sealed === 1 && !sealedOk)) continue; | |
| 101 | const live = db | |
| 102 | .query<{ at: string }, [string, string]>( | |
| 103 | `SELECT COALESCE(r.serving_at, r.finished_at) AS at FROM deploy_pull dp | |
| 104 | JOIN run r ON r.id = dp.run_id | |
| 105 | WHERE dp.issue_id = ?1 AND r.kind = 'deploy' AND r.env = 'production' AND r.state = 'ok' | |
| 106 | AND COALESCE(r.serving_at, r.finished_at) >= ?2 | |
| 107 | ORDER BY 1 LIMIT 1`, | |
| 108 | ) | |
| 109 | .get(p.id, p.at); | |
| 110 | if (!live) continue; | |
| 111 | const hours = Math.max(0, (Date.parse(live.at) - Date.parse(ask.created_at)) / 3_600_000); | |
| 112 | const seen = first.get(n); | |
| 113 | if (!seen || hours < seen.hours) first.set(n, { pillar, hours }); | |
| 114 | break; | |
| 115 | } | |
| 116 | } | |
| 117 | return view.set.pillars.map(({ name }) => { | |
| 118 | const hs = [...first.values()].filter((f) => f.pillar === name).map((f) => f.hours).sort((a, b) => a - b); | |
| 119 | if (hs.length === 0) return { pillar: name, askToLive: undefined }; | |
| 120 | const mid = hs.length >> 1; | |
| 121 | const median = hs.length % 2 ? hs[mid]! : (hs[mid - 1]! + hs[mid]!) / 2; | |
| 122 | return { pillar: name, askToLive: { median, slowest: hs[hs.length - 1]! } }; | |
| 123 | }); | |
| 124 | } | |
| 125 | ||
| 126 | const simple = (rows: ReturnType<typeof askToLiveOf>) => | |
| 127 | rows.map((r) => ({ | |
| 128 | pillar: r.pillar, | |
| 129 | askToLive: r.askToLive && { median: r.askToLive.medianHours, slowest: r.askToLive.slowest.hours }, | |
| 130 | })); | |
| 131 | ||
| 132 | test("the answer is the per-pull answer, for a writer and for a stranger", () => { | |
| 133 | const writer = askToLiveOf(app.db, repo, view, { sealed: true }); | |
| 134 | const stranger = askToLiveFor(app, repo, view, undefined); | |
| 135 | expect(simple(writer)).toEqual(reference(true) as never); | |
| 136 | expect(simple(stranger)).toEqual(reference(false) as never); | |
| 137 | expect(writer.filter((r) => r.askToLive).length).toBe(PILLARS); | |
| 138 | // Sealed asks are left out for the stranger, so the two differ. | |
| 139 | expect(simple(writer)).not.toEqual(simple(stranger)); | |
| 140 | expect(sealedScope(app.db, repo, undefined)).toEqual({}); | |
| 141 | }); | |
| 142 | ||
| 143 | test(`${PULLS} merged pulls over ${PILLARS} pillars answer within the budget`, async () => { | |
| 144 | askToLiveFor(app, repo, view, undefined); | |
| 145 | const base = 5; // ms | |
| 146 | let best = Infinity; | |
The test body is synchronous, so |
||
| 147 | for (let i = 0; i < 5; i++) { | |
outdated
The old code takes about 31 ms here, inside this 50 ms budget, so this assertion cannot fail without the change. Either grow the fixture until the old code misses the budget, or also assert the query plan ( |
||
| 148 | const started = performance.now(); | |
| 149 | askToLiveFor(app, repo, view, undefined); | |
| 150 | best = Math.min(best, performance.now() - started); | |
| 151 | } | |
| 152 | expectBudget(best, { ms: 50, against: base }); | |
| 153 | }); |
Low: each pull id falls in exactly one chunk, and each chunk is already
ORDER BY at, r.id. So every pull's list is already in order, and this re-sort only drops ther.idtiebreak. It can go.