Public: anyone can read this; the crew can change it.
A repository's home keeps `now` under 5 ms on the live forge: trevato/forge's home under 50 ms #6231
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6227-a-repository-s-home-keeps-now-under-5-ms → main 3 commits fork
3 files
changed +125
−7
src/core/dependencies.ts +38 −5
| @@ -12,7 +12,7 @@ import type { Database } from "bun:sqlite"; | ||
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | 13 | import { getIssuesByIds, getIssuesOfRepo, type CloseReason, type Issue } from "./issues.ts"; |
| 14 | 14 | import { pillarsOf, structuralIds, type Pillars } from "./pillars.ts"; |
| 15 | import { getRepoById } from "./repos.ts"; | |
| 15 | import { getRepoById, type Repo } from "./repos.ts"; | |
| 16 | 16 | import { unadoptedAmong } from "../crew/direct.ts"; |
| 17 | 17 | import { raisedAt, raisedInRepo } from "./thought_links.ts"; |
| 18 | 18 | import { limbRanks, limbTree, readPlaces } from "./limb_order.ts"; |
| @@ -241,7 +241,7 @@ export interface Graph { | ||
| 241 | 241 | * would take along before anything is written. |
| 242 | 242 | */ |
| 243 | 243 | export function deferredIn( |
| 244 | graph: Pick<Graph, "issues" | "edges">, | |
| 244 | graph: { issues: readonly Pick<Issue, "id" | "state">[]; edges: readonly Edge[] }, | |
| 245 | 245 | gate: string | undefined, |
| 246 | 246 | later: string | undefined, |
| 247 | 247 | also: Iterable<string> = [], |
| @@ -303,6 +303,39 @@ export function linkedIssueIds(db: Database, ids: readonly string[]): Set<string | ||
| 303 | 303 | return out; |
| 304 | 304 | } |
| 305 | 305 | |
| 306 | /** | |
| 307 | * `repoGraph(...).deferred` for a repository, reading only what it needs: the | |
| 308 | * open issues and the edges that start at one. Closed issues and the edges | |
| 309 | * between them never change what is deferred, so a repository with thousands | |
| 310 | * of finished issues costs what its open ones do. Nothing is deferred | |
| 311 | * without an open Later gate, which is checked before any edge is read. | |
| 312 | */ | |
| 313 | export function deferredOf(db: Database, repo: Repo): ReadonlyMap<string, string> { | |
| 314 | const set = pillarsOf(db, repo); | |
| 315 | const gate = set.gate?.id; | |
| 316 | if (gate === undefined || set.gate?.state !== "open") return new Map(); | |
| 317 | const issues = db | |
| 318 | .query<{ id: string; number: number; title: string }, [string, number]>( | |
| 319 | `SELECT id, number, title FROM issue | |
| 320 | WHERE repo_id = ?1 AND kind = 'issue' AND state = 'open' AND sealed = 0 | |
| 321 | ORDER BY number LIMIT ?2`, | |
| 322 | ) | |
| 323 | .all(repo.id, GRAPH_MAX) | |
| 324 | .map((i) => ({ ...i, state: "open" as const })); | |
| 325 | const open = new Set(issues.map((i) => i.id)); | |
| 326 | const edges = db | |
| 327 | .query<{ issue_id: string; blocked_by_id: string }, [string]>( | |
| 328 | `SELECT d.issue_id, d.blocked_by_id FROM issue_dependency d | |
| 329 | JOIN issue i ON i.id = d.issue_id | |
| 330 | WHERE i.repo_id = ?1 AND i.state = 'open' AND i.kind = 'issue'`, | |
| 331 | ) | |
| 332 | .all(repo.id) | |
| 333 | .filter((r) => open.has(r.issue_id) && open.has(r.blocked_by_id)) | |
| 334 | .map((r) => ({ issueId: r.issue_id, blockedById: r.blocked_by_id })); | |
| 335 | const graph = { issues, edges }; | |
| 336 | return deferredWhy(graph, deferredIn(graph, gate, set.later?.id), gate, set.later?.id); | |
| 337 | } | |
| 338 | ||
| 306 | 339 | export function repoGraph( |
| 307 | 340 | db: Database, |
| 308 | 341 | repoId: string, |
| @@ -426,7 +459,7 @@ const NAMED_LIMBS = 2; | ||
| 426 | 459 | * it — and then which: the deferred limbs above it, by number and title. |
| 427 | 460 | */ |
| 428 | 461 | function deferredWhy( |
| 429 | graph: Pick<Graph, "issues" | "edges">, | |
| 462 | graph: { issues: readonly Pick<Issue, "id" | "number" | "title">[]; edges: readonly Edge[] }, | |
| 430 | 463 | deferred: ReadonlySet<string>, |
| 431 | 464 | gate: string | undefined, |
| 432 | 465 | later: string | undefined, |
| @@ -447,8 +480,8 @@ function deferredWhy( | ||
| 447 | 480 | ]); |
| 448 | 481 | } |
| 449 | 482 | // The deferred limbs an issue hangs from: what above it was itself filed. |
| 450 | const limbs = (id: string, seen: Set<string>): Issue[] => { | |
| 451 | const found: Issue[] = []; | |
| 483 | const limbs = (id: string, seen: Set<string>): Pick<Issue, "id" | "number" | "title">[] => { | |
| 484 | const found: Pick<Issue, "id" | "number" | "title">[] = []; | |
| 452 | 485 | for (const waiter of waiters.get(id) ?? []) { |
| 453 | 486 | if (seen.has(waiter)) continue; |
| 454 | 487 | seen.add(waiter); |
src/web/studio/tell.ts +2 −2
| @@ -17,8 +17,8 @@ import { alreadyDone } from "../../core/done_on_main.ts"; | ||
| 17 | 17 | import { appsRuntime } from "../../apps/runtime.ts"; |
| 18 | 18 | import { getAppByRepo } from "../../core/apps.ts"; |
| 19 | 19 | import { LATER_TITLED } from "../../core/pillars.ts"; |
| 20 | import { deferredOf, linkedIssueIds } from "../../core/dependencies.ts"; | |
| 20 | 21 | import { hasPicture } from "../../core/note_pictures.ts"; |
| 21 | import { linkedIssueIds, repoGraph } from "../../core/dependencies.ts"; | |
| 22 | 22 | import { |
| 23 | 23 | getIssuesByNumbers, |
| 24 | 24 | labelsFor, |
| @@ -306,7 +306,7 @@ function deferredWhy( | ||
| 306 | 306 | if (byRepo === undefined) deferrals.set(wordCtx, (byRepo = new Map())); |
| 307 | 307 | let known = byRepo.get(repo.id); |
| 308 | 308 | if (known === undefined) |
| 309 | byRepo.set(repo.id, (known = repoGraph(wordCtx.db, repo.id).deferred)); | |
| 309 | byRepo.set(repo.id, (known = deferredOf(wordCtx.db, repo))); | |
| 310 | 310 | return known.get(issue.id); |
| 311 | 311 | } |
| 312 | 312 |
test/app-home-batched.test.ts +85
| @@ -7,13 +7,59 @@ import { enableApp } from "../src/actions/apps.ts"; | ||
| 7 | 7 | import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts"; |
| 8 | 8 | import { createApp, type App } from "../src/app.ts"; |
| 9 | 9 | import { forkRelation } from "../src/core/following.ts"; |
| 10 | import { deferredOf, repoGraph } from "../src/core/dependencies.ts"; | |
| 11 | import { getRepo } from "../src/core/repos.ts"; | |
| 10 | 12 | import { createUser } from "../src/core/users.ts"; |
| 11 | 13 | import { serve, type Servers } from "../src/http/server.ts"; |
| 12 | 14 | import { proveAll } from "./helpers/verified.ts"; |
| 15 | import { baselineMs, expectBudget } from "./helpers/budget.ts"; | |
| 13 | 16 | import { quietLogs } from "./helpers/globals.ts"; |
| 14 | 17 | |
| 15 | 18 | quietLogs(); |
| 16 | 19 | |
| 20 | /** | |
| 21 | * The live shape of trevato/forge: 20 pillar umbrellas among the 300 open | |
| 22 | * issues, 2,000 closed (a quarter of them landed pulls), and 3,500 edges. | |
| 23 | */ | |
| 24 | function seedLive(db: App["db"], repoId: string, author: string, from: number): void { | |
| 25 | const open = (i: number) => `${repoId}i${i}`; | |
| 26 | let n = 10_000 + from; | |
| 27 | for (let k = 0; k < 20; k++) | |
| 28 | db.query( | |
| 29 | `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state) | |
| 30 | VALUES (?1, ?2, ?3, 'issue', ?4, '', ?5, 'open')`, | |
| 31 | ).run(`${repoId}u${k}`, repoId, ++n, `P${k} Pillar ${k}: umbrella`, author); | |
| 32 | db.query( | |
| 33 | `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state) | |
| 34 | VALUES (?1, ?2, ?3, 'issue', 'Later gate: parked', '', ?4, 'open')`, | |
| 35 | ).run(`${repoId}gate`, repoId, ++n, author); | |
| 36 | const closed: string[] = []; | |
| 37 | for (let i = 0; i < 2000; i++) { | |
| 38 | const id = `${repoId}c${i}`; | |
| 39 | closed.push(id); | |
| 40 | const pull = i % 4 === 0; | |
| 41 | db.query( | |
| 42 | `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state) | |
| 43 | VALUES (?1, ?2, ?3, ?4, ?5, '', ?6, 'closed')`, | |
| 44 | ).run(id, repoId, ++n, pull ? "pull" : "issue", `Done ${i}`, author); | |
| 45 | if (pull) | |
| 46 | db.query( | |
| 47 | `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) | |
| 48 | VALUES (?1, 'main', ?2, 'x', 'y', ?3)`, | |
| 49 | ).run(id, `c${i}`, new Date(Date.now() - i * 3_600_000).toISOString()); | |
| 50 | } | |
| 51 | const edges = new Set<string>(); | |
| 52 | const add = (a: string, b: string) => { | |
| 53 | if (a === b || edges.has(`${a}>${b}`)) return; | |
| 54 | edges.add(`${a}>${b}`); | |
| 55 | db.query(`INSERT INTO issue_dependency (issue_id, blocked_by_id, actor_id) VALUES (?1, ?2, ?3)`).run(a, b, author); | |
| 56 | }; | |
| 57 | for (let i = 0; i < 280; i++) add(`${repoId}u${i % 20}`, open(i)); | |
| 58 | for (let i = 0; i < 30; i++) add(open(i), `${repoId}gate`); | |
| 59 | for (let i = 1; edges.size < 3500; i++) | |
| 60 | add(closed[i % 2000]!, closed[(i * 7 + 1 + Math.floor(i / 2000)) % 2000]!); | |
| 61 | } | |
| 62 | ||
| 17 | 63 | let root = ""; |
| 18 | 64 | let app: App; |
| 19 | 65 | let servers: Servers; |
| @@ -160,3 +206,42 @@ test("following stays cheap on a diverged fork (regression guard; this pull does | ||
| 160 | 206 | await queriesFor(path); |
| 161 | 207 | expect(Math.abs((await queriesFor(path)) - small[path]!)).toBeLessThanOrEqual(SLACK); |
| 162 | 208 | }); |
| 209 | ||
| 210 | async function phases(path: string): Promise<Record<string, number>> { | |
| 211 | const best: Record<string, number> = {}; | |
| 212 | for (let i = 0; i < 6; i++) { | |
| 213 | const res = await fetch(`${servers.url}${path}`, { headers: { cookie } }); | |
| 214 | expect(res.status).toBe(200); | |
| 215 | const body = await res.text(); | |
| 216 | if (path.endsWith("-fork")) expect(body.replace(/\s+/g, " ")).toContain("needs a decision"); | |
| 217 | for (const part of (res.headers.get("server-timing") ?? "").split(", ")) { | |
| 218 | const m = /^([a-z]+);dur=([\d.]+)/.exec(part); | |
| 219 | if (m) best[m[1]!] = Math.min(best[m[1]!] ?? Infinity, Number(m[2])); | |
| 220 | } | |
| 221 | } | |
| 222 | return best; | |
| 223 | } | |
| 224 | ||
| 225 | test("now stays within budget on the live shape: 300 open issues, 2,000 closed, 3,500 edges, 20 pillars", async () => { | |
| 226 | const repo = getRepo(app.db, "alice", "vault")!; | |
| 227 | const author = app.db.query<{ id: string }, []>("SELECT id FROM user WHERE username = 'alice'").get()!.id; | |
| 228 | app.db.transaction(() => { | |
| 229 | for (let i = 200; i < 280; i++) | |
| 230 | app.db.query( | |
| 231 | `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state) | |
| 232 | VALUES (?1, ?2, ?3, 'issue', ?4, '', ?5, 'open')`, | |
| 233 | ).run(`${repo.id}i${i}`, repo.id, 5000 + i, `Tell ${i}`, author); | |
| 234 | seedLive(app.db, repo.id, author, 0); | |
| 235 | })(); | |
| 236 | const against = await baselineMs(servers.url); | |
| 237 | const best = await phases("/alice/vault"); | |
| 238 | expectBudget(best.now!, { ms: 5, against }); | |
| 239 | expectBudget(best.shipped!, { ms: 5, against }); | |
| 240 | }); | |
| 241 | ||
| 242 | test("the lean deferred read says what the whole graph says, and something is deferred", () => { | |
| 243 | const repo = getRepo(app.db, "alice", "vault")!; | |
| 244 | const whole = repoGraph(app.db, repo.id).deferred; | |
| 245 | expect(whole.size).toBeGreaterThan(0); | |
| 246 | expect([...deferredOf(app.db, repo)].sort()).toEqual([...whole].sort()); | |
| 247 | }); |
new above
offline