Public: anyone can read this; the crew can change it.
Find and fix the root cause of the Ready pipeline bottleneck. #11836
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/11834-find-and-fix-the-root-cause-of-the-ready → main 1 commit fork
2 files
changed +88
−1
src/flow/pipeline.ts +10 −1
| @@ -103,8 +103,17 @@ function waitingTally(db: Database, sql: string, ...args: string[]): Tally { | ||
| 103 | 103 | // their open issues by `issue_repo_kind_state_number`, never every issue |
| 104 | 104 | // (#7896); and the ask's runs, their branch's pulls by `pull_head_branch`, |
| 105 | 105 | // then each pull's issue, never every open issue (#3693). |
| 106 | // | |
| 107 | // `since` is when the issue became ready, not when it was opened (#11834): the | |
| 108 | // latest of its opening, the close of a blocker that held it, and the end of | |
| 109 | // the last agent run on it. An old issue whose blocker closed an hour ago has | |
| 110 | // waited an hour for a builder, not fifty-six days. | |
| 106 | 111 | const READY = ` |
| 107 | SELECT i.created_at AS since | |
| 112 | SELECT MAX(i.created_at, | |
| 113 | COALESCE((SELECT MAX(b.closed_at) FROM issue_dependency d JOIN issue b ON b.id = d.blocked_by_id | |
| 114 | WHERE d.issue_id = i.id), ''), | |
| 115 | COALESCE((SELECT MAX(COALESCE(x.finished_at, x.created_at)) FROM run x | |
| 116 | WHERE x.issue_id = i.id AND x.kind = 'agent'), '')) AS since | |
| 108 | 117 | FROM repo r CROSS JOIN issue i ON i.repo_id = r.id |
| 109 | 118 | WHERE i.kind = 'issue' AND i.state = 'open' AND r.autonomy != 'off' |
| 110 | 119 | AND (r.direction_issue_id IS NULL OR r.direction_issue_id != i.id) |
test/flow-ready-since.test.ts added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | /** | |
| 2 | * #11834: the Ready stage's age is when an issue became ready, not when it | |
| 3 | * was opened. An issue opened 56 days ago whose blocker closed an hour ago, | |
| 4 | * or whose last agent run ended an hour ago, has waited an hour for a | |
| 5 | * builder; reading 56 days named Ready the bottleneck while checks (27 min) | |
| 6 | * were the real, and only, queue. | |
| 7 | */ | |
| 8 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { openIssue } from "../src/actions/issues.ts"; | |
| 14 | import { createRepo, updateRepoSettings } from "../src/actions/repos.ts"; | |
| 15 | import { addDependency } from "../src/core/dependencies.ts"; | |
| 16 | import { createRun } from "../src/core/runs.ts"; | |
| 17 | import { createUser } from "../src/core/users.ts"; | |
| 18 | import { bottleneck, readPipeline } from "../src/flow/pipeline.ts"; | |
| 19 | ||
| 20 | const HOUR = 3_600_000; | |
| 21 | const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); | |
| 22 | let dataDir: string; | |
| 23 | let app: App; | |
| 24 | ||
| 25 | const ready = () => readPipeline(app.db).find((s) => s.key === "ready")!; | |
| 26 | ||
| 27 | beforeAll(async () => { | |
| 28 | dataDir = await mkdtemp(join(tmpdir(), "forge-ready-since-")); | |
| 29 | app = createApp({ dataDir, port: 0 }); | |
| 30 | }); | |
| 31 | afterAll(async () => { | |
| 32 | app.close(); | |
| 33 | await rm(dataDir, { recursive: true, force: true }); | |
| 34 | }); | |
| 35 | ||
| 36 | test("ready age starts at the unblocking or the last run's end, not at opening", async () => { | |
| 37 | const u = await createUser(app.db, { | |
| 38 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 39 | password: "a-good-passphrase", adminIfFirst: true, | |
| 40 | }); | |
| 41 | if (!u.ok) throw new Error(u.error); | |
| 42 | const made = await createRepo(app, u.value, { name: "shop" }); | |
| 43 | if (!made.ok) throw new Error(String(made.error)); | |
| 44 | const set = await updateRepoSettings(app, u.value, made.value, { | |
| 45 | autonomy: "assisted", autonomyDirection: "x", autonomySlots: 2, autonomyCeilingUsd: 15, autonomyBy: u.value.id, | |
| 46 | }); | |
| 47 | if (!set.ok) throw new Error(String(set.error)); | |
| 48 | const repo = set.value; | |
| 49 | const open = (title: string, at: number) => { | |
| 50 | const i = openIssue(app, u.value, repo, { title }); | |
| 51 | if (!i.ok) throw new Error(String(i.error)); | |
| 52 | app.db.query("UPDATE issue SET created_at = ?1 WHERE id = ?2").run(ago(at), i.value.id); | |
| 53 | return i.value; | |
| 54 | }; | |
| 55 | ||
| 56 | // Untouched since opening: waited since opening. | |
| 57 | const plain = open("plain", 56 * 24 * HOUR); | |
| 58 | expect(ready().oldestWaiting).toBe((app.db.query("SELECT created_at c FROM issue WHERE id = ?1").get(plain.id) as { c: string }).c); | |
| 59 | ||
| 60 | // Blocked for weeks, the blocker closed an hour ago: waiting one hour. | |
| 61 | app.db.query("UPDATE issue SET created_at = ?1 WHERE id = ?2").run(ago(HOUR / 4), plain.id); | |
| 62 | const blocker = open("blocker", 60 * 24 * HOUR); | |
| 63 | const blocked = open("blocked", 56 * 24 * HOUR); | |
| 64 | const linked = addDependency(app.db, blocked, blocker, u.value.id); | |
| 65 | if (!linked.ok) throw new Error(linked.error); | |
| 66 | expect(ready().waiting).toBe(2); // plain + blocker | |
| 67 | const closedAt = ago(HOUR); | |
| 68 | app.db.query("UPDATE issue SET state = 'closed', closed_at = ?1 WHERE id = ?2").run(closedAt, blocker.id); | |
| 69 | expect(ready().waiting).toBe(2); // plain + blocked | |
| 70 | expect(ready().oldestWaiting).toBe(closedAt); | |
| 71 | ||
| 72 | // A builder's run on `plain` ended two hours ago and left nothing in flight. | |
| 73 | const run = createRun(app.db, { repoId: repo.id, kind: "agent", reason: "assigned", issueId: blocked.id }); | |
| 74 | const ended = ago(30 * 60_000); | |
| 75 | app.db.query("UPDATE run SET state = 'failed', finished_at = ?1 WHERE id = ?2").run(ended, run.id); | |
| 76 | expect(ready().oldestWaiting).toBe(ended > closedAt ? ended : closedAt); | |
| 77 | expect(bottleneck(readPipeline(app.db))?.key).toBe("ready"); | |
| 78 | }); |
new above
offline