Public: anyone can read this; the crew can change it.
The forge computes the "stuck pull" flow alarm (#5930) #6360
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6342-the-forge-computes-the-stuck-pull-flow-a → main 3 commits fork 5 threads · 5 unresolved
src/flow/alarm_stuck.ts added +134
| @@ -0,0 +1,134 @@ | ||
| 1 | /** | |
| 2 | * The "stuck pull" flow alarm (#5930): nothing approved sits still. It fires | |
| 3 | * when an approved, open pull has had no event — a run, a check, a review, a | |
| 4 | * push — for `STUCK_MINUTES`, and names the one that has waited longest as | |
| 5 | * `owner/name#N`. Held pulls never count: a person said "not yet". Scoped | |
| 6 | * like the other flow readings: the repositories the crew finishes (or one | |
| 7 | * `repoId`), and `publicOnly` leaves private ones out for a reader who must | |
| 8 | * not learn their names. Pure over the database and `now`; nothing draws it. | |
| 9 | */ | |
| 10 | ||
| 11 | import type { Database } from "bun:sqlite"; | |
| 12 | import type { App } from "../app.ts"; | |
| 13 | import { reviewSummariesFor } from "../core/reviews.ts"; | |
| 14 | ||
| 15 | export const STUCK_MINUTES = 30; | |
| 16 | ||
| 17 | export interface StuckAlarm { | |
| 18 | readonly fired: boolean; | |
| 19 | /** When the oldest stuck pull's last event happened; set only when fired. */ | |
| 20 | readonly since?: string; | |
| 21 | readonly says: string; | |
| 22 | } | |
| 23 | ||
| 24 | export interface StuckScope { | |
| 25 | /** Leave out private repositories. */ | |
| 26 | readonly publicOnly?: boolean; | |
| 27 | /** Only this repository, autonomous or not; default is every autonomous one. */ | |
| 28 | readonly repoId?: string; | |
| 29 | } | |
| 30 | ||
| 31 | interface PullRow { | |
| 32 | id: string; | |
| 33 | repo: string; | |
| 34 | number: number; | |
| 35 | repo_id: string; | |
| 36 | head_repo_id: string | null; | |
| 37 | head_branch: string; | |
| 38 | head_oid: string; | |
| 39 | } | |
| 40 | interface Last { | |
| 41 | at: string; | |
| 42 | what: string; | |
| 43 | } | |
| 44 | ||
| 45 | /** Newest first wins; ISO timestamps compare as text. */ | |
| 46 | const later = (a: Last | undefined, b: Last): Last => | |
| 47 | a === undefined || b.at > a.at ? b : a; | |
| 48 | ||
| 49 | function lastEvent(db: Database, p: PullRow): Last | undefined { | |
| 50 | let last: Last | undefined; | |
| 51 | const review = db | |
| 52 | .query<{ at: string }, [string]>( | |
| 53 | "SELECT MAX(created_at) AS at FROM review WHERE issue_id = ?1", | |
| 54 | ) | |
| 55 | .get(p.id); | |
| 56 | if (review?.at) last = later(last, { at: review.at, what: "a review" }); | |
| 57 | const runs = db | |
| 58 | .query< | |
| 59 | { kind: string; state: string; at: string }, | |
| 60 | [string] | |
| 61 | >(`SELECT kind, state, COALESCE(finished_at, started_at, created_at) AS at | |
| 62 | FROM run WHERE issue_id = ?1 ORDER BY at DESC, id DESC LIMIT 1`) | |
| 63 | .get(p.id); | |
| 64 | if (runs !== null) { | |
| 65 | const what = | |
| 66 | runs.kind !== "check" | |
| 67 | ? "a run" | |
| 68 | : runs.state === "failed" | |
| 69 | ? "a red check" | |
| 70 | : runs.state === "ok" | |
outdated
This parses every push payload in the repository, once per approved pull. Add a lower bound on |
||
| 71 | ? "a green check" | |
| 72 | : "a check"; | |
| 73 | last = later(last, { at: runs.at, what }); | |
| 74 | } | |
| 75 | const released = db | |
| 76 | .query<{ at: string | null }, [string]>( | |
| 77 | "SELECT MAX(released_at) AS at FROM pull_hold WHERE issue_id = ?1", | |
| 78 | ) | |
| 79 | .get(p.id); | |
| 80 | if (released?.at) last = later(last, { at: released.at, what: "a release" }); | |
| 81 | // Exact ref match on the parsed payload: no LIKE, so no wildcards in a branch | |
| 82 | // name. A push older than the events found so far cannot change the answer, | |
| 83 | // so only newer ones are parsed. | |
| 84 | const push = db | |
| 85 | .query< | |
| 86 | { at: string | null }, | |
| 87 | [string, string, string] | |
| 88 | >(`SELECT MAX(e.created_at) AS at FROM event e, json_each(e.payload, '$.updates') u | |
| 89 | WHERE e.kind = 'push' AND e.repo_id = ?1 AND e.created_at > ?3 | |
| 90 | AND json_extract(u.value, '$.ref') = ?2`) | |
| 91 | .get(p.head_repo_id ?? p.repo_id, `refs/heads/${p.head_branch}`, last?.at ?? ""); | |
| 92 | if (push?.at) last = later(last, { at: push.at, what: "a push" }); | |
| 93 | return last; | |
| 94 | } | |
| 95 | ||
| 96 | function sat(ms: number): string { | |
outdated
High: this reads every open pull on the forge, every repository, public and private. Scope it to the crew's repositories (or a |
||
| 97 | const m = Math.floor(ms / 60_000); | |
| 98 | const [n, unit] = | |
| 99 | m >= 1440 ? [Math.floor(m / 1440), "day"] : m >= 60 ? [Math.floor(m / 60), "hour"] : [m, "minute"]; | |
| 100 | return `${n} ${unit}${n === 1 ? "" : "s"}`; | |
| 101 | } | |
| 102 | ||
outdated
N+1: use |
||
| 103 | export function alarmStuck(from: App | Database, now: Date, scope: StuckScope = {}): StuckAlarm { | |
| 104 | const db = "db" in from ? from.db : from; | |
| 105 | const pulls = db | |
| 106 | .query<PullRow, [string | null, number]>( | |
| 107 | `SELECT issue.id, o.username || '/' || r.name AS repo, issue.number, issue.repo_id, | |
| 108 | pull.head_repo_id, pull.head_branch, pull.head_oid | |
| 109 | FROM issue JOIN pull ON pull.issue_id = issue.id | |
| 110 | JOIN repo r ON r.id = issue.repo_id JOIN user o ON o.id = r.owner_id | |
| 111 | WHERE issue.kind = 'pull' AND issue.state = 'open' AND pull.merged_at IS NULL | |
| 112 | AND (CASE WHEN ?1 IS NULL THEN r.autonomy != 'off' ELSE r.id = ?1 END) | |
| 113 | AND (?2 = 0 OR r.visibility = 'public') | |
| 114 | AND NOT EXISTS (SELECT 1 FROM pull_hold h WHERE h.issue_id = issue.id AND h.released_at IS NULL)`, | |
outdated
|
||
| 115 | ) | |
| 116 | .all(scope.repoId ?? null, scope.publicOnly ? 1 : 0); | |
| 117 | const verdicts = reviewSummariesFor(db, pulls.map((p) => ({ id: p.id, headOid: p.head_oid }))); | |
| 118 | let oldest: { repo: string; number: number; last: Last } | undefined; | |
| 119 | for (const p of pulls) { | |
| 120 | const s = verdicts.get(p.id); | |
| 121 | if (s === undefined || s.approvals === 0 || s.changesRequested > 0) continue; | |
| 122 | const last = lastEvent(db, p); | |
| 123 | if (last === undefined || now.getTime() - Date.parse(last.at) < STUCK_MINUTES * 60_000) continue; | |
| 124 | if (oldest === undefined || last.at < oldest.last.at) oldest = { repo: p.repo, number: p.number, last }; | |
| 125 | } | |
| 126 | if (oldest === undefined) return { fired: false, says: "No approved pull is sitting still." }; | |
| 127 | const { repo, number, last } = oldest; | |
| 128 | const hhmm = new Date(last.at).toISOString().slice(11, 16); | |
| 129 | return { | |
| 130 | fired: true, | |
| 131 | since: last.at, | |
| 132 | says: `${repo}#${number} has sat approved for ${sat(now.getTime() - Date.parse(last.at))}; its last event was ${last.what} at ${hhmm}.`, | |
| 133 | }; | |
| 134 | } |
test/flow-alarm-stuck.test.ts added +187
| @@ -0,0 +1,187 @@ | ||
| 1 | /** The stuck-pull alarm (#5930): approved, open, unheld, and quiet for 30 minutes. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createUser } from "../src/core/users.ts"; | |
| 8 | import { alarmStuck } from "../src/flow/alarm_stuck.ts"; | |
| 9 | ||
| 10 | const NOW = new Date("2026-10-26T12:00:00.000Z"); | |
| 11 | const ago = (m: number): string => new Date(NOW.getTime() - m * 60_000).toISOString(); | |
| 12 | ||
| 13 | let dir: string; | |
| 14 | let app: App; | |
| 15 | let n = 0; | |
| 16 | const userId = (name: string): string => | |
| 17 | app.db.query<{ id: string }, [string]>("SELECT id FROM user WHERE username = ?1").get(name)!.id; | |
| 18 | ||
| 19 | beforeAll(async () => { | |
| 20 | dir = await mkdtemp(join(tmpdir(), "alarm-stuck-")); | |
| 21 | app = createApp({ dataDir: dir, port: 0 }); | |
| 22 | const u = await createUser(app.db, { | |
| 23 | origin: "seed", | |
| 24 | trusted: true, | |
| 25 | username: "alice", | |
| 26 | email: "[email protected]", | |
| 27 | password: "a-good-passphrase", | |
| 28 | adminIfFirst: true, | |
| 29 | }); | |
| 30 | if (!u.ok) throw new Error(u.error); | |
| 31 | app.db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r', ?1, 'r')", [u.value.id]); | |
| 32 | app.db.run("UPDATE repo SET autonomy = 'auto' WHERE id = 'r'"); | |
| 33 | const b = await createUser(app.db, { | |
| 34 | origin: "seed", | |
| 35 | trusted: true, | |
| 36 | username: "bob", | |
| 37 | email: "[email protected]", | |
| 38 | password: "a-good-passphrase", | |
| 39 | }); | |
| 40 | if (!b.ok) throw new Error(b.error); | |
| 41 | }); | |
| 42 | afterAll(async () => { | |
| 43 | app.close?.(); | |
| 44 | await rm(dir, { recursive: true, force: true }); | |
| 45 | }); | |
| 46 | ||
| 47 | /** An open pull with one approval and one check run finishing `eventMin` ago. */ | |
| 48 | function pull(opts: { repo?: string; eventMin: number; approved?: boolean; held?: boolean; state?: string; check?: string; oid?: string; merged?: boolean }): number { | |
| 49 | n++; | |
| 50 | const db = app.db; | |
| 51 | const [u] = db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").all(); | |
| 52 | db.run( | |
| 53 | `INSERT INTO issue (id, repo_id, number, kind, title, author_id, state, created_at) | |
| 54 | VALUES (?1, ?6, ?2, 'pull', 't', ?3, ?4, ?5)`, | |
| 55 | [`p${n}`, n, u!.id, opts.state ?? "open", ago(10_000), opts.repo ?? "r"], | |
| 56 | ); | |
| 57 | db.run("INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?1, 'main', ?2, '', 'h', ?3)", [`p${n}`, `b${n}`, opts.merged ? ago(1) : null]); | |
| 58 | if (opts.approved !== false) | |
| 59 | db.run("INSERT INTO review (id, issue_id, author_id, state, commit_oid, created_at) VALUES (?1, ?2, ?3, 'approved', ?5, ?4)", [`rv${n}`, `p${n}`, u!.id, ago(opts.eventMin + 500), opts.oid ?? "h"]); | |
| 60 | db.run( | |
| 61 | `INSERT INTO run (id, repo_id, kind, issue_id, state, commit_oid, created_at, finished_at) | |
| 62 | VALUES (?1, 'r', 'check', ?2, ?3, 'h', ?4, ?4)`, | |
| 63 | [`run${n}`, `p${n}`, opts.check ?? "failed", ago(opts.eventMin)], | |
| 64 | ); | |
| 65 | if (opts.held) db.run("INSERT INTO pull_hold (id, issue_id, held_by, why) VALUES (?1, ?2, ?3, 'wait')", [`h${n}`, `p${n}`, u!.id]); | |
| 66 | return n; | |
| 67 | } | |
| 68 | const reset = (): void => { | |
| 69 | for (const t of ["event", "pull_hold", "run", "review", "pull"]) app.db.run(`DELETE FROM ${t}`); | |
| 70 | app.db.run("DELETE FROM issue"); | |
| 71 | }; | |
| 72 | ||
| 73 | describe("alarmStuck", () => { | |
| 74 | test("quiet forge: not fired", () => { | |
| 75 | reset(); | |
| 76 | expect(alarmStuck(app, NOW)).toEqual({ fired: false, says: "No approved pull is sitting still." }); | |
| 77 | }); | |
| 78 | ||
| 79 | test("an approved pull with an event 31 minutes ago fires and names it", () => { | |
| 80 | reset(); | |
| 81 | const num = pull({ eventMin: 31 }); | |
| 82 | const a = alarmStuck(app.db, NOW); | |
| 83 | expect(a.fired).toBe(true); | |
| 84 | expect(a.since).toBe(ago(31)); | |
| 85 | expect(a.says).toBe(`alice/r#${num} has sat approved for 31 minutes; its last event was a red check at 11:29.`); | |
| 86 | }); | |
| 87 | ||
| 88 | test("an event 29 minutes ago does not fire", () => { | |
| 89 | reset(); | |
| 90 | pull({ eventMin: 29 }); | |
| 91 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 92 | }); | |
| 93 | ||
| 94 | test("a held, unapproved, or closed pull never counts", () => { | |
| 95 | reset(); | |
| 96 | pull({ eventMin: 600, held: true }); | |
| 97 | pull({ eventMin: 600, approved: false }); | |
| 98 | pull({ eventMin: 600, state: "closed" }); | |
| 99 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 100 | }); | |
| 101 | ||
| 102 | test("names the oldest, in days", () => { | |
| 103 | reset(); | |
| 104 | pull({ eventMin: 90 }); | |
| 105 | const old = pull({ eventMin: 5 * 1440, check: "ok" }); | |
| 106 | const a = alarmStuck(app, NOW); | |
| 107 | expect(a.says).toStartWith(`alice/r#${old} has sat approved for 5 days; its last event was a green check at `); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a stale approval (for an old head) or a merged pull never counts", () => { | |
| 111 | reset(); | |
| 112 | pull({ eventMin: 600, oid: "old" }); | |
| 113 | pull({ eventMin: 600, merged: true }); | |
| 114 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 115 | }); | |
| 116 | ||
| 117 | test("changes requested at the current head excludes the pull", () => { | |
| 118 | reset(); | |
| 119 | const num = pull({ eventMin: 600 }); | |
| 120 | expect(alarmStuck(app, NOW).fired).toBe(true); | |
| 121 | const other = userId("bob"); | |
| 122 | app.db.run( | |
| 123 | "INSERT INTO review (id, issue_id, author_id, state, commit_oid, created_at) VALUES ('cr', ?1, ?2, 'changes_requested', 'h', ?3)", | |
| 124 | [`p${num}`, other, ago(599)], | |
| 125 | ); | |
| 126 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 127 | }); | |
| 128 | ||
| 129 | const push = (branch: string, minutes: number): void => { | |
| 130 | app.db.run( | |
| 131 | "INSERT INTO event (id, kind, repo_id, payload, created_at) VALUES (?1, 'push', 'r', ?2, ?3)", | |
| 132 | [`e${++n}`, JSON.stringify({ updates: [{ ref: `refs/heads/${branch}`, newOid: "x" }] }), ago(minutes)], | |
| 133 | ); | |
| 134 | }; | |
| 135 | ||
| 136 | test("a recent push to the head branch is an event; one to a look-alike branch is not", () => { | |
| 137 | reset(); | |
| 138 | const num = pull({ eventMin: 600 }); | |
| 139 | const branch = `b${num}`; | |
| 140 | // Underscore and percent are LIKE wildcards; neither may match another branch. | |
| 141 | push(branch.replace("b", "x"), 5); | |
| 142 | push("%", 5); | |
| 143 | expect(alarmStuck(app, NOW).fired).toBe(true); | |
| 144 | push(branch, 5); | |
| 145 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 146 | }); | |
| 147 | ||
| 148 | test("a branch name with wildcards matches only itself", () => { | |
| 149 | reset(); | |
| 150 | const num = pull({ eventMin: 600 }); | |
| 151 | app.db.run("UPDATE pull SET head_branch = 'a_b' WHERE issue_id = ?1", [`p${num}`]); | |
| 152 | push("axb", 5); | |
| 153 | expect(alarmStuck(app, NOW).fired).toBe(true); | |
| 154 | push("a_b", 5); | |
| 155 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 156 | }); | |
| 157 | ||
| 158 | test("a released hold is an event", () => { | |
| 159 | reset(); | |
| 160 | const num = pull({ eventMin: 600, held: true }); | |
| 161 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 162 | app.db.run("UPDATE pull_hold SET released_at = ?2 WHERE issue_id = ?1", [`p${num}`, ago(600)]); | |
| 163 | expect(alarmStuck(app, NOW).fired).toBe(true); | |
| 164 | app.db.run("UPDATE pull_hold SET released_at = ?2 WHERE issue_id = ?1", [`p${num}`, ago(5)]); | |
| 165 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 166 | }); | |
| 167 | ||
| 168 | test("a repository the crew does not finish does not count, unless asked for by id", () => { | |
| 169 | reset(); | |
| 170 | app.db.run("INSERT INTO repo (id, owner_id, name) VALUES ('q', ?1, 'quiet')", [userId("alice")]); | |
| 171 | const num = pull({ eventMin: 600, repo: "q" }); | |
| 172 | expect(alarmStuck(app, NOW).fired).toBe(false); | |
| 173 | expect(alarmStuck(app, NOW, { repoId: "q" }).says).toStartWith(`alice/quiet#${num} has sat`); | |
| 174 | expect(alarmStuck(app, NOW, { repoId: "r" }).fired).toBe(false); | |
| 175 | app.db.run("DELETE FROM repo WHERE id = 'q'"); | |
| 176 | }); | |
| 177 | ||
| 178 | test("publicOnly leaves a private repository out", () => { | |
| 179 | reset(); | |
| 180 | app.db.run("UPDATE repo SET visibility = 'private' WHERE id = 'r'"); | |
| 181 | pull({ eventMin: 600 }); | |
| 182 | expect(alarmStuck(app, NOW).fired).toBe(true); | |
| 183 | expect(alarmStuck(app, NOW, { publicOnly: true }).fired).toBe(false); | |
| 184 | app.db.run("UPDATE repo SET visibility = 'public' WHERE id = 'r'"); | |
| 185 | expect(alarmStuck(app, NOW, { publicOnly: true }).fired).toBe(true); | |
| 186 | }); | |
| 187 | }); |
Exported but called only by tests. Keep the follow-up that surfaces it (banner, API, tty) at the front of the flow queue.