Public: anyone can read this; the crew can change it.
Sentinel: The lane has stopped: nothing has merged and nothing said so #14344
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14337-sentinel-the-lane-has-stopped-nothing-ha → main 2 commits fork 1 thread · 1 unresolved
5 files
changed +193
−14
docs/features/lane-stopped.md +2
| @@ -7,4 +7,6 @@ On 2026-11-12 nothing merged for 12 h 51 min and nothing said so: every check wa | ||
| 7 | 7 | |
| 8 | 8 | It is an ask for the owner and one Sentinel issue (`src/crew/sentinel_lane.ts`), closed at the next hunt after a pull merges and the cause is gone, unless a person commented on it. The forge's own notes under the owner's name (a park note, live triage, first look) are no one's word and do not hold it open (`src/crew/person_spoke.ts`, #14263). `land_past_gate` does not open for it: the break-glass rests on red gates only (`landingStopped` in `src/checks/gate_stopped.ts`), since a merge drought with pulls waiting on review is not a red gate. |
| 9 | 9 | |
| 10 | **A stop that is the review queue's (#14337).** When the lane has stopped, the alarm reads the review queue (`reviewQueue`): an open pull is green when its newest finished check passed on the pull's current head and finished after the last merge. A pass on a head that has since moved, or on a base a merge has since moved, is stale and not counted (#3316); a person's direct push to main is not seen. The alarm says how many pulls are green and names them (up to ten), so the owner knows what to merge. When green pulls outnumber the open pulls whose newest check failed, and no one cause is failing many, the stop is the owner's ask on /flow only: no Sentinel issue is filed, because no builder can merge and the issue only parked runs that declined. An open lane issue closes, saying so, at the next hunt. One green pull among many reds is still filed, with the green pulls in its evidence. The hunt reads the alarm once and both files and closes from it. | |
| 11 | ||
| 10 | 12 | Left out: the Sentinel issue's evidence carries the failure count, so it is commented again as that count moves. |
src/crew/sentinel.ts +16 −4
| @@ -51,6 +51,7 @@ import { forgeCeiling } from "../flow/ceiling.ts"; | ||
| 51 | 51 | import { personSpoke } from "./person_spoke.ts"; |
| 52 | 52 | import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts"; |
| 53 | 53 | import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts"; |
| 54 | import { type LaneAlarm, laneAlarm } from "../flow/alarm_lane.ts"; | |
| 54 | 55 | import type { FindingKind } from "../hunt/finding.ts"; |
| 55 | 56 | import { addLabels, kindLabels } from "../hunt/label.ts"; |
| 56 | 57 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; |
| @@ -594,10 +595,18 @@ function sharedLines( | ||
| 594 | 595 | } |
| 595 | 596 | |
| 596 | 597 | /** Everything the sentinel reads, in one pass. */ |
| 597 | export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null, ledger?: ErrorLedger): Finding[] { | |
| 598 | export function findFaults( | |
| 599 | db: Database, | |
| 600 | now = Date.now(), | |
| 601 | forgeRepoId?: string, | |
| 602 | dataDir?: string, | |
| 603 | ceilingUsd: number | null = null, | |
| 604 | ledger?: ErrorLedger, | |
| 605 | lane: LaneAlarm = laneAlarm(db, new Date(now), { ceilingUsd }), | |
| 606 | ): Finding[] { | |
| 598 | 607 | const found = [ |
| 599 | 608 | ...mainAhead(db, now), |
| 600 | ...laneStopped(db, now, ceilingUsd), | |
| 609 | ...laneStopped(lane), | |
| 601 | 610 | ...budgetBreaches(db, now, forgeRepoId, dataDir), |
| 602 | 611 | ...heldSlots(db, now), |
| 603 | 612 | ...strayTriggers(db), |
| @@ -1049,7 +1058,10 @@ export function hunt( | ||
| 1049 | 1058 | let commented = 0; |
| 1050 | 1059 | const [owner = "", name = ""] = repoPath.split("/"); |
| 1051 | 1060 | const repo = getRepo(app.db, owner, name); |
| 1052 | const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config), app.signals.errors); | |
| 1061 | const ceilingUsd = forgeCeiling(app.config); | |
| 1062 | // Read once: the lane finding and the lane issue's closing both follow it. | |
| 1063 | const lane = laneAlarm(app.db, new Date(now), { ceilingUsd }); | |
| 1064 | const findings = find(app.db, now, repo?.id, app.config.dataDir, ceilingUsd, app.signals.errors, lane); | |
| 1053 | 1065 | const actor = sentinelActor(app.db); |
| 1054 | 1066 | if (repo !== undefined && actor !== undefined) { |
| 1055 | 1067 | settleShared(app, repo, actor, now); |
| @@ -1128,7 +1140,7 @@ export function hunt( | ||
| 1128 | 1140 | const active = new Set(findings.map((f) => f.key)); |
| 1129 | 1141 | closeHeld(app, repo, actor, active, now); |
| 1130 | 1142 | closeCaughtUp(app, repo, actor, active, now); |
| 1131 | closeLaneMoving(app, repo, actor, active, now); | |
| 1143 | closeLaneMoving(app, repo, actor, active, now, lane); | |
| 1132 | 1144 | } |
| 1133 | 1145 | // A hunt with nowhere to file is recorded as such, never as a quiet one. |
| 1134 | 1146 | const unfiled = repo === undefined || actor === undefined; |
src/crew/sentinel_lane.ts +17 −8
| @@ -2,21 +2,24 @@ | ||
| 2 | 2 | * The sentinel's half of the lane-stopped alarm (#11507): nothing merging for |
| 3 | 3 | * the limit, or one cause failing many pulls, is one finding; its issue closes |
| 4 | 4 | * at the next hunt after the lane moves, unless a person spoke on it. |
| 5 | * | |
| 6 | * A stop that is only the review queue (green pulls waiting on a person to | |
| 7 | * merge, no one cause failing many) is the owner's ask on /flow, not a defect | |
| 8 | * for the crew: no builder can merge, so filing it only parks a run (#14337). | |
| 5 | 9 | */ |
| 6 | 10 | |
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | 11 | import type { App } from "../app.ts"; |
| 9 | 12 | import type { Repo } from "../core/repos.ts"; |
| 10 | 13 | import type { User } from "../core/users.ts"; |
| 11 | import { laneAlarm } from "../flow/alarm_lane.ts"; | |
| 14 | import type { LaneAlarm } from "../flow/alarm_lane.ts"; | |
| 12 | 15 | import { closeCaughtUp } from "./sentinel_main_ahead.ts"; |
| 13 | 16 | import type { Finding } from "./sentinel.ts"; |
| 14 | 17 | |
| 15 | 18 | export const LANE_KEY = "lane-stopped"; |
| 16 | 19 | |
| 17 | export function laneStopped(db: Database, now: number, ceilingUsd: number | null): Finding[] { | |
| 18 | const a = laneAlarm(db, new Date(now), { ceilingUsd }); | |
| 19 | if (!a.fired) return []; | |
| 20 | /** The lane finding from this hunt's alarm, read once per hunt with the crew's ceiling. */ | |
| 21 | export function laneStopped(a: LaneAlarm): Finding[] { | |
| 22 | if (!a.fired || a.onPeople === true) return []; | |
| 20 | 23 | return [{ |
| 21 | 24 | key: LANE_KEY, |
| 22 | 25 | title: a.forge === true ? "One failure is stopping many pulls: the forge's, not theirs" : "The lane has stopped: nothing has merged and nothing said so", |
| @@ -28,7 +31,13 @@ export function laneStopped(db: Database, now: number, ceilingUsd: number | null | ||
| 28 | 31 | }]; |
| 29 | 32 | } |
| 30 | 33 | |
| 31 | /** Close the open lane issue once the lane moves; `active` is this hunt's finding keys. */ | |
| 32 | export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void { | |
| 33 | closeCaughtUp(app, repo, actor, active, now, LANE_KEY, "The lane moved: a pull merged and no one failure is stopping many"); | |
| 34 | /** | |
| 35 | * Close the open lane issue once the lane moves; `active` is this hunt's finding keys, `a` the same | |
| 36 | * alarm {@link laneStopped} read. | |
| 37 | */ | |
| 38 | export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number, a: LaneAlarm): void { | |
| 39 | const cleared = a.onPeople === true | |
| 40 | ? `No one failure is stopping pulls: ${a.green} green ${a.green === 1 ? "pull waits" : "pulls wait"} on a person to merge, which is the owner's ask on /flow, not the crew's to fix` | |
| 41 | : "The lane moved: a pull merged and no one failure is stopping many"; | |
| 42 | closeCaughtUp(app, repo, actor, active, now, LANE_KEY, cleared); | |
| 34 | 43 | } |
src/flow/alarm_lane.ts +68 −1
| @@ -42,6 +42,18 @@ export interface LaneAlarm { | ||
| 42 | 42 | /** Whether one cause is failing many pulls. */ |
| 43 | 43 | readonly forge?: boolean; |
| 44 | 44 | readonly facts?: readonly string[]; |
| 45 | /** | |
| 46 | * Open pulls whose newest finished check passed on their current head | |
| 47 | * since the last merge: they wait on a person to merge. Set when the lane | |
| 48 | * has stopped. | |
| 49 | */ | |
| 50 | readonly green?: number | undefined; | |
| 51 | /** | |
| 52 | * Whether the stop is the review queue's: green pulls wait on a person, | |
| 53 | * outnumber the red ones, and no one cause is failing many. The owner is asked; the crew has | |
| 54 | * nothing to fix (#14337). | |
| 55 | */ | |
| 56 | readonly onPeople?: boolean; | |
| 45 | 57 | } |
| 46 | 58 | |
| 47 | 59 | /** The minutes without a merge before the alarm; a whole number from 1 to 10080 is the owner's, else 90. */ |
| @@ -145,6 +157,50 @@ function ownRepoId(db: Database): string | undefined { | ||
| 145 | 157 | return owner && name ? getRepo(db, owner, name)?.id : undefined; |
| 146 | 158 | } |
| 147 | 159 | |
| 160 | /** The open pulls a person could merge now, by number, and how many open pulls are red. */ | |
| 161 | export interface Queue { | |
| 162 | /** | |
| 163 | * Numbers of the open pulls whose newest finished check passed on their current head since | |
| 164 | * `since`, oldest first. | |
| 165 | */ | |
| 166 | readonly green: readonly number[]; | |
| 167 | /** Open pulls whose newest finished check failed. */ | |
| 168 | readonly red: number; | |
| 169 | } | |
| 170 | ||
| 171 | /** | |
| 172 | * The review queue of the repository. A pull is green only when its newest | |
| 173 | * finished check passed on the pull's current head and finished at or after | |
| 174 | * `since` (the last merge): a pass on a head that has moved, or on a base a | |
| 175 | * merge has since moved, is not one the gate can land (#3316). A person's | |
| 176 | * direct push to main is not seen here. | |
| 177 | */ | |
| 178 | export function reviewQueue(db: Database, repoId: string, since: string): Queue { | |
| 179 | const rows = db | |
| 180 | .query<{ number: number; state: string; commit_oid: string; finished_at: string; head_oid: string }, [string]>( | |
| 181 | `SELECT i.number, r.state, r.commit_oid, r.finished_at, p.head_oid | |
| 182 | FROM issue i JOIN pull p ON p.issue_id = i.id | |
| 183 | JOIN run r ON r.id = (SELECT r2.id FROM run r2 | |
| 184 | WHERE r2.issue_id = i.id AND r2.kind = 'check' AND r2.state IN ('ok','failed','cancelled') | |
| 185 | ORDER BY r2.finished_at DESC, r2.id DESC LIMIT 1) | |
| 186 | WHERE i.repo_id = ?1 AND i.kind = 'pull' AND i.state = 'open' | |
| 187 | ORDER BY i.number`, | |
| 188 | ) | |
| 189 | .all(repoId); | |
| 190 | const green: number[] = []; | |
| 191 | let red = 0; | |
| 192 | for (const r of rows) { | |
| 193 | if (r.state === "failed") red++; | |
| 194 | else if (r.state === "ok" && r.head_oid !== "" && r.commit_oid === r.head_oid && r.finished_at >= since) green.push(r.number); | |
| 195 | } | |
| 196 | return { green, red }; | |
| 197 | } | |
| 198 | ||
| 199 | /** At most this many green pulls are named. */ | |
| 200 | const NAMED_MAX = 10; | |
| 201 | const named = (numbers: readonly number[]): string => | |
| 202 | numbers.slice(0, NAMED_MAX).map((n) => `#${n}`).join(", ") + (numbers.length > NAMED_MAX ? ` and ${numbers.length - NAMED_MAX} more` : ""); | |
| 203 | ||
| 148 | 204 | const count = (n: number, noun: string) => `${n} ${noun}${n === 1 ? "" : "s"}`; |
| 149 | 205 | |
| 150 | 206 | export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: number | null } = { ceilingUsd: null }): LaneAlarm { |
| @@ -171,6 +227,7 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: | ||
| 171 | 227 | const facts: string[] = []; |
| 172 | 228 | let line: string | undefined; |
| 173 | 229 | let since: string; |
| 230 | let queue: Queue | undefined; | |
| 174 | 231 | if (stopped) { |
| 175 | 232 | const failed = failuresSince(db, repoId, last!); |
| 176 | 233 | const top = groups(failed)[0]; |
| @@ -182,6 +239,12 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: | ||
| 182 | 239 | ); |
| 183 | 240 | facts.push(`last merge at ${last}`, `${count(open, "pull")} open`, `${failed.length} checks failed since the last merge`); |
| 184 | 241 | if (top !== undefined) facts.push(`commonest failure, ${top.count} times: ${top.line}`); |
| 242 | queue = reviewQueue(db, repoId, last!); | |
| 243 | const g = queue.green.length; | |
| 244 | if (g > 0) { | |
| 245 | parts.push(`${count(g, "pull")} passed ${g === 1 ? "its check and waits" : "their checks and wait"} on a person to merge: ${named(queue.green)}`); | |
| 246 | facts.push(`${count(g, "pull")} green, waiting on a person to merge: ${named(queue.green)}`); | |
| 247 | } | |
| 185 | 248 | } else since = shared!.since; |
| 186 | 249 | if (shared !== undefined) { |
| 187 | 250 | line = line ?? shared.line; |
| @@ -191,5 +254,9 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: | ||
| 191 | 254 | ); |
| 192 | 255 | facts.push(`forge failure on ${count(shared.pulls.size, "pull")} within the hour since ${shared.since}: ${shared.line}`); |
| 193 | 256 | } |
| 194 | return { fired: true, since, says: `${parts.join(". ")}.`.replace(/\.\.$/, "."), line, forge: shared !== undefined, facts }; | |
| 257 | // The queue's stop only when green pulls outnumber red ones: one green | |
| 258 | // pull among many unrelated reds is still the crew's to hear about. | |
| 259 | const green = queue?.green.length; | |
| 260 | const onPeople = shared === undefined && queue !== undefined && queue.green.length > queue.red; | |
| 261 | return { fired: true, since, says: `${parts.join(". ")}.`.replace(/\.\.$/, "."), line, forge: shared !== undefined, facts, green, onPeople }; | |
| 195 | 262 | } |
test/flow-alarm-lane.test.ts +90 −1
| @@ -49,7 +49,7 @@ function pull(state: "open" | "closed", mergedMinAgo?: number): string { | ||
| 49 | 49 | n++; |
| 50 | 50 | app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, ?5, ?6)`).run(issueId, repoId, 2000 + n, userId, state, ago(600)); |
| 51 | 51 | app.db |
| 52 | .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, ?5)`) | |
| 52 | .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', 'c', ?3, ?4, ?5)`) | |
| 53 | 53 | .run(issueId, `b/${n}`, repoId, mergedMinAgo === undefined ? null : ago(mergedMinAgo), mergedMinAgo === undefined ? "" : "9".repeat(40)); |
| 54 | 54 | return issueId; |
| 55 | 55 | } |
| @@ -60,6 +60,13 @@ function failed(issueId: string, minAgo: number, summary = LINE, detail = "{}"): | ||
| 60 | 60 | .run(runId, repoId, issueId, summary, detail, ago(minAgo)); |
| 61 | 61 | return runId; |
| 62 | 62 | } |
| 63 | /** A passed check on `oid`; every pull's head is `c`. */ | |
| 64 | function passed(issueId: string, minAgo: number, oid = "c"): void { | |
| 65 | app.db | |
| 66 | .query(`INSERT INTO run (id, repo_id, issue_id, kind, env, state, summary, check_detail, commit_oid, created_at, finished_at) VALUES (?1, ?2, ?3, 'check', 'pull-1', 'ok', 'check passed', '{}', ?5, ?4, ?4)`) | |
| 67 | .run(`${String(++n).padStart(4, "0")}${id()}`, repoId, issueId, ago(minAgo), oid); | |
| 68 | } | |
| 69 | const numberOf = (issueId: string) => app.db.query<{ number: number }, [string]>(`SELECT number FROM issue WHERE id = ?1`).get(issueId)!.number; | |
| 63 | 70 | const reset = () => { |
| 64 | 71 | app.db.query(`DELETE FROM run`).run(); |
| 65 | 72 | app.db.query(`DELETE FROM pull`).run(); |
| @@ -167,6 +174,88 @@ describe("the lane has stopped", () => { | ||
| 167 | 174 | }); |
| 168 | 175 | }); |
| 169 | 176 | |
| 177 | describe("a stop that is the review queue's (#14337)", () => { | |
| 178 | const open = () => app.db.query<{ body: string }, []>(`SELECT body FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: %lane has stopped%'`).all(); | |
| 179 | const closing = () => | |
| 180 | app.db.query<{ body: string }, []>(`SELECT c.body FROM comment c JOIN issue i ON i.id = c.issue_id WHERE i.title LIKE 'Sentinel: %lane has stopped%' ORDER BY c.created_at DESC LIMIT 1`).get()?.body ?? ""; | |
| 181 | ||
| 182 | test("green pulls outnumbering red: the owner is told which to merge, the crew gets no defect", () => { | |
| 183 | reset(); | |
| 184 | pull("closed", 200); | |
| 185 | const pulls = Array.from({ length: 3 }, () => pull("open")); | |
| 186 | failed(pulls[0]!, 150); | |
| 187 | passed(pulls[0]!, 120); | |
| 188 | passed(pulls[1]!, 110); | |
| 189 | failed(pulls[2]!, 100, "gate: rule broke"); | |
| 190 | const names = `#${numberOf(pulls[0]!)}, #${numberOf(pulls[1]!)}`; | |
| 191 | const a = laneAlarm(app.db, NOW); | |
| 192 | expect(a.fired).toBe(true); | |
| 193 | expect(a.green).toBe(2); | |
| 194 | expect(a.onPeople).toBe(true); | |
| 195 | expect(a.says).toContain(`2 pulls passed their checks and wait on a person to merge: ${names}`); | |
| 196 | expect(a.facts).toContain(`2 pulls green, waiting on a person to merge: ${names}`); | |
| 197 | // The owner's /flow alarm carries the count and the pulls to merge. | |
| 198 | const flow = lane(); | |
| 199 | expect(flow).toHaveLength(1); | |
| 200 | expect(flow[0]!.says).toContain(`2 pulls passed their checks and wait on a person to merge: ${names}`); | |
| 201 | hunt(app, "trevato/forge", NOW.getTime()); | |
| 202 | expect(open()).toHaveLength(0); | |
| 203 | }); | |
| 204 | ||
| 205 | test("one green pull among many unrelated reds is still filed for the crew", () => { | |
| 206 | reset(); | |
| 207 | pull("closed", 200); | |
| 208 | const green = pull("open"); | |
| 209 | passed(green, 120); | |
| 210 | for (let i = 0; i < 8; i++) failed(pull("open"), 100 - i, `gate: rule ${i} broke`); | |
| 211 | const a = laneAlarm(app.db, NOW); | |
| 212 | expect(a.green).toBe(1); | |
| 213 | expect(a.onPeople).toBe(false); | |
| 214 | expect(a.facts).toContain(`1 pull green, waiting on a person to merge: #${numberOf(green)}`); | |
| 215 | hunt(app, "trevato/forge", NOW.getTime()); | |
| 216 | expect(open()).toHaveLength(1); | |
| 217 | }); | |
| 218 | ||
| 219 | test("a newer red check makes a pull not green, and with none green the crew is told", () => { | |
| 220 | reset(); | |
| 221 | pull("closed", 200); | |
| 222 | const p = pull("open"); | |
| 223 | passed(p, 120); | |
| 224 | failed(p, 100, "gate: rule broke"); | |
| 225 | const a = laneAlarm(app.db, NOW); | |
| 226 | expect(a.green).toBe(0); | |
| 227 | expect(a.onPeople).toBe(false); | |
| 228 | hunt(app, "trevato/forge", NOW.getTime()); | |
| 229 | expect(open()).toHaveLength(1); | |
| 230 | }); | |
| 231 | ||
| 232 | test("a pass on a head that has moved, or from before the last merge, is stale: not green (#3316)", () => { | |
| 233 | reset(); | |
| 234 | pull("closed", 200); | |
| 235 | passed(pull("open"), 120, "old-head"); | |
| 236 | passed(pull("open"), 250); | |
| 237 | const a = laneAlarm(app.db, NOW); | |
| 238 | expect(a.fired).toBe(true); | |
| 239 | expect(a.green).toBe(0); | |
| 240 | expect(a.onPeople).toBe(false); | |
| 241 | hunt(app, "trevato/forge", NOW.getTime()); | |
| 242 | expect(open()).toHaveLength(1); | |
| 243 | }); | |
| 244 | ||
| 245 | test("an open lane issue closes, saying the queue waits on a person, once a pull goes green", () => { | |
| 246 | reset(); | |
| 247 | pull("closed", 200); | |
| 248 | const p = pull("open"); | |
| 249 | failed(p, 100, "gate: rule broke"); | |
| 250 | hunt(app, "trevato/forge", NOW.getTime()); | |
| 251 | expect(open()).toHaveLength(1); | |
| 252 | passed(p, 50); | |
| 253 | hunt(app, "trevato/forge", NOW.getTime() + 60_000); | |
| 254 | expect(open()).toHaveLength(0); | |
| 255 | expect(closing()).toContain("1 green pull waits on a person to merge"); | |
| 256 | }); | |
| 257 | }); | |
| 258 | ||
| 170 | 259 | describe("one cause failing many pulls", () => { |
| 171 | 260 | test("five pulls in an hour on one masked line are the forge's: alarm names it, the check is the forge's", () => { |
| 172 | 261 | reset(); |
new above
offline