Public: anyone can read this; the crew can change it.

Sentinel: The lane has stopped: nothing has merged and nothing said so #14344

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14337-sentinel-the-lane-has-stopped-nothing-ha → main 2 commits fork 1 thread · 1 unresolved

  • Merged. scout/14337-sentinel-the-lane-has-stopped-nothing-ha was merged into main as 7de9397.

Review

Sign in to review this change.
5 files changed +193 −14
docs/features/lane-stopped.md +2
@@ -7,4 +7,6 @@ On 2026-11-12 nothing merged for 12 h 51 min and nothing said so: every check wa
77
88 It is an ask for the owner and one Sentinel issue (`src/crew/sentinel_lane.ts`), closed at the next hunt after a pull merges and the cause is gone, unless a person commented on it. The forge's own notes under the owner's name (a park note, live triage, first look) are no one's word and do not hold it open (`src/crew/person_spoke.ts`, #14263). `land_past_gate` does not open for it: the break-glass rests on red gates only (`landingStopped` in `src/checks/gate_stopped.ts`), since a merge drought with pulls waiting on review is not a red gate.
99
10 **A stop that is the review queue's (#14337).** When the lane has stopped, the alarm reads the review queue (`reviewQueue`): an open pull is green when its newest finished check passed on the pull's current head and finished after the last merge. A pass on a head that has since moved, or on a base a merge has since moved, is stale and not counted (#3316); a person's direct push to main is not seen. The alarm says how many pulls are green and names them (up to ten), so the owner knows what to merge. When green pulls outnumber the open pulls whose newest check failed, and no one cause is failing many, the stop is the owner's ask on /flow only: no Sentinel issue is filed, because no builder can merge and the issue only parked runs that declined. An open lane issue closes, saying so, at the next hunt. One green pull among many reds is still filed, with the green pulls in its evidence. The hunt reads the alarm once and both files and closes from it.
11
1012 Left out: the Sentinel issue's evidence carries the failure count, so it is commented again as that count moves.
src/crew/sentinel.ts +16 −4
@@ -51,6 +51,7 @@ import { forgeCeiling } from "../flow/ceiling.ts";
5151 import { personSpoke } from "./person_spoke.ts";
5252 import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts";
5353 import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts";
54 import { type LaneAlarm, laneAlarm } from "../flow/alarm_lane.ts";
5455 import type { FindingKind } from "../hunt/finding.ts";
5556 import { addLabels, kindLabels } from "../hunt/label.ts";
5657 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
@@ -594,10 +595,18 @@ function sharedLines(
594595 }
595596
596597 /** Everything the sentinel reads, in one pass. */
597 export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null, ledger?: ErrorLedger): Finding[] {
598 export function findFaults(
599 db: Database,
600 now = Date.now(),
601 forgeRepoId?: string,
602 dataDir?: string,
603 ceilingUsd: number | null = null,
604 ledger?: ErrorLedger,
605 lane: LaneAlarm = laneAlarm(db, new Date(now), { ceilingUsd }),
606 ): Finding[] {
598607 const found = [
599608 ...mainAhead(db, now),
600 ...laneStopped(db, now, ceilingUsd),
609 ...laneStopped(lane),
601610 ...budgetBreaches(db, now, forgeRepoId, dataDir),
602611 ...heldSlots(db, now),
603612 ...strayTriggers(db),
@@ -1049,7 +1058,10 @@ export function hunt(
10491058 let commented = 0;
10501059 const [owner = "", name = ""] = repoPath.split("/");
10511060 const repo = getRepo(app.db, owner, name);
1052 const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config), app.signals.errors);
1061 const ceilingUsd = forgeCeiling(app.config);
1062 // Read once: the lane finding and the lane issue's closing both follow it.
1063 const lane = laneAlarm(app.db, new Date(now), { ceilingUsd });
1064 const findings = find(app.db, now, repo?.id, app.config.dataDir, ceilingUsd, app.signals.errors, lane);
10531065 const actor = sentinelActor(app.db);
10541066 if (repo !== undefined && actor !== undefined) {
10551067 settleShared(app, repo, actor, now);
@@ -1128,7 +1140,7 @@ export function hunt(
11281140 const active = new Set(findings.map((f) => f.key));
11291141 closeHeld(app, repo, actor, active, now);
11301142 closeCaughtUp(app, repo, actor, active, now);
1131 closeLaneMoving(app, repo, actor, active, now);
1143 closeLaneMoving(app, repo, actor, active, now, lane);
11321144 }
11331145 // A hunt with nowhere to file is recorded as such, never as a quiet one.
11341146 const unfiled = repo === undefined || actor === undefined;
src/crew/sentinel_lane.ts +17 −8
@@ -2,21 +2,24 @@
22 * The sentinel's half of the lane-stopped alarm (#11507): nothing merging for
33 * the limit, or one cause failing many pulls, is one finding; its issue closes
44 * at the next hunt after the lane moves, unless a person spoke on it.
5 *
6 * A stop that is only the review queue (green pulls waiting on a person to
7 * merge, no one cause failing many) is the owner's ask on /flow, not a defect
8 * for the crew: no builder can merge, so filing it only parks a run (#14337).
59 */
610
7 import type { Database } from "bun:sqlite";
811 import type { App } from "../app.ts";
912 import type { Repo } from "../core/repos.ts";
1013 import type { User } from "../core/users.ts";
11 import { laneAlarm } from "../flow/alarm_lane.ts";
14 import type { LaneAlarm } from "../flow/alarm_lane.ts";
1215 import { closeCaughtUp } from "./sentinel_main_ahead.ts";
1316 import type { Finding } from "./sentinel.ts";
1417
1518 export const LANE_KEY = "lane-stopped";
1619
17 export function laneStopped(db: Database, now: number, ceilingUsd: number | null): Finding[] {
18 const a = laneAlarm(db, new Date(now), { ceilingUsd });
19 if (!a.fired) return [];
20 /** The lane finding from this hunt's alarm, read once per hunt with the crew's ceiling. */
21 export function laneStopped(a: LaneAlarm): Finding[] {
22 if (!a.fired || a.onPeople === true) return [];
2023 return [{
2124 key: LANE_KEY,
2225 title: a.forge === true ? "One failure is stopping many pulls: the forge's, not theirs" : "The lane has stopped: nothing has merged and nothing said so",
@@ -28,7 +31,13 @@ export function laneStopped(db: Database, now: number, ceilingUsd: number | null
2831 }];
2932 }
3033
31 /** Close the open lane issue once the lane moves; `active` is this hunt's finding keys. */
32 export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void {
33 closeCaughtUp(app, repo, actor, active, now, LANE_KEY, "The lane moved: a pull merged and no one failure is stopping many");
34 /**
35 * Close the open lane issue once the lane moves; `active` is this hunt's finding keys, `a` the same
36 * alarm {@link laneStopped} read.
37 */
38 export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number, a: LaneAlarm): void {
39 const cleared = a.onPeople === true
40 ? `No one failure is stopping pulls: ${a.green} green ${a.green === 1 ? "pull waits" : "pulls wait"} on a person to merge, which is the owner's ask on /flow, not the crew's to fix`
41 : "The lane moved: a pull merged and no one failure is stopping many";
42 closeCaughtUp(app, repo, actor, active, now, LANE_KEY, cleared);
3443 }
src/flow/alarm_lane.ts +68 −1
@@ -42,6 +42,18 @@ export interface LaneAlarm {
4242 /** Whether one cause is failing many pulls. */
4343 readonly forge?: boolean;
4444 readonly facts?: readonly string[];
45 /**
46 * Open pulls whose newest finished check passed on their current head
47 * since the last merge: they wait on a person to merge. Set when the lane
48 * has stopped.
49 */
50 readonly green?: number | undefined;
51 /**
52 * Whether the stop is the review queue's: green pulls wait on a person,
53 * outnumber the red ones, and no one cause is failing many. The owner is asked; the crew has
54 * nothing to fix (#14337).
55 */
56 readonly onPeople?: boolean;
4557 }
4658
4759 /** The minutes without a merge before the alarm; a whole number from 1 to 10080 is the owner's, else 90. */
@@ -145,6 +157,50 @@ function ownRepoId(db: Database): string | undefined {
145157 return owner && name ? getRepo(db, owner, name)?.id : undefined;
146158 }
147159
160 /** The open pulls a person could merge now, by number, and how many open pulls are red. */
161 export interface Queue {
162 /**
163 * Numbers of the open pulls whose newest finished check passed on their current head since
164 * `since`, oldest first.
165 */
166 readonly green: readonly number[];
167 /** Open pulls whose newest finished check failed. */
168 readonly red: number;
169 }
170
171 /**
172 * The review queue of the repository. A pull is green only when its newest
173 * finished check passed on the pull's current head and finished at or after
174 * `since` (the last merge): a pass on a head that has moved, or on a base a
175 * merge has since moved, is not one the gate can land (#3316). A person's
176 * direct push to main is not seen here.
177 */
178 export function reviewQueue(db: Database, repoId: string, since: string): Queue {
179 const rows = db
180 .query<{ number: number; state: string; commit_oid: string; finished_at: string; head_oid: string }, [string]>(
181 `SELECT i.number, r.state, r.commit_oid, r.finished_at, p.head_oid
182 FROM issue i JOIN pull p ON p.issue_id = i.id
183 JOIN run r ON r.id = (SELECT r2.id FROM run r2
184 WHERE r2.issue_id = i.id AND r2.kind = 'check' AND r2.state IN ('ok','failed','cancelled')
185 ORDER BY r2.finished_at DESC, r2.id DESC LIMIT 1)
186 WHERE i.repo_id = ?1 AND i.kind = 'pull' AND i.state = 'open'
187 ORDER BY i.number`,
188 )
189 .all(repoId);
190 const green: number[] = [];
191 let red = 0;
192 for (const r of rows) {
193 if (r.state === "failed") red++;
194 else if (r.state === "ok" && r.head_oid !== "" && r.commit_oid === r.head_oid && r.finished_at >= since) green.push(r.number);
195 }
196 return { green, red };
197 }
198
199 /** At most this many green pulls are named. */
200 const NAMED_MAX = 10;
201 const named = (numbers: readonly number[]): string =>
202 numbers.slice(0, NAMED_MAX).map((n) => `#${n}`).join(", ") + (numbers.length > NAMED_MAX ? ` and ${numbers.length - NAMED_MAX} more` : "");
203
148204 const count = (n: number, noun: string) => `${n} ${noun}${n === 1 ? "" : "s"}`;
149205
150206 export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: number | null } = { ceilingUsd: null }): LaneAlarm {
@@ -171,6 +227,7 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd:
171227 const facts: string[] = [];
172228 let line: string | undefined;
173229 let since: string;
230 let queue: Queue | undefined;
174231 if (stopped) {
175232 const failed = failuresSince(db, repoId, last!);
176233 const top = groups(failed)[0];
@@ -182,6 +239,12 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd:
182239 );
183240 facts.push(`last merge at ${last}`, `${count(open, "pull")} open`, `${failed.length} checks failed since the last merge`);
184241 if (top !== undefined) facts.push(`commonest failure, ${top.count} times: ${top.line}`);
242 queue = reviewQueue(db, repoId, last!);
243 const g = queue.green.length;
244 if (g > 0) {
245 parts.push(`${count(g, "pull")} passed ${g === 1 ? "its check and waits" : "their checks and wait"} on a person to merge: ${named(queue.green)}`);
246 facts.push(`${count(g, "pull")} green, waiting on a person to merge: ${named(queue.green)}`);
247 }
185248 } else since = shared!.since;
186249 if (shared !== undefined) {
187250 line = line ?? shared.line;
@@ -191,5 +254,9 @@ export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd:
191254 );
192255 facts.push(`forge failure on ${count(shared.pulls.size, "pull")} within the hour since ${shared.since}: ${shared.line}`);
193256 }
194 return { fired: true, since, says: `${parts.join(". ")}.`.replace(/\.\.$/, "."), line, forge: shared !== undefined, facts };
257 // The queue's stop only when green pulls outnumber red ones: one green
258 // pull among many unrelated reds is still the crew's to hear about.
259 const green = queue?.green.length;
260 const onPeople = shared === undefined && queue !== undefined && queue.green.length > queue.red;
261 return { fired: true, since, says: `${parts.join(". ")}.`.replace(/\.\.$/, "."), line, forge: shared !== undefined, facts, green, onPeople };
195262 }
test/flow-alarm-lane.test.ts +90 −1
@@ -49,7 +49,7 @@ function pull(state: "open" | "closed", mergedMinAgo?: number): string {
4949 n++;
5050 app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, ?5, ?6)`).run(issueId, repoId, 2000 + n, userId, state, ago(600));
5151 app.db
52 .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, ?5)`)
52 .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', 'c', ?3, ?4, ?5)`)
5353 .run(issueId, `b/${n}`, repoId, mergedMinAgo === undefined ? null : ago(mergedMinAgo), mergedMinAgo === undefined ? "" : "9".repeat(40));
5454 return issueId;
5555 }
@@ -60,6 +60,13 @@ function failed(issueId: string, minAgo: number, summary = LINE, detail = "{}"):
6060 .run(runId, repoId, issueId, summary, detail, ago(minAgo));
6161 return runId;
6262 }
63 /** A passed check on `oid`; every pull's head is `c`. */
64 function passed(issueId: string, minAgo: number, oid = "c"): void {
65 app.db
66 .query(`INSERT INTO run (id, repo_id, issue_id, kind, env, state, summary, check_detail, commit_oid, created_at, finished_at) VALUES (?1, ?2, ?3, 'check', 'pull-1', 'ok', 'check passed', '{}', ?5, ?4, ?4)`)
67 .run(`${String(++n).padStart(4, "0")}${id()}`, repoId, issueId, ago(minAgo), oid);
68 }
69 const numberOf = (issueId: string) => app.db.query<{ number: number }, [string]>(`SELECT number FROM issue WHERE id = ?1`).get(issueId)!.number;
6370 const reset = () => {
6471 app.db.query(`DELETE FROM run`).run();
6572 app.db.query(`DELETE FROM pull`).run();
@@ -167,6 +174,88 @@ describe("the lane has stopped", () => {
167174 });
168175 });
169176
177 describe("a stop that is the review queue's (#14337)", () => {
178 const open = () => app.db.query<{ body: string }, []>(`SELECT body FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: %lane has stopped%'`).all();
179 const closing = () =>
180 app.db.query<{ body: string }, []>(`SELECT c.body FROM comment c JOIN issue i ON i.id = c.issue_id WHERE i.title LIKE 'Sentinel: %lane has stopped%' ORDER BY c.created_at DESC LIMIT 1`).get()?.body ?? "";
181
182 test("green pulls outnumbering red: the owner is told which to merge, the crew gets no defect", () => {
183 reset();
184 pull("closed", 200);
185 const pulls = Array.from({ length: 3 }, () => pull("open"));
186 failed(pulls[0]!, 150);
187 passed(pulls[0]!, 120);
188 passed(pulls[1]!, 110);
189 failed(pulls[2]!, 100, "gate: rule broke");
190 const names = `#${numberOf(pulls[0]!)}, #${numberOf(pulls[1]!)}`;
191 const a = laneAlarm(app.db, NOW);
192 expect(a.fired).toBe(true);
193 expect(a.green).toBe(2);
194 expect(a.onPeople).toBe(true);
195 expect(a.says).toContain(`2 pulls passed their checks and wait on a person to merge: ${names}`);
196 expect(a.facts).toContain(`2 pulls green, waiting on a person to merge: ${names}`);
197 // The owner's /flow alarm carries the count and the pulls to merge.
198 const flow = lane();
199 expect(flow).toHaveLength(1);
200 expect(flow[0]!.says).toContain(`2 pulls passed their checks and wait on a person to merge: ${names}`);
201 hunt(app, "trevato/forge", NOW.getTime());
202 expect(open()).toHaveLength(0);
203 });
204
205 test("one green pull among many unrelated reds is still filed for the crew", () => {
206 reset();
207 pull("closed", 200);
208 const green = pull("open");
209 passed(green, 120);
210 for (let i = 0; i < 8; i++) failed(pull("open"), 100 - i, `gate: rule ${i} broke`);
211 const a = laneAlarm(app.db, NOW);
212 expect(a.green).toBe(1);
213 expect(a.onPeople).toBe(false);
214 expect(a.facts).toContain(`1 pull green, waiting on a person to merge: #${numberOf(green)}`);
215 hunt(app, "trevato/forge", NOW.getTime());
216 expect(open()).toHaveLength(1);
217 });
218
219 test("a newer red check makes a pull not green, and with none green the crew is told", () => {
220 reset();
221 pull("closed", 200);
222 const p = pull("open");
223 passed(p, 120);
224 failed(p, 100, "gate: rule broke");
225 const a = laneAlarm(app.db, NOW);
226 expect(a.green).toBe(0);
227 expect(a.onPeople).toBe(false);
228 hunt(app, "trevato/forge", NOW.getTime());
229 expect(open()).toHaveLength(1);
230 });
231
232 test("a pass on a head that has moved, or from before the last merge, is stale: not green (#3316)", () => {
233 reset();
234 pull("closed", 200);
235 passed(pull("open"), 120, "old-head");
236 passed(pull("open"), 250);
237 const a = laneAlarm(app.db, NOW);
238 expect(a.fired).toBe(true);
239 expect(a.green).toBe(0);
240 expect(a.onPeople).toBe(false);
241 hunt(app, "trevato/forge", NOW.getTime());
242 expect(open()).toHaveLength(1);
243 });
244
245 test("an open lane issue closes, saying the queue waits on a person, once a pull goes green", () => {
246 reset();
247 pull("closed", 200);
248 const p = pull("open");
249 failed(p, 100, "gate: rule broke");
250 hunt(app, "trevato/forge", NOW.getTime());
251 expect(open()).toHaveLength(1);
252 passed(p, 50);
253 hunt(app, "trevato/forge", NOW.getTime() + 60_000);
254 expect(open()).toHaveLength(0);
255 expect(closing()).toContain("1 green pull waits on a person to merge");
256 });
257 });
258
170259 describe("one cause failing many pulls", () => {
171260 test("five pulls in an hour on one masked line are the forge's: alarm names it, the check is the forge's", () => {
172261 reset();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs