Public: anyone can read this; the crew can change it.

Sentinel files one issue per cause, not one per run, and stays quiet about symptoms of an open incident #14665

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14655-sentinel-files-one-issue-per-cause-not-o → main 4 commits fork

  • Merged. mason/14655-sentinel-files-one-issue-per-cause-not-o was merged into main as a6f610c.

Review

Sign in to review this change.
6 files changed +496 −5
docs/SENTINEL.md +14
@@ -25,6 +25,20 @@ issue, and only when its evidence changed; one whose issue was closed and
2525 returned is filed again. It files in `FORGE_SENTINEL_REPO` (default
2626 `trevato/forge`) as the `sentinel` agent, else the first admin.
2727
28 **One issue per cause** (#14655, `src/crew/sentinel_causes.ts`). A finding is keyed by its
29 cause, not its instance: a run killed for its budget files under
30 `degraded:budget:<run kind>:<what ran out>` (e.g. `degraded:budget:review:time`), and a new
31 kill adds to that issue's evidence (`instances`, `latest run`, `latest at`) instead of
32 opening another. A shared test was already one cause per test and keeps its key. While an
33 incident is open (the lane stopped, `clone_failing`, a `queue:*` degraded, a crash loop) the
34 budget kills and stuck runs it explains are `symptom:` lines on the incident's issue, not
35 issues. A cause-keyed issue not seen for 24 hours closes itself with its last evidence (a
36 person's comment keeps it open). One detector is new: `cloneFailing` files
37 `degraded:clone-failing` when the clone breaker (`crew/clone_breaker.ts`) has tripped, so the
38 failed clones have an incident to be symptoms of. Otherwise detection is unchanged; only filing
39 is. A kill is keyed by what ran out: `time` (the forge's "took N min, limit M"), `runs-per-day`,
40 `dollars-per-day` (the maintainer's cap), else `other`.
41
2842 **How it is labelled** (#8856). Each finding declares a kind, and is filed
2943 with its labels, so the bug clock (`src/flow/bugs.ts`) runs from the filing
3044 and nobody has to sort it:
src/crew/sentinel.ts +10 −4
@@ -55,6 +55,7 @@ import { type LaneAlarm, laneAlarm } from "../flow/alarm_lane.ts";
5555 import type { FindingKind } from "../hunt/finding.ts";
5656 import { addLabels, kindLabels } from "../hunt/label.ts";
5757 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
58 import { cloneFailing, foldByCause, closeQuiet } from "./sentinel_causes.ts";
5859 import { seenCause } from "./sentinel_seen.ts";
5960 import { memoryWatch, type MemoryWatch } from "../perf/memory_watch.ts";
6061
@@ -104,6 +105,8 @@ export interface Finding {
104105 readonly cause?: string;
105106 /** What the detector read, for the issue's evidence block (#13033): path and measurement. */
106107 readonly seen?: Evidence;
108 /** Symptoms of an open incident were added to its evidence (#14655): told at most once a day. */
109 readonly folded?: true;
107110 }
108111
109112 const QUOTE_MAX = 200;
@@ -621,7 +624,8 @@ export function findFaults(
621624 ledger?: ErrorLedger,
622625 lane: LaneAlarm = laneAlarm(db, new Date(now), { ceilingUsd }),
623626 ): Finding[] {
624 const found = [
627 const found = foldByCause(db, [
628 ...cloneFailing(db, now),
625629 ...mainAhead(db, now),
626630 ...laneStopped(lane),
627631 ...budgetBreaches(db, now, forgeRepoId, dataDir),
@@ -631,7 +635,7 @@ export function findFaults(
631635 ...runStreaks(db),
632636 ...repeatedErrors(now, ledger),
633637 ...degradedCauses(db, now, forgeRepoId, dataDir),
634 ];
638 ]);
635639 const keys = new Set<string>();
636640 return found
637641 .filter((f) => !keys.has(f.key) && keys.add(f.key))
@@ -1105,7 +1109,8 @@ export function hunt(
11051109 // numbers. Told at most once a day, so a cause degraded all day is one issue with a
11061110 // comment a day.
11071111 const toldAt = Date.parse(tracked.commented_at ?? tracked.first_seen);
1108 if (tracked.evidence !== evidence && !(f.key.startsWith("degraded:") && now - toldAt < COMMENT_EVERY_MS)) {
1112 const toldToday = (f.key.startsWith("degraded:") || f.folded === true) && now - toldAt < COMMENT_EVERY_MS;
1113 if (tracked.evidence !== evidence && !toldToday) {
11091114 const issue = getIssueById(app.db, tracked.issue_id);
11101115 if (issue === undefined) continue;
11111116 const said = addComment(app, actor, repo, issue, `Still there at ${at}, evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`);
@@ -1170,6 +1175,7 @@ export function hunt(
11701175 if (repo !== undefined && actor !== undefined) {
11711176 const active = new Set(findings.map((f) => f.key));
11721177 closeHeld(app, repo, actor, active, now);
1178 closeQuiet(app, repo, actor, active, now);
11731179 closeCaughtUp(app, repo, actor, active, now);
11741180 closeLaneMoving(app, repo, actor, active, now, lane);
11751181 }
@@ -1241,7 +1247,7 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null {
12411247 // Every tick, before a builder takes it: see `settleShared`.
12421248 const actor = sentinelActor(app.db);
12431249 if (ownRepo !== undefined && actor !== undefined) settleShared(app, ownRepo, actor, now);
1244 const breaches = budgetBreaches(app.db, now, ownRepo?.id);
1250 const breaches = foldByCause(app.db, budgetBreaches(app.db, now, ownRepo?.id));
12451251 if (since < SIGNAL_GAP_MS && breaches.length === 0) return null;
12461252 if (since < HUNT_EVERY_MS) {
12471253 // A breach is not held back by the gap: the crew hears within a tick.
src/crew/sentinel_causes.ts added +209
@@ -0,0 +1,209 @@
1 /**
2 * One issue per cause, not per instance (#14655). A budget kill names the run it
3 * killed and a shared test names a hash, so each new instance used to be a new
4 * issue: 30 for one clone storm. Here a finding's key is its cause (`budget:<run
5 * kind>:<what ran out>`), instances of one cause fold into one
6 * finding that counts them, and while an incident alarm is open the symptoms it
7 * explains are lines of its issue, not issues of their own. Detection is untouched.
8 */
9
10 import type { Database } from "bun:sqlite";
11 import type { App } from "../app.ts";
12 import { addComment, setIssueState } from "../actions/issues.ts";
13 import { getIssueById } from "../core/issues.ts";
14 import type { Repo } from "../core/repos.ts";
15 import type { User } from "../core/users.ts";
16 import { createLog } from "../kernel/log.ts";
17 import { cloneBreaker } from "./clone_breaker.ts";
18 import { personSpoke } from "./person_spoke.ts";
19 import type { Finding } from "./sentinel.ts";
20
21 const log = createLog("sentinel");
22
23 /** A cause with no recurrence for this long closes its own issue. */
24 const QUIET_CLOSE_MS = 24 * 3_600_000;
25
26 /** The key of the clone storm's issue: the incident its failed runs are symptoms of. */
27 const CLONE_KEY = "degraded:clone-failing";
28
29 /** Key families that name a cause, not an instance: the ones that close when quiet. */
30 const CAUSE_KEY_PREFIXES: readonly string[] = [
31 "degraded:budget:",
32 "degraded:stuck:",
33 CLONE_KEY,
34 ];
35
36 const part = (text: string): string => text.toLowerCase().replace(/[^a-z0-9._/-]+/g, "-").slice(0, 60);
37
38 interface RunRow {
39 readonly kind: string;
40 readonly summary: string | null;
41 readonly finished_at: string | null;
42 }
43
44 const runOf = (db: Database, id: string): RunRow | null =>
45 db.query<RunRow, [string]>("SELECT kind, summary, finished_at FROM run WHERE id = ?1").get(id);
46
47 /**
48 * What a budget kill ran out of. The kills the forge writes say "budget: <what> took N min, limit
49 * M" (`budgetSummary`: time), and the maintainer's cap says "budget: runs per day, ..." or
50 * "budget: dollars per day, ..." (`maintain/budget.ts`). Anything else is "other".
51 */
52 export function ranOut(summary: string | null): string {
53 const text = summary ?? "";
54 if (/took \d+ min, limit \d+/.test(text)) return "time";
55 if (text.startsWith("budget: runs per day")) return "runs-per-day";
56 if (text.startsWith("budget: dollars per day")) return "dollars-per-day";
57 return "other";
58 }
59
60 /**
61 * The key a finding of this cause is filed under: the cause, not the run or hash that showed it.
62 */
63 export function causeKey(db: Database, f: Finding): string {
64 const cause = f.cause;
65 if (cause === undefined) return f.key;
66 const run = /^(budget|stuck):(.+)$/.exec(cause);
67 if (run !== null) {
68 const row = runOf(db, run[2] ?? "");
69 const kind = part(row?.kind ?? "run");
70 return run[1] === "budget" ? `degraded:budget:${kind}:${ranOut(row?.summary ?? null)}` : `degraded:stuck:${kind}`;
71 }
72 return f.key;
73 }
74
75 /**
76 * The clone storm as a finding, when the breaker has tripped: the incident its dead runs belong to.
77 */
78 export function cloneFailing(db: Database, now: number): Finding[] {
79 const c = cloneBreaker(db, now);
80 if (!c.fired) return [];
81 return [{
82 key: CLONE_KEY,
83 title: "Crew runs are dying in their clone",
84 evidence: [`says: ${c.says}`, `since: ${c.since}`, "href: /boards"],
85 pillar: "P0 Trust: the forge says when it is not well, and is well again",
86 kind: "defect",
87 lookFor: "A crew run gets past its clone, and the last runs do not die cloning.",
88 urgent: true,
89 cause: "clone_failing",
90 }];
91 }
92
93 /**
94 * Whether a finding is an incident alarm that is open now: the lane, the clone, a queue or the
95 * engine.
96 */
97 function isIncident(f: Finding): boolean {
98 if (f.endedAt !== undefined) return false;
99 return f.key === "lane-stopped" || f.key === CLONE_KEY || /^(queue:|crash-loop$)/.test(f.cause ?? "");
100 }
101
102 /** A symptom of an incident: a run killed for its budget, or one that stopped moving. */
103 const isSymptom = (f: Finding): boolean => /^(budget|stuck):/.test(f.cause ?? "");
104
105 /** The newest instance of a group: the one whose run finished last, else the last read. */
106 function latestOf(db: Database, group: readonly Finding[]): { f: Finding; at: string | undefined } {
107 let best = { f: group[group.length - 1]!, at: undefined as string | undefined };
108 for (const f of group) {
109 const id = /^(?:budget|stuck):(.+)$/.exec(f.cause ?? "")?.[1];
110 const at = id === undefined ? undefined : runOf(db, id)?.finished_at ?? undefined;
111 if (at !== undefined && (best.at === undefined || at >= best.at)) best = { f, at };
112 }
113 return best;
114 }
115
116 /** One line a cause's instances add to its issue: how many, the latest run, the latest time. */
117 function instancesLine(db: Database, group: readonly Finding[]): string[] {
118 const { f, at } = latestOf(db, group);
119 const run = /^(?:budget|stuck):(.+)$/.exec(f.cause ?? "")?.[1];
120 return [
121 `instances: ${group.length}`,
122 ...(run === undefined ? [] : [`latest run: ${run}`]),
123 ...(at === undefined ? [] : [`latest at: ${at}`]),
124 ];
125 }
126
127 /**
128 * Key each finding by its cause, fold the instances of one cause into one finding
129 * (the newest instance's evidence, plus its count), then fold the symptoms of an
130 * open incident into that incident's finding. Order is kept; the result has one
131 * finding per key.
132 */
133 export function foldByCause(db: Database, found: readonly Finding[]): Finding[] {
134 const groups = new Map<string, Finding[]>();
135 for (const f of found) {
136 const key = causeKey(db, f);
137 const list = groups.get(key);
138 if (list === undefined) groups.set(key, [{ ...f, key }]);
139 // The same instance read by two detectors (a breach and the day's total) counts once.
140 else if (f.cause === undefined || !list.some((g) => g.cause === f.cause)) list.push({ ...f, key });
141 }
142 const folded: Finding[] = [];
143 for (const [key, group] of groups) {
144 // A group without a run cause (a shared test, a lane) keeps its one finding as it is.
145 if (!/^degraded:(budget|stuck):/.test(key)) {
146 folded.push(group[0]!);
147 continue;
148 }
149 const { f } = latestOf(db, group);
150 const name = key.replace(/^degraded:/, "");
151 folded.push({
152 ...f,
153 key,
154 title: key.startsWith("degraded:stuck:")
155 ? `${name}: runs of this kind stopped moving`
156 : `${name} is past its budget: runs of this kind were stopped by it`,
157 evidence: [...f.evidence, ...instancesLine(db, group)],
158 });
159 }
160 const incident = folded.find(isIncident);
161 if (incident === undefined) return folded;
162 const symptoms = folded.filter(isSymptom);
163 if (symptoms.length === 0) return folded;
164 const lines = symptoms.map((s) => {
165 const n = s.evidence.find((e) => e.startsWith("instances: "))?.slice(11) ?? "1";
166 const latest = s.evidence.find((e) => e.startsWith("latest run: "))?.slice(12);
167 const at = s.evidence.find((e) => e.startsWith("latest at: "))?.slice(11);
168 return `symptom: ${s.key.replace(/^degraded:/, "")} x${n}${latest === undefined ? "" : `, latest run ${latest}`}${at === undefined ? "" : ` at ${at}`}`;
169 });
170 return folded
171 .filter((f) => !isSymptom(f))
172 .map((f) => (f === incident ? { ...f, evidence: [...f.evidence, ...lines], folded: true as const } : f));
173 }
174
175 /**
176 * A cause-keyed issue whose cause has not been seen for 24 hours closes itself with the
177 * last evidence it held, so nothing lingers. `active` are this hunt's keys. One a person
178 * spoke on stays: they decide.
179 */
180 export function closeQuiet(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void {
181 const open = app.db
182 .query<{ key: string; issue_id: string; evidence: string; last_seen: string }, []>(
183 `SELECT t.key, t.issue_id, t.evidence, t.last_seen FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id WHERE i.state = 'open'`,
184 )
185 .all();
186 for (const t of open) {
187 try {
188 if (active.has(t.key) || !CAUSE_KEY_PREFIXES.some((p) => t.key.startsWith(p))) continue;
189 if (!(now - Date.parse(t.last_seen) >= QUIET_CLOSE_MS) || personSpoke(app.db, t.issue_id)) continue;
190 const issue = getIssueById(app.db, t.issue_id);
191 if (issue === undefined || issue.repoId !== repo.id) continue;
192 const said = addComment(
193 app,
194 actor,
195 repo,
196 issue,
197 `Not seen since ${t.last_seen}: this cause has not recurred for 24 hours. Last evidence:\n${t.evidence.split("\n").map((e) => `- ${e}`).join("\n")}\nClosing as done; if it comes back it is filed again.`,
198 );
199 if (!said.ok) {
200 log.warn("sentinel could not comment", { key: t.key, error: String(said.error) });
201 continue;
202 }
203 const closed = setIssueState(app, actor, repo, issue, "closed", { reason: "done", system: true });
204 if (!closed.ok) log.warn("sentinel could not close", { key: t.key, error: String(closed.error) });
205 } catch (error) {
206 log.error("sentinel close failed", { key: t.key, error: String(error) });
207 }
208 }
209 }
test/crew-held.test.ts +1 −1
@@ -292,7 +292,7 @@ describe("the tick says why each ready issue cannot start", () => {
292292 app.db.query("UPDATE degraded_span SET closed_at = ?2 WHERE id = ?1").run(span.id, now);
293293 }
294294 const filed = listIssues(app.db, repo.id, { state: "open" } as never).filter((i) => i.title.startsWith("Sentinel: "));
295 const finding = filed.find((i) => i.title.includes("stuck:held-test"))!;
295 const finding = filed.find((i) => i.title.includes("stuck:run: runs of this kind stopped moving") && i.body.includes("One lane is past its budget."))!;
296296 expect(finding).toBeDefined();
297297 expect(finding.assigneeId).toBeNull();
298298 // Only the finding is left to start, so the tick's pick is it.
test/sentinel-causes.test.ts added +259
@@ -0,0 +1,259 @@
1 /** One issue per cause (#14655): the key format, the symptom folding, the 24-hour close. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { quietLogs } from "./helpers/globals.ts";
7 import { addComment } from "../src/actions/issues.ts";
8 import { createApp, type App } from "../src/app.ts";
9 import { createRepo } from "../src/actions/repos.ts";
10 import { getIssueById, listComments, listIssues } from "../src/core/issues.ts";
11 import type { Repo } from "../src/core/repos.ts";
12 import { createRun } from "../src/core/runs.ts";
13 import { createUser, type User } from "../src/core/users.ts";
14 import { cloneFailing, foldByCause, ranOut } from "../src/crew/sentinel_causes.ts";
15 import { findFaults, hunt, sentinelTick } from "../src/crew/sentinel.ts";
16
17 const T = Date.parse("2026-11-20T12:00:00.000Z");
18 const HOUR = 3_600_000;
19 const iso = (ms: number): string => new Date(ms).toISOString();
20 quietLogs();
21
22 let app: App;
23 let repo: Repo;
24 let dir: string;
25 let alice: User;
26
27 beforeAll(async () => {
28 dir = await mkdtemp(join(tmpdir(), "forge-sentinel-causes-"));
29 app = createApp({ dataDir: dir, port: 0 });
30 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
31 if (!a.ok) throw new Error(a.error);
32 const made = await createRepo(app, a.value, { name: "forge", visibility: "private" });
33 if (!made.ok) throw new Error(String(made.error));
34 alice = a.value;
35 repo = made.value;
36 process.env.FORGE_SENTINEL_REPO = "alice/forge";
37 });
38 afterAll(async () => {
39 delete process.env.FORGE_SENTINEL_REPO;
40 app.db.close();
41 await rm(dir, { recursive: true, force: true });
42 });
43
44 /** A killed run and its open vitals span, as `vitals.ts` records one. */
45 function killed(kind: string, minutes: number, at: number): string {
46 const run = createRun(app.db, { repoId: repo.id, kind: kind as "check" });
47 app.db
48 .query("UPDATE run SET state = 'failed', summary = ?2, finished_at = ?3 WHERE id = ?1")
49 .run(run.id, `budget: gate took ${minutes} min, limit 30`, iso(at));
50 app.db
51 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'A run was killed twice.', '/settings/capacity', ?2)")
52 .run(`budget:${run.id}`, at);
53 return run.id;
54 }
55 const open = () => listIssues(app.db, repo.id, { state: "open" });
56 const clearSpans = () => app.db.query("DELETE FROM degraded_span").run();
57
58 describe("a cause is one issue", () => {
59 test("the key is the run kind and what ran out, not the run", () => {
60 clearSpans();
61 const a = killed("check", 41, T - 3 * HOUR);
62 const b = killed("check", 52, T - HOUR);
63 const found = findFaults(app.db, T, repo.id);
64 expect(found.map((f) => f.key)).toEqual(["degraded:budget:check:time"]);
65 const f = found[0]!;
66 expect(f.title).not.toContain(a);
67 expect(f.evidence).toContain("instances: 2");
68 expect(f.evidence).toContain(`latest run: ${b}`);
69 expect(f.evidence).toContain(`latest at: ${iso(T - HOUR)}`);
70 });
71
72 test("a new instance is a comment on the one issue, not a new issue", () => {
73 clearSpans();
74 killed("check", 41, T - 3 * HOUR);
75 expect(hunt(app, "alice/forge", T).filed).toBe(1);
76 killed("check", 60, T - HOUR);
77 const r = hunt(app, "alice/forge", T + 25 * HOUR - 1);
78 expect(r.filed).toBe(0);
79 expect(open().length).toBe(1);
80 expect(listComments(app.db, open()[0]!.id).map((c) => c.body).join("\n")).toContain("instances: 2");
81 });
82 });
83
84 describe("symptoms of an open incident", () => {
85 test("runs killed while the lane is stopped are lines of its issue, not issues", () => {
86 clearSpans();
87 app.db.query("DELETE FROM sentinel_finding").run();
88 for (let i = 0; i < 30; i++) killed("agent", 40 + i, T - HOUR + i * 1000);
89 app.db
90 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES ('queue:agent', 'The agent queue has waited over 30 minutes.', '/settings/capacity', ?1)")
91 .run(T - 2 * HOUR);
92 const found = findFaults(app.db, T, repo.id);
93 const keys = found.map((f) => f.key);
94 expect(keys.filter((k) => k.startsWith("degraded:budget:"))).toEqual([]);
95 const incident = found.find((f) => f.key === "degraded:queue-agent")!;
96 const line = incident.evidence.find((e) => e.startsWith("symptom: "))!;
97 expect(line).toStartWith("symptom: budget:agent:time x30, latest run ");
98 expect(line).toEndWith(` at ${iso(T - HOUR + 29 * 1000)}`);
99 });
100
101 test("with the clone storm open, thirty killed runs are one issue counting them", () => {
102 clearSpans();
103 app.db.query("DELETE FROM sentinel_finding").run();
104 app.db.query("DELETE FROM run").run();
105 for (let i = 0; i < 6; i++) {
106 const run = createRun(app.db, { repoId: repo.id, kind: "agent" });
107 app.db
108 .query("UPDATE run SET state = 'failed', summary = 'clone failed: fatal', finished_at = ?2 WHERE id = ?1")
109 .run(run.id, iso(T - 10 * 60_000 + i * 1000));
110 }
111 for (let i = 0; i < 30; i++) killed("agent", 40 + i, T - HOUR + i * 1000);
112 const found = findFaults(app.db, T, repo.id);
113 const clone = found.filter((f) => f.key === "degraded:clone-failing");
114 expect(clone.length).toBe(1);
115 expect(found.filter((f) => f.key.startsWith("degraded:budget:"))).toEqual([]);
116 expect(clone[0]!.evidence.some((e) => e.startsWith("symptom: budget:agent:time x30"))).toBe(true);
117 });
118 });
119
120 describe("a cause that stops recurring closes its own issue", () => {
121 test("after 24 hours, with the last evidence; not before", () => {
122 clearSpans();
123 app.db.query("DELETE FROM sentinel_finding").run();
124 app.db.query("DELETE FROM run").run();
125 for (const i of open()) app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(i.id);
126 killed("check", 41, T - 2 * HOUR);
127 hunt(app, "alice/forge", T);
128 const [issue] = open();
129 expect(issue).toBeDefined();
130 clearSpans();
131 hunt(app, "alice/forge", T + 23 * HOUR);
132 expect(getIssueById(app.db, issue!.id)!.state).toBe("open");
133 hunt(app, "alice/forge", T + 25 * HOUR);
134 const closed = getIssueById(app.db, issue!.id)!;
135 expect(closed.state).toBe("closed");
136 expect(closed.closeReason).toBe("done");
137 const last = listComments(app.db, issue!.id).at(-1)!.body;
138 expect(last).toContain("has not recurred for 24 hours");
139 expect(last).toContain("instances: 1");
140 });
141 });
142
143 /** A stuck run's span, as `vitals.ts` records one. */
144 function stuck(kind: string, at: number, closedAt?: number): string {
145 const run = createRun(app.db, { repoId: repo.id, kind: kind as "check" });
146 app.db
147 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES (?1, 'Run is stuck: no heartbeat for 4 min (limit 3).', '/x', ?2, ?3)")
148 .run(`stuck:${run.id}`, at, closedAt ?? null);
149 return run.id;
150 }
151 const queueSpan = (closedAt?: number): void => {
152 app.db
153 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('queue:check', 'The check queue has waited over 30 minutes.', '/settings/capacity', ?1, ?2)")
154 .run(T - 2 * HOUR, closedAt ?? null);
155 };
156 const reset = (): void => {
157 clearSpans();
158 app.db.query("DELETE FROM sentinel_finding").run();
159 app.db.query("DELETE FROM run").run();
160 for (const i of open()) app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(i.id);
161 };
162
163 describe("what a kill ran out of", () => {
164 test("time, runs per day, dollars per day, and anything else is other", () => {
165 expect(ranOut("budget: gate took 41 min, limit 30")).toBe("time");
166 expect(ranOut("budget: runs per day, 20 of 20 used")).toBe("runs-per-day");
167 expect(ranOut("budget: dollars per day, $5.00 of $5.00 spent")).toBe("dollars-per-day");
168 expect(ranOut("budget: something new")).toBe("other");
169 expect(ranOut(null)).toBe("other");
170 });
171 });
172
173 describe("a stuck run is keyed by its run kind", () => {
174 test("two stuck runs of one kind are one finding, titled as stuck and not as a budget", () => {
175 reset();
176 stuck("agent", T - 20 * 60_000);
177 stuck("agent", T - 10 * 60_000);
178 const found = findFaults(app.db, T, repo.id).filter((f) => f.key.startsWith("degraded:stuck:"));
179 expect(found.map((f) => f.key)).toEqual(["degraded:stuck:agent"]);
180 expect(found[0]!.title).toBe("stuck:agent: runs of this kind stopped moving");
181 expect(found[0]!.evidence).toContain("instances: 2");
182 });
183 });
184
185 describe("an open queue incident swallows stuck runs; a closed one does not", () => {
186 test("open: the stuck run is a symptom line; closed: it files its own issue", () => {
187 reset();
188 stuck("agent", T - 10 * 60_000);
189 queueSpan();
190 const during = findFaults(app.db, T, repo.id);
191 expect(during.some((f) => f.key === "degraded:stuck:agent")).toBe(false);
192 const incident = during.find((f) => f.key === "degraded:queue-check")!;
193 expect(incident.evidence.some((e) => e.startsWith("symptom: stuck:agent x1"))).toBe(true);
194 reset();
195 stuck("agent", T - 10 * 60_000);
196 queueSpan(T - HOUR);
197 const after = findFaults(app.db, T, repo.id);
198 expect(after.some((f) => f.key === "degraded:stuck:agent")).toBe(true);
199 expect(after.find((f) => f.key === "degraded:queue-check")?.endedAt).toBeDefined();
200 expect(after.some((f) => f.evidence.some((e) => e.startsWith("symptom: ")))).toBe(false);
201 });
202
203 test("an ended incident absorbs nothing, given as plain findings", () => {
204 reset();
205 stuck("agent", T - 10 * 60_000);
206 const [sym] = findFaults(app.db, T, repo.id).filter((f) => f.key === "degraded:stuck:agent");
207 const ended = { ...sym!, key: "degraded:queue-x", cause: "queue:check", endedAt: iso(T), evidence: [] };
208 const out = foldByCause(app.db, [ended, sym!]);
209 expect(out.map((f) => f.key).sort()).toEqual(["degraded:queue-x", "degraded:stuck:agent"]);
210 });
211 });
212
213 describe("the clone storm is an incident only while the breaker holds", () => {
214 test("no failed clones, no finding", () => {
215 reset();
216 expect(cloneFailing(app.db, T)).toEqual([]);
217 });
218 });
219
220 describe("closing a quiet cause", () => {
221 test("a clone-failing issue closes after 24 hours without recurrence", () => {
222 reset();
223 for (let i = 0; i < 6; i++) {
224 const run = createRun(app.db, { repoId: repo.id, kind: "agent" });
225 app.db
226 .query("UPDATE run SET state = 'failed', summary = 'clone failed: fatal', finished_at = ?2 WHERE id = ?1")
227 .run(run.id, iso(T - 10 * 60_000 + i * 1000));
228 }
229 hunt(app, "alice/forge", T);
230 const [issue] = open();
231 expect(issue).toBeDefined();
232 app.db.query("DELETE FROM run").run();
233 hunt(app, "alice/forge", T + 25 * HOUR);
234 expect(getIssueById(app.db, issue!.id)!.state).toBe("closed");
235 });
236
237 test("an issue a person commented on stays open", () => {
238 reset();
239 killed("check", 41, T - 2 * HOUR);
240 hunt(app, "alice/forge", T);
241 const [issue] = open();
242 expect(issue).toBeDefined();
243 const said = addComment(app, alice, repo, issue!, "I am on this one.");
244 expect(said.ok).toBe(true);
245 clearSpans();
246 hunt(app, "alice/forge", T + 25 * HOUR);
247 expect(getIssueById(app.db, issue!.id)!.state).toBe("open");
248 });
249 });
250
251 describe("the tick", () => {
252 test("a new kill of a cause already filed wakes no early hunt", () => {
253 reset();
254 killed("check", 41, T - 2 * HOUR);
255 expect(hunt(app, "alice/forge", T).filed).toBe(1);
256 killed("check", 55, T + 60_000);
257 expect(sentinelTick(app, T + 11 * 60_000)).toBeNull();
258 });
259 });
test/sentinel.test.ts +3
@@ -736,6 +736,9 @@ describe("a budget breach is crew work at once", () => {
736736
737737 test("a breach brings a hunt within a tick, inside the signal gap", () => {
738738 const t = Date.parse("2026-10-14T12:00:00.000Z");
739 // No incident open: a stuck run beside an open queue incident is that incident's symptom
740 // (#14655).
741 bapp.db.query("DELETE FROM degraded_span WHERE cause = 'queue:check'").run();
739742 hunt(bapp, "alice/forge", t);
740743 bapp.db
741744 .query(

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs