Public: anyone can read this; the crew can change it.
Sentinel files one issue per cause, not one per run, and stays quiet about symptoms of an open incident #14665
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14655-sentinel-files-one-issue-per-cause-not-o → main 4 commits fork
6 files
changed +496
−5
docs/SENTINEL.md +14
| @@ -25,6 +25,20 @@ issue, and only when its evidence changed; one whose issue was closed and | ||
| 25 | 25 | returned is filed again. It files in `FORGE_SENTINEL_REPO` (default |
| 26 | 26 | `trevato/forge`) as the `sentinel` agent, else the first admin. |
| 27 | 27 | |
| 28 | **One issue per cause** (#14655, `src/crew/sentinel_causes.ts`). A finding is keyed by its | |
| 29 | cause, not its instance: a run killed for its budget files under | |
| 30 | `degraded:budget:<run kind>:<what ran out>` (e.g. `degraded:budget:review:time`), and a new | |
| 31 | kill adds to that issue's evidence (`instances`, `latest run`, `latest at`) instead of | |
| 32 | opening another. A shared test was already one cause per test and keeps its key. While an | |
| 33 | incident is open (the lane stopped, `clone_failing`, a `queue:*` degraded, a crash loop) the | |
| 34 | budget kills and stuck runs it explains are `symptom:` lines on the incident's issue, not | |
| 35 | issues. A cause-keyed issue not seen for 24 hours closes itself with its last evidence (a | |
| 36 | person's comment keeps it open). One detector is new: `cloneFailing` files | |
| 37 | `degraded:clone-failing` when the clone breaker (`crew/clone_breaker.ts`) has tripped, so the | |
| 38 | failed clones have an incident to be symptoms of. Otherwise detection is unchanged; only filing | |
| 39 | is. A kill is keyed by what ran out: `time` (the forge's "took N min, limit M"), `runs-per-day`, | |
| 40 | `dollars-per-day` (the maintainer's cap), else `other`. | |
| 41 | ||
| 28 | 42 | **How it is labelled** (#8856). Each finding declares a kind, and is filed |
| 29 | 43 | with its labels, so the bug clock (`src/flow/bugs.ts`) runs from the filing |
| 30 | 44 | and nobody has to sort it: |
src/crew/sentinel.ts +10 −4
| @@ -55,6 +55,7 @@ import { type LaneAlarm, laneAlarm } from "../flow/alarm_lane.ts"; | ||
| 55 | 55 | import type { FindingKind } from "../hunt/finding.ts"; |
| 56 | 56 | import { addLabels, kindLabels } from "../hunt/label.ts"; |
| 57 | 57 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; |
| 58 | import { cloneFailing, foldByCause, closeQuiet } from "./sentinel_causes.ts"; | |
| 58 | 59 | import { seenCause } from "./sentinel_seen.ts"; |
| 59 | 60 | import { memoryWatch, type MemoryWatch } from "../perf/memory_watch.ts"; |
| 60 | 61 | |
| @@ -104,6 +105,8 @@ export interface Finding { | ||
| 104 | 105 | readonly cause?: string; |
| 105 | 106 | /** What the detector read, for the issue's evidence block (#13033): path and measurement. */ |
| 106 | 107 | readonly seen?: Evidence; |
| 108 | /** Symptoms of an open incident were added to its evidence (#14655): told at most once a day. */ | |
| 109 | readonly folded?: true; | |
| 107 | 110 | } |
| 108 | 111 | |
| 109 | 112 | const QUOTE_MAX = 200; |
| @@ -621,7 +624,8 @@ export function findFaults( | ||
| 621 | 624 | ledger?: ErrorLedger, |
| 622 | 625 | lane: LaneAlarm = laneAlarm(db, new Date(now), { ceilingUsd }), |
| 623 | 626 | ): Finding[] { |
| 624 | const found = [ | |
| 627 | const found = foldByCause(db, [ | |
| 628 | ...cloneFailing(db, now), | |
| 625 | 629 | ...mainAhead(db, now), |
| 626 | 630 | ...laneStopped(lane), |
| 627 | 631 | ...budgetBreaches(db, now, forgeRepoId, dataDir), |
| @@ -631,7 +635,7 @@ export function findFaults( | ||
| 631 | 635 | ...runStreaks(db), |
| 632 | 636 | ...repeatedErrors(now, ledger), |
| 633 | 637 | ...degradedCauses(db, now, forgeRepoId, dataDir), |
| 634 | ]; | |
| 638 | ]); | |
| 635 | 639 | const keys = new Set<string>(); |
| 636 | 640 | return found |
| 637 | 641 | .filter((f) => !keys.has(f.key) && keys.add(f.key)) |
| @@ -1105,7 +1109,8 @@ export function hunt( | ||
| 1105 | 1109 | // numbers. Told at most once a day, so a cause degraded all day is one issue with a |
| 1106 | 1110 | // comment a day. |
| 1107 | 1111 | const toldAt = Date.parse(tracked.commented_at ?? tracked.first_seen); |
| 1108 | if (tracked.evidence !== evidence && !(f.key.startsWith("degraded:") && now - toldAt < COMMENT_EVERY_MS)) { | |
| 1112 | const toldToday = (f.key.startsWith("degraded:") || f.folded === true) && now - toldAt < COMMENT_EVERY_MS; | |
| 1113 | if (tracked.evidence !== evidence && !toldToday) { | |
| 1109 | 1114 | const issue = getIssueById(app.db, tracked.issue_id); |
| 1110 | 1115 | if (issue === undefined) continue; |
| 1111 | 1116 | const said = addComment(app, actor, repo, issue, `Still there at ${at}, evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`); |
| @@ -1170,6 +1175,7 @@ export function hunt( | ||
| 1170 | 1175 | if (repo !== undefined && actor !== undefined) { |
| 1171 | 1176 | const active = new Set(findings.map((f) => f.key)); |
| 1172 | 1177 | closeHeld(app, repo, actor, active, now); |
| 1178 | closeQuiet(app, repo, actor, active, now); | |
| 1173 | 1179 | closeCaughtUp(app, repo, actor, active, now); |
| 1174 | 1180 | closeLaneMoving(app, repo, actor, active, now, lane); |
| 1175 | 1181 | } |
| @@ -1241,7 +1247,7 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null { | ||
| 1241 | 1247 | // Every tick, before a builder takes it: see `settleShared`. |
| 1242 | 1248 | const actor = sentinelActor(app.db); |
| 1243 | 1249 | if (ownRepo !== undefined && actor !== undefined) settleShared(app, ownRepo, actor, now); |
| 1244 | const breaches = budgetBreaches(app.db, now, ownRepo?.id); | |
| 1250 | const breaches = foldByCause(app.db, budgetBreaches(app.db, now, ownRepo?.id)); | |
| 1245 | 1251 | if (since < SIGNAL_GAP_MS && breaches.length === 0) return null; |
| 1246 | 1252 | if (since < HUNT_EVERY_MS) { |
| 1247 | 1253 | // A breach is not held back by the gap: the crew hears within a tick. |
src/crew/sentinel_causes.ts added +209
| @@ -0,0 +1,209 @@ | ||
| 1 | /** | |
| 2 | * One issue per cause, not per instance (#14655). A budget kill names the run it | |
| 3 | * killed and a shared test names a hash, so each new instance used to be a new | |
| 4 | * issue: 30 for one clone storm. Here a finding's key is its cause (`budget:<run | |
| 5 | * kind>:<what ran out>`), instances of one cause fold into one | |
| 6 | * finding that counts them, and while an incident alarm is open the symptoms it | |
| 7 | * explains are lines of its issue, not issues of their own. Detection is untouched. | |
| 8 | */ | |
| 9 | ||
| 10 | import type { Database } from "bun:sqlite"; | |
| 11 | import type { App } from "../app.ts"; | |
| 12 | import { addComment, setIssueState } from "../actions/issues.ts"; | |
| 13 | import { getIssueById } from "../core/issues.ts"; | |
| 14 | import type { Repo } from "../core/repos.ts"; | |
| 15 | import type { User } from "../core/users.ts"; | |
| 16 | import { createLog } from "../kernel/log.ts"; | |
| 17 | import { cloneBreaker } from "./clone_breaker.ts"; | |
| 18 | import { personSpoke } from "./person_spoke.ts"; | |
| 19 | import type { Finding } from "./sentinel.ts"; | |
| 20 | ||
| 21 | const log = createLog("sentinel"); | |
| 22 | ||
| 23 | /** A cause with no recurrence for this long closes its own issue. */ | |
| 24 | const QUIET_CLOSE_MS = 24 * 3_600_000; | |
| 25 | ||
| 26 | /** The key of the clone storm's issue: the incident its failed runs are symptoms of. */ | |
| 27 | const CLONE_KEY = "degraded:clone-failing"; | |
| 28 | ||
| 29 | /** Key families that name a cause, not an instance: the ones that close when quiet. */ | |
| 30 | const CAUSE_KEY_PREFIXES: readonly string[] = [ | |
| 31 | "degraded:budget:", | |
| 32 | "degraded:stuck:", | |
| 33 | CLONE_KEY, | |
| 34 | ]; | |
| 35 | ||
| 36 | const part = (text: string): string => text.toLowerCase().replace(/[^a-z0-9._/-]+/g, "-").slice(0, 60); | |
| 37 | ||
| 38 | interface RunRow { | |
| 39 | readonly kind: string; | |
| 40 | readonly summary: string | null; | |
| 41 | readonly finished_at: string | null; | |
| 42 | } | |
| 43 | ||
| 44 | const runOf = (db: Database, id: string): RunRow | null => | |
| 45 | db.query<RunRow, [string]>("SELECT kind, summary, finished_at FROM run WHERE id = ?1").get(id); | |
| 46 | ||
| 47 | /** | |
| 48 | * What a budget kill ran out of. The kills the forge writes say "budget: <what> took N min, limit | |
| 49 | * M" (`budgetSummary`: time), and the maintainer's cap says "budget: runs per day, ..." or | |
| 50 | * "budget: dollars per day, ..." (`maintain/budget.ts`). Anything else is "other". | |
| 51 | */ | |
| 52 | export function ranOut(summary: string | null): string { | |
| 53 | const text = summary ?? ""; | |
| 54 | if (/took \d+ min, limit \d+/.test(text)) return "time"; | |
| 55 | if (text.startsWith("budget: runs per day")) return "runs-per-day"; | |
| 56 | if (text.startsWith("budget: dollars per day")) return "dollars-per-day"; | |
| 57 | return "other"; | |
| 58 | } | |
| 59 | ||
| 60 | /** | |
| 61 | * The key a finding of this cause is filed under: the cause, not the run or hash that showed it. | |
| 62 | */ | |
| 63 | export function causeKey(db: Database, f: Finding): string { | |
| 64 | const cause = f.cause; | |
| 65 | if (cause === undefined) return f.key; | |
| 66 | const run = /^(budget|stuck):(.+)$/.exec(cause); | |
| 67 | if (run !== null) { | |
| 68 | const row = runOf(db, run[2] ?? ""); | |
| 69 | const kind = part(row?.kind ?? "run"); | |
| 70 | return run[1] === "budget" ? `degraded:budget:${kind}:${ranOut(row?.summary ?? null)}` : `degraded:stuck:${kind}`; | |
| 71 | } | |
| 72 | return f.key; | |
| 73 | } | |
| 74 | ||
| 75 | /** | |
| 76 | * The clone storm as a finding, when the breaker has tripped: the incident its dead runs belong to. | |
| 77 | */ | |
| 78 | export function cloneFailing(db: Database, now: number): Finding[] { | |
| 79 | const c = cloneBreaker(db, now); | |
| 80 | if (!c.fired) return []; | |
| 81 | return [{ | |
| 82 | key: CLONE_KEY, | |
| 83 | title: "Crew runs are dying in their clone", | |
| 84 | evidence: [`says: ${c.says}`, `since: ${c.since}`, "href: /boards"], | |
| 85 | pillar: "P0 Trust: the forge says when it is not well, and is well again", | |
| 86 | kind: "defect", | |
| 87 | lookFor: "A crew run gets past its clone, and the last runs do not die cloning.", | |
| 88 | urgent: true, | |
| 89 | cause: "clone_failing", | |
| 90 | }]; | |
| 91 | } | |
| 92 | ||
| 93 | /** | |
| 94 | * Whether a finding is an incident alarm that is open now: the lane, the clone, a queue or the | |
| 95 | * engine. | |
| 96 | */ | |
| 97 | function isIncident(f: Finding): boolean { | |
| 98 | if (f.endedAt !== undefined) return false; | |
| 99 | return f.key === "lane-stopped" || f.key === CLONE_KEY || /^(queue:|crash-loop$)/.test(f.cause ?? ""); | |
| 100 | } | |
| 101 | ||
| 102 | /** A symptom of an incident: a run killed for its budget, or one that stopped moving. */ | |
| 103 | const isSymptom = (f: Finding): boolean => /^(budget|stuck):/.test(f.cause ?? ""); | |
| 104 | ||
| 105 | /** The newest instance of a group: the one whose run finished last, else the last read. */ | |
| 106 | function latestOf(db: Database, group: readonly Finding[]): { f: Finding; at: string | undefined } { | |
| 107 | let best = { f: group[group.length - 1]!, at: undefined as string | undefined }; | |
| 108 | for (const f of group) { | |
| 109 | const id = /^(?:budget|stuck):(.+)$/.exec(f.cause ?? "")?.[1]; | |
| 110 | const at = id === undefined ? undefined : runOf(db, id)?.finished_at ?? undefined; | |
| 111 | if (at !== undefined && (best.at === undefined || at >= best.at)) best = { f, at }; | |
| 112 | } | |
| 113 | return best; | |
| 114 | } | |
| 115 | ||
| 116 | /** One line a cause's instances add to its issue: how many, the latest run, the latest time. */ | |
| 117 | function instancesLine(db: Database, group: readonly Finding[]): string[] { | |
| 118 | const { f, at } = latestOf(db, group); | |
| 119 | const run = /^(?:budget|stuck):(.+)$/.exec(f.cause ?? "")?.[1]; | |
| 120 | return [ | |
| 121 | `instances: ${group.length}`, | |
| 122 | ...(run === undefined ? [] : [`latest run: ${run}`]), | |
| 123 | ...(at === undefined ? [] : [`latest at: ${at}`]), | |
| 124 | ]; | |
| 125 | } | |
| 126 | ||
| 127 | /** | |
| 128 | * Key each finding by its cause, fold the instances of one cause into one finding | |
| 129 | * (the newest instance's evidence, plus its count), then fold the symptoms of an | |
| 130 | * open incident into that incident's finding. Order is kept; the result has one | |
| 131 | * finding per key. | |
| 132 | */ | |
| 133 | export function foldByCause(db: Database, found: readonly Finding[]): Finding[] { | |
| 134 | const groups = new Map<string, Finding[]>(); | |
| 135 | for (const f of found) { | |
| 136 | const key = causeKey(db, f); | |
| 137 | const list = groups.get(key); | |
| 138 | if (list === undefined) groups.set(key, [{ ...f, key }]); | |
| 139 | // The same instance read by two detectors (a breach and the day's total) counts once. | |
| 140 | else if (f.cause === undefined || !list.some((g) => g.cause === f.cause)) list.push({ ...f, key }); | |
| 141 | } | |
| 142 | const folded: Finding[] = []; | |
| 143 | for (const [key, group] of groups) { | |
| 144 | // A group without a run cause (a shared test, a lane) keeps its one finding as it is. | |
| 145 | if (!/^degraded:(budget|stuck):/.test(key)) { | |
| 146 | folded.push(group[0]!); | |
| 147 | continue; | |
| 148 | } | |
| 149 | const { f } = latestOf(db, group); | |
| 150 | const name = key.replace(/^degraded:/, ""); | |
| 151 | folded.push({ | |
| 152 | ...f, | |
| 153 | key, | |
| 154 | title: key.startsWith("degraded:stuck:") | |
| 155 | ? `${name}: runs of this kind stopped moving` | |
| 156 | : `${name} is past its budget: runs of this kind were stopped by it`, | |
| 157 | evidence: [...f.evidence, ...instancesLine(db, group)], | |
| 158 | }); | |
| 159 | } | |
| 160 | const incident = folded.find(isIncident); | |
| 161 | if (incident === undefined) return folded; | |
| 162 | const symptoms = folded.filter(isSymptom); | |
| 163 | if (symptoms.length === 0) return folded; | |
| 164 | const lines = symptoms.map((s) => { | |
| 165 | const n = s.evidence.find((e) => e.startsWith("instances: "))?.slice(11) ?? "1"; | |
| 166 | const latest = s.evidence.find((e) => e.startsWith("latest run: "))?.slice(12); | |
| 167 | const at = s.evidence.find((e) => e.startsWith("latest at: "))?.slice(11); | |
| 168 | return `symptom: ${s.key.replace(/^degraded:/, "")} x${n}${latest === undefined ? "" : `, latest run ${latest}`}${at === undefined ? "" : ` at ${at}`}`; | |
| 169 | }); | |
| 170 | return folded | |
| 171 | .filter((f) => !isSymptom(f)) | |
| 172 | .map((f) => (f === incident ? { ...f, evidence: [...f.evidence, ...lines], folded: true as const } : f)); | |
| 173 | } | |
| 174 | ||
| 175 | /** | |
| 176 | * A cause-keyed issue whose cause has not been seen for 24 hours closes itself with the | |
| 177 | * last evidence it held, so nothing lingers. `active` are this hunt's keys. One a person | |
| 178 | * spoke on stays: they decide. | |
| 179 | */ | |
| 180 | export function closeQuiet(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void { | |
| 181 | const open = app.db | |
| 182 | .query<{ key: string; issue_id: string; evidence: string; last_seen: string }, []>( | |
| 183 | `SELECT t.key, t.issue_id, t.evidence, t.last_seen FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id WHERE i.state = 'open'`, | |
| 184 | ) | |
| 185 | .all(); | |
| 186 | for (const t of open) { | |
| 187 | try { | |
| 188 | if (active.has(t.key) || !CAUSE_KEY_PREFIXES.some((p) => t.key.startsWith(p))) continue; | |
| 189 | if (!(now - Date.parse(t.last_seen) >= QUIET_CLOSE_MS) || personSpoke(app.db, t.issue_id)) continue; | |
| 190 | const issue = getIssueById(app.db, t.issue_id); | |
| 191 | if (issue === undefined || issue.repoId !== repo.id) continue; | |
| 192 | const said = addComment( | |
| 193 | app, | |
| 194 | actor, | |
| 195 | repo, | |
| 196 | issue, | |
| 197 | `Not seen since ${t.last_seen}: this cause has not recurred for 24 hours. Last evidence:\n${t.evidence.split("\n").map((e) => `- ${e}`).join("\n")}\nClosing as done; if it comes back it is filed again.`, | |
| 198 | ); | |
| 199 | if (!said.ok) { | |
| 200 | log.warn("sentinel could not comment", { key: t.key, error: String(said.error) }); | |
| 201 | continue; | |
| 202 | } | |
| 203 | const closed = setIssueState(app, actor, repo, issue, "closed", { reason: "done", system: true }); | |
| 204 | if (!closed.ok) log.warn("sentinel could not close", { key: t.key, error: String(closed.error) }); | |
| 205 | } catch (error) { | |
| 206 | log.error("sentinel close failed", { key: t.key, error: String(error) }); | |
| 207 | } | |
| 208 | } | |
| 209 | } |
test/crew-held.test.ts +1 −1
| @@ -292,7 +292,7 @@ describe("the tick says why each ready issue cannot start", () => { | ||
| 292 | 292 | app.db.query("UPDATE degraded_span SET closed_at = ?2 WHERE id = ?1").run(span.id, now); |
| 293 | 293 | } |
| 294 | 294 | const filed = listIssues(app.db, repo.id, { state: "open" } as never).filter((i) => i.title.startsWith("Sentinel: ")); |
| 295 | const finding = filed.find((i) => i.title.includes("stuck:held-test"))!; | |
| 295 | const finding = filed.find((i) => i.title.includes("stuck:run: runs of this kind stopped moving") && i.body.includes("One lane is past its budget."))!; | |
| 296 | 296 | expect(finding).toBeDefined(); |
| 297 | 297 | expect(finding.assigneeId).toBeNull(); |
| 298 | 298 | // Only the finding is left to start, so the tick's pick is it. |
test/sentinel-causes.test.ts added +259
| @@ -0,0 +1,259 @@ | ||
| 1 | /** One issue per cause (#14655): the key format, the symptom folding, the 24-hour close. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { quietLogs } from "./helpers/globals.ts"; | |
| 7 | import { addComment } from "../src/actions/issues.ts"; | |
| 8 | import { createApp, type App } from "../src/app.ts"; | |
| 9 | import { createRepo } from "../src/actions/repos.ts"; | |
| 10 | import { getIssueById, listComments, listIssues } from "../src/core/issues.ts"; | |
| 11 | import type { Repo } from "../src/core/repos.ts"; | |
| 12 | import { createRun } from "../src/core/runs.ts"; | |
| 13 | import { createUser, type User } from "../src/core/users.ts"; | |
| 14 | import { cloneFailing, foldByCause, ranOut } from "../src/crew/sentinel_causes.ts"; | |
| 15 | import { findFaults, hunt, sentinelTick } from "../src/crew/sentinel.ts"; | |
| 16 | ||
| 17 | const T = Date.parse("2026-11-20T12:00:00.000Z"); | |
| 18 | const HOUR = 3_600_000; | |
| 19 | const iso = (ms: number): string => new Date(ms).toISOString(); | |
| 20 | quietLogs(); | |
| 21 | ||
| 22 | let app: App; | |
| 23 | let repo: Repo; | |
| 24 | let dir: string; | |
| 25 | let alice: User; | |
| 26 | ||
| 27 | beforeAll(async () => { | |
| 28 | dir = await mkdtemp(join(tmpdir(), "forge-sentinel-causes-")); | |
| 29 | app = createApp({ dataDir: dir, port: 0 }); | |
| 30 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 31 | if (!a.ok) throw new Error(a.error); | |
| 32 | const made = await createRepo(app, a.value, { name: "forge", visibility: "private" }); | |
| 33 | if (!made.ok) throw new Error(String(made.error)); | |
| 34 | alice = a.value; | |
| 35 | repo = made.value; | |
| 36 | process.env.FORGE_SENTINEL_REPO = "alice/forge"; | |
| 37 | }); | |
| 38 | afterAll(async () => { | |
| 39 | delete process.env.FORGE_SENTINEL_REPO; | |
| 40 | app.db.close(); | |
| 41 | await rm(dir, { recursive: true, force: true }); | |
| 42 | }); | |
| 43 | ||
| 44 | /** A killed run and its open vitals span, as `vitals.ts` records one. */ | |
| 45 | function killed(kind: string, minutes: number, at: number): string { | |
| 46 | const run = createRun(app.db, { repoId: repo.id, kind: kind as "check" }); | |
| 47 | app.db | |
| 48 | .query("UPDATE run SET state = 'failed', summary = ?2, finished_at = ?3 WHERE id = ?1") | |
| 49 | .run(run.id, `budget: gate took ${minutes} min, limit 30`, iso(at)); | |
| 50 | app.db | |
| 51 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'A run was killed twice.', '/settings/capacity', ?2)") | |
| 52 | .run(`budget:${run.id}`, at); | |
| 53 | return run.id; | |
| 54 | } | |
| 55 | const open = () => listIssues(app.db, repo.id, { state: "open" }); | |
| 56 | const clearSpans = () => app.db.query("DELETE FROM degraded_span").run(); | |
| 57 | ||
| 58 | describe("a cause is one issue", () => { | |
| 59 | test("the key is the run kind and what ran out, not the run", () => { | |
| 60 | clearSpans(); | |
| 61 | const a = killed("check", 41, T - 3 * HOUR); | |
| 62 | const b = killed("check", 52, T - HOUR); | |
| 63 | const found = findFaults(app.db, T, repo.id); | |
| 64 | expect(found.map((f) => f.key)).toEqual(["degraded:budget:check:time"]); | |
| 65 | const f = found[0]!; | |
| 66 | expect(f.title).not.toContain(a); | |
| 67 | expect(f.evidence).toContain("instances: 2"); | |
| 68 | expect(f.evidence).toContain(`latest run: ${b}`); | |
| 69 | expect(f.evidence).toContain(`latest at: ${iso(T - HOUR)}`); | |
| 70 | }); | |
| 71 | ||
| 72 | test("a new instance is a comment on the one issue, not a new issue", () => { | |
| 73 | clearSpans(); | |
| 74 | killed("check", 41, T - 3 * HOUR); | |
| 75 | expect(hunt(app, "alice/forge", T).filed).toBe(1); | |
| 76 | killed("check", 60, T - HOUR); | |
| 77 | const r = hunt(app, "alice/forge", T + 25 * HOUR - 1); | |
| 78 | expect(r.filed).toBe(0); | |
| 79 | expect(open().length).toBe(1); | |
| 80 | expect(listComments(app.db, open()[0]!.id).map((c) => c.body).join("\n")).toContain("instances: 2"); | |
| 81 | }); | |
| 82 | }); | |
| 83 | ||
| 84 | describe("symptoms of an open incident", () => { | |
| 85 | test("runs killed while the lane is stopped are lines of its issue, not issues", () => { | |
| 86 | clearSpans(); | |
| 87 | app.db.query("DELETE FROM sentinel_finding").run(); | |
| 88 | for (let i = 0; i < 30; i++) killed("agent", 40 + i, T - HOUR + i * 1000); | |
| 89 | app.db | |
| 90 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES ('queue:agent', 'The agent queue has waited over 30 minutes.', '/settings/capacity', ?1)") | |
| 91 | .run(T - 2 * HOUR); | |
| 92 | const found = findFaults(app.db, T, repo.id); | |
| 93 | const keys = found.map((f) => f.key); | |
| 94 | expect(keys.filter((k) => k.startsWith("degraded:budget:"))).toEqual([]); | |
| 95 | const incident = found.find((f) => f.key === "degraded:queue-agent")!; | |
| 96 | const line = incident.evidence.find((e) => e.startsWith("symptom: "))!; | |
| 97 | expect(line).toStartWith("symptom: budget:agent:time x30, latest run "); | |
| 98 | expect(line).toEndWith(` at ${iso(T - HOUR + 29 * 1000)}`); | |
| 99 | }); | |
| 100 | ||
| 101 | test("with the clone storm open, thirty killed runs are one issue counting them", () => { | |
| 102 | clearSpans(); | |
| 103 | app.db.query("DELETE FROM sentinel_finding").run(); | |
| 104 | app.db.query("DELETE FROM run").run(); | |
| 105 | for (let i = 0; i < 6; i++) { | |
| 106 | const run = createRun(app.db, { repoId: repo.id, kind: "agent" }); | |
| 107 | app.db | |
| 108 | .query("UPDATE run SET state = 'failed', summary = 'clone failed: fatal', finished_at = ?2 WHERE id = ?1") | |
| 109 | .run(run.id, iso(T - 10 * 60_000 + i * 1000)); | |
| 110 | } | |
| 111 | for (let i = 0; i < 30; i++) killed("agent", 40 + i, T - HOUR + i * 1000); | |
| 112 | const found = findFaults(app.db, T, repo.id); | |
| 113 | const clone = found.filter((f) => f.key === "degraded:clone-failing"); | |
| 114 | expect(clone.length).toBe(1); | |
| 115 | expect(found.filter((f) => f.key.startsWith("degraded:budget:"))).toEqual([]); | |
| 116 | expect(clone[0]!.evidence.some((e) => e.startsWith("symptom: budget:agent:time x30"))).toBe(true); | |
| 117 | }); | |
| 118 | }); | |
| 119 | ||
| 120 | describe("a cause that stops recurring closes its own issue", () => { | |
| 121 | test("after 24 hours, with the last evidence; not before", () => { | |
| 122 | clearSpans(); | |
| 123 | app.db.query("DELETE FROM sentinel_finding").run(); | |
| 124 | app.db.query("DELETE FROM run").run(); | |
| 125 | for (const i of open()) app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(i.id); | |
| 126 | killed("check", 41, T - 2 * HOUR); | |
| 127 | hunt(app, "alice/forge", T); | |
| 128 | const [issue] = open(); | |
| 129 | expect(issue).toBeDefined(); | |
| 130 | clearSpans(); | |
| 131 | hunt(app, "alice/forge", T + 23 * HOUR); | |
| 132 | expect(getIssueById(app.db, issue!.id)!.state).toBe("open"); | |
| 133 | hunt(app, "alice/forge", T + 25 * HOUR); | |
| 134 | const closed = getIssueById(app.db, issue!.id)!; | |
| 135 | expect(closed.state).toBe("closed"); | |
| 136 | expect(closed.closeReason).toBe("done"); | |
| 137 | const last = listComments(app.db, issue!.id).at(-1)!.body; | |
| 138 | expect(last).toContain("has not recurred for 24 hours"); | |
| 139 | expect(last).toContain("instances: 1"); | |
| 140 | }); | |
| 141 | }); | |
| 142 | ||
| 143 | /** A stuck run's span, as `vitals.ts` records one. */ | |
| 144 | function stuck(kind: string, at: number, closedAt?: number): string { | |
| 145 | const run = createRun(app.db, { repoId: repo.id, kind: kind as "check" }); | |
| 146 | app.db | |
| 147 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES (?1, 'Run is stuck: no heartbeat for 4 min (limit 3).', '/x', ?2, ?3)") | |
| 148 | .run(`stuck:${run.id}`, at, closedAt ?? null); | |
| 149 | return run.id; | |
| 150 | } | |
| 151 | const queueSpan = (closedAt?: number): void => { | |
| 152 | app.db | |
| 153 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('queue:check', 'The check queue has waited over 30 minutes.', '/settings/capacity', ?1, ?2)") | |
| 154 | .run(T - 2 * HOUR, closedAt ?? null); | |
| 155 | }; | |
| 156 | const reset = (): void => { | |
| 157 | clearSpans(); | |
| 158 | app.db.query("DELETE FROM sentinel_finding").run(); | |
| 159 | app.db.query("DELETE FROM run").run(); | |
| 160 | for (const i of open()) app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(i.id); | |
| 161 | }; | |
| 162 | ||
| 163 | describe("what a kill ran out of", () => { | |
| 164 | test("time, runs per day, dollars per day, and anything else is other", () => { | |
| 165 | expect(ranOut("budget: gate took 41 min, limit 30")).toBe("time"); | |
| 166 | expect(ranOut("budget: runs per day, 20 of 20 used")).toBe("runs-per-day"); | |
| 167 | expect(ranOut("budget: dollars per day, $5.00 of $5.00 spent")).toBe("dollars-per-day"); | |
| 168 | expect(ranOut("budget: something new")).toBe("other"); | |
| 169 | expect(ranOut(null)).toBe("other"); | |
| 170 | }); | |
| 171 | }); | |
| 172 | ||
| 173 | describe("a stuck run is keyed by its run kind", () => { | |
| 174 | test("two stuck runs of one kind are one finding, titled as stuck and not as a budget", () => { | |
| 175 | reset(); | |
| 176 | stuck("agent", T - 20 * 60_000); | |
| 177 | stuck("agent", T - 10 * 60_000); | |
| 178 | const found = findFaults(app.db, T, repo.id).filter((f) => f.key.startsWith("degraded:stuck:")); | |
| 179 | expect(found.map((f) => f.key)).toEqual(["degraded:stuck:agent"]); | |
| 180 | expect(found[0]!.title).toBe("stuck:agent: runs of this kind stopped moving"); | |
| 181 | expect(found[0]!.evidence).toContain("instances: 2"); | |
| 182 | }); | |
| 183 | }); | |
| 184 | ||
| 185 | describe("an open queue incident swallows stuck runs; a closed one does not", () => { | |
| 186 | test("open: the stuck run is a symptom line; closed: it files its own issue", () => { | |
| 187 | reset(); | |
| 188 | stuck("agent", T - 10 * 60_000); | |
| 189 | queueSpan(); | |
| 190 | const during = findFaults(app.db, T, repo.id); | |
| 191 | expect(during.some((f) => f.key === "degraded:stuck:agent")).toBe(false); | |
| 192 | const incident = during.find((f) => f.key === "degraded:queue-check")!; | |
| 193 | expect(incident.evidence.some((e) => e.startsWith("symptom: stuck:agent x1"))).toBe(true); | |
| 194 | reset(); | |
| 195 | stuck("agent", T - 10 * 60_000); | |
| 196 | queueSpan(T - HOUR); | |
| 197 | const after = findFaults(app.db, T, repo.id); | |
| 198 | expect(after.some((f) => f.key === "degraded:stuck:agent")).toBe(true); | |
| 199 | expect(after.find((f) => f.key === "degraded:queue-check")?.endedAt).toBeDefined(); | |
| 200 | expect(after.some((f) => f.evidence.some((e) => e.startsWith("symptom: ")))).toBe(false); | |
| 201 | }); | |
| 202 | ||
| 203 | test("an ended incident absorbs nothing, given as plain findings", () => { | |
| 204 | reset(); | |
| 205 | stuck("agent", T - 10 * 60_000); | |
| 206 | const [sym] = findFaults(app.db, T, repo.id).filter((f) => f.key === "degraded:stuck:agent"); | |
| 207 | const ended = { ...sym!, key: "degraded:queue-x", cause: "queue:check", endedAt: iso(T), evidence: [] }; | |
| 208 | const out = foldByCause(app.db, [ended, sym!]); | |
| 209 | expect(out.map((f) => f.key).sort()).toEqual(["degraded:queue-x", "degraded:stuck:agent"]); | |
| 210 | }); | |
| 211 | }); | |
| 212 | ||
| 213 | describe("the clone storm is an incident only while the breaker holds", () => { | |
| 214 | test("no failed clones, no finding", () => { | |
| 215 | reset(); | |
| 216 | expect(cloneFailing(app.db, T)).toEqual([]); | |
| 217 | }); | |
| 218 | }); | |
| 219 | ||
| 220 | describe("closing a quiet cause", () => { | |
| 221 | test("a clone-failing issue closes after 24 hours without recurrence", () => { | |
| 222 | reset(); | |
| 223 | for (let i = 0; i < 6; i++) { | |
| 224 | const run = createRun(app.db, { repoId: repo.id, kind: "agent" }); | |
| 225 | app.db | |
| 226 | .query("UPDATE run SET state = 'failed', summary = 'clone failed: fatal', finished_at = ?2 WHERE id = ?1") | |
| 227 | .run(run.id, iso(T - 10 * 60_000 + i * 1000)); | |
| 228 | } | |
| 229 | hunt(app, "alice/forge", T); | |
| 230 | const [issue] = open(); | |
| 231 | expect(issue).toBeDefined(); | |
| 232 | app.db.query("DELETE FROM run").run(); | |
| 233 | hunt(app, "alice/forge", T + 25 * HOUR); | |
| 234 | expect(getIssueById(app.db, issue!.id)!.state).toBe("closed"); | |
| 235 | }); | |
| 236 | ||
| 237 | test("an issue a person commented on stays open", () => { | |
| 238 | reset(); | |
| 239 | killed("check", 41, T - 2 * HOUR); | |
| 240 | hunt(app, "alice/forge", T); | |
| 241 | const [issue] = open(); | |
| 242 | expect(issue).toBeDefined(); | |
| 243 | const said = addComment(app, alice, repo, issue!, "I am on this one."); | |
| 244 | expect(said.ok).toBe(true); | |
| 245 | clearSpans(); | |
| 246 | hunt(app, "alice/forge", T + 25 * HOUR); | |
| 247 | expect(getIssueById(app.db, issue!.id)!.state).toBe("open"); | |
| 248 | }); | |
| 249 | }); | |
| 250 | ||
| 251 | describe("the tick", () => { | |
| 252 | test("a new kill of a cause already filed wakes no early hunt", () => { | |
| 253 | reset(); | |
| 254 | killed("check", 41, T - 2 * HOUR); | |
| 255 | expect(hunt(app, "alice/forge", T).filed).toBe(1); | |
| 256 | killed("check", 55, T + 60_000); | |
| 257 | expect(sentinelTick(app, T + 11 * 60_000)).toBeNull(); | |
| 258 | }); | |
| 259 | }); |
test/sentinel.test.ts +3
| @@ -736,6 +736,9 @@ describe("a budget breach is crew work at once", () => { | ||
| 736 | 736 | |
| 737 | 737 | test("a breach brings a hunt within a tick, inside the signal gap", () => { |
| 738 | 738 | const t = Date.parse("2026-10-14T12:00:00.000Z"); |
| 739 | // No incident open: a stuck run beside an open queue incident is that incident's symptom | |
| 740 | // (#14655). | |
| 741 | bapp.db.query("DELETE FROM degraded_span WHERE cause = 'queue:check'").run(); | |
| 739 | 742 | hunt(bapp, "alice/forge", t); |
| 740 | 743 | bapp.db |
| 741 | 744 | .query( |
new above
offline