Public: anyone can read this; the crew can change it.
Sentinel: queue:agent was degraded 14 minutes today: The agent queue has waited over 30 minutes. #7191
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/7160-sentinel-queue-agent-was-degraded-14-min → main 2 commits fork 4 threads · 4 unresolved
2 files
changed +30
−1
src/kernel/vitals.ts +18 −1
| @@ -259,6 +259,18 @@ export interface LaneRun { | ||
| 259 | 259 | readonly since: string; |
| 260 | 260 | } |
| 261 | 261 | |
| 262 | ||
| 263 | /** Agent slots are held per issue (`pump` in crew/dispatch.ts); any other kind holds one per run. */ | |
| 264 | function holders(issues: (string | null)[], perIssue: boolean): number { | |
| 265 | return perIssue ? new Set(issues).size : issues.length; | |
| 266 | } | |
| 267 | ||
| 268 | /** What a long queue is waiting behind: how many wait, how many run, and the slots in force. */ | |
| 269 | function queueSentence(kind: string, queued: number, running: number, slots: number | null): string { | |
| 270 | const behind = slots === null ? `${running} running` : `${running} of ${slots} slots running`; | |
| 271 | return `The ${kind} queue has waited over ${QUEUE_WAIT_MAX_MS / 60_000} minutes: ${queued} queued, ${behind}.`; | |
| 272 | } | |
| 273 | ||
| 262 | 274 | /** |
| 263 | 275 | * The lane's one sentence, with its cause: how many wait, since when, and |
| 264 | 276 | * what runs. With one slot: "22 checks wait behind one lane; the oldest since |
| @@ -373,7 +385,12 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 373 | 385 | if (waited > QUEUE_WAIT_MAX_MS) |
| 374 | 386 | need( |
| 375 | 387 | `queue:${row.kind}`, |
| 376 | `The ${row.kind} queue has waited over ${QUEUE_WAIT_MAX_MS / 60_000} minutes.`, | |
| 388 | queueSentence( | |
Use capacityCount(input.db, "crew.concurrency", <boot default>) like pump, so an invalid stored value cannot print NaN. |
||
| 389 | row.kind, | |
| 390 | row.queued, | |
| 391 | holders(running.filter((r) => r.run.kind === row.kind).map((r) => r.run.issueId), row.kind === "agent"), | |
| 392 | row.kind === "agent" ? Number(capacityValue(input.db, "crew.concurrency")) : null, | |
| 393 | ), | |
| 377 | 394 | new Date(row.oldest).toISOString(), |
| 378 | 395 | "/settings/capacity", |
| 379 | 396 | ); |
test/vitals.test.ts +12
| @@ -324,6 +324,18 @@ test("degraded names each cause, stable, with when it began", async () => { | ||
| 324 | 324 | const q = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: later }).degraded.find((d) => d.cause === "queue:agent"); |
| 325 | 325 | expect(q?.since).toBe(new Date(oldest).toISOString()); |
| 326 | 326 | expect(q?.href).toBe("/settings/capacity"); |
| 327 | app.db.query("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('crew.concurrency', '3', 'test', ?1)").run(new Date().toISOString()); | |
outdated
|
||
| 328 | const busy = createRun(app.db, { repoId, kind: "agent" }); | |
| 329 | app.db.query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?2 WHERE id=?1").run(busy.id, new Date(later).toISOString()); | |
| 330 | const held = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: later }).degraded.find((d) => d.cause === "queue:agent"); | |
| 331 | expect(held?.sentence).toBe("The agent queue has waited over 30 minutes: 1 queued, 1 of 3 slots running."); | |
| 332 | app.db.query("DELETE FROM run").run(); | |
| 333 | app.db.query("DELETE FROM forge_setting WHERE key='crew.concurrency'").run(); | |
| 334 | createRun(app.db, { repoId, kind: "agent" }); | |
| 335 | createRun(app.db, { repoId, kind: "deploy", env: "production" }); | |
| 336 | const d = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: later }).degraded.find((x) => x.cause === "queue:deploy"); | |
| 337 | expect(d?.sentence).toBe("The deploy queue has waited over 30 minutes: 1 queued, 0 running."); | |
| 338 | app.db.query("DELETE FROM run WHERE kind='deploy'").run(); | |
| 327 | 339 | |
| 328 | 340 | // stuck: one per run, keyed by run id, since is when it last moved. |
| 329 | 341 | const run = createRun(app.db, { repoId, kind: "deploy", env: "production" }); |
new above
offline
The dispatcher compares
crew.concurrencywith the number of distinct issues among running agent runs (pumpin crew/dispatch.ts). Count the same thing here, so the sentence says exactly what holds the slots.