Public: anyone can read this; the crew can change it.
Sentinel: queue:agent was degraded 18 minutes today: The agent queue has waited over 30 minutes: 16 queued, 12 of 12 slots running; held on purpose: 3 by its own issue's ru… #14647
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14597-sentinel-queue-agent-was-degraded-18-min → main 1 commit fork
4 files
changed +82
−15
src/core/runs.ts +11 −4
| @@ -1746,7 +1746,7 @@ export function queueDepth( | ||
| 1746 | 1746 | ): { kind: RunKind; queued: number; oldest: string }[] { |
| 1747 | 1747 | return db |
| 1748 | 1748 | .query<{ kind: RunKind; queued: number; oldest: string }, [string]>( |
| 1749 | `SELECT kind, COUNT(*) AS queued, MIN(created_at) AS oldest | |
| 1749 | `SELECT kind, COUNT(*) AS queued, MIN(MAX(created_at, COALESCE(not_before, created_at))) AS oldest | |
| 1750 | 1750 | FROM run |
| 1751 | 1751 | WHERE state = 'queued' AND (not_before IS NULL OR not_before <= ?1) |
| 1752 | 1752 | GROUP BY kind`, |
| @@ -1760,12 +1760,16 @@ export function queueDepth( | ||
| 1760 | 1760 | * count and oldest of the rest. Two queries, however deep the queue, both on |
| 1761 | 1761 | * `run_queue`: the `state IN` repeats the partial index's own condition so |
| 1762 | 1762 | * SQLite may use it, and `+kind` keeps it off the per-repository agent index. |
| 1763 | * A run waits for a slot from the latest of when it was queued, its | |
| 1764 | * `not_before`, and `lastHeld` (run id → when a pass last held it, #14597): | |
| 1765 | * a run just released from a hold has not waited the hours it was held. | |
| 1763 | 1766 | */ |
| 1764 | 1767 | export function queuedApart( |
| 1765 | 1768 | db: Database, |
| 1766 | 1769 | kind: RunKind, |
| 1767 | 1770 | heldIds: readonly string[], |
| 1768 | 1771 | now = Date.now(), |
| 1772 | lastHeld: ReadonlyMap<string, string> = new Map(), | |
| 1769 | 1773 | ): { held: string[]; waiting: number; oldest: string | null } { |
| 1770 | 1774 | const ids = JSON.stringify(heldIds); |
| 1771 | 1775 | const at = new Date(now).toISOString(); |
| @@ -1777,11 +1781,14 @@ export function queuedApart( | ||
| 1777 | 1781 | .all(kind, at, ids) |
| 1778 | 1782 | .map((r) => r.id); |
| 1779 | 1783 | const rest = db |
| 1780 | .query<{ waiting: number; oldest: string | null }, [RunKind, string, string]>( | |
| 1781 | `SELECT COUNT(*) AS waiting, MIN(created_at) AS oldest FROM run | |
| 1784 | .query<{ waiting: number; oldest: string | null }, [RunKind, string, string, string]>( | |
| 1785 | `SELECT COUNT(*) AS waiting, | |
| 1786 | MIN(MAX(created_at, COALESCE(not_before, created_at), | |
| 1787 | COALESCE((SELECT value FROM json_each(?4) WHERE key = run.id), created_at))) AS oldest | |
| 1788 | FROM run | |
| 1782 | 1789 | WHERE ${eligible} AND id NOT IN (SELECT value FROM json_each(?3))`, |
| 1783 | 1790 | ) |
| 1784 | .get(kind, at, ids)!; | |
| 1791 | .get(kind, at, ids, JSON.stringify(Object.fromEntries(lastHeld)))!; | |
| 1785 | 1792 | return { held, waiting: rest.waiting, oldest: rest.oldest }; |
| 1786 | 1793 | } |
| 1787 | 1794 |
src/kernel/vitals.ts +6 −3
| @@ -26,7 +26,7 @@ import { | ||
| 26 | 26 | queueDepth, |
| 27 | 27 | queuedApart, |
| 28 | 28 | } from "../core/runs.ts"; |
| 29 | import { HOLD_WORDS, heldRuns, type HoldCause } from "../runs/held.ts"; | |
| 29 | import { HOLD_WORDS, heldRuns, lastHeldAt, type HoldCause } from "../runs/held.ts"; | |
| 30 | 30 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 31 | 31 | import { ownRepoPath } from "./own_repo.ts"; |
| 32 | 32 | import { BUDGETS, limitSinceRaised, type BudgetLimits } from "./budgets.ts"; |
| @@ -567,9 +567,12 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 567 | 567 | let waiting = row.queued; |
| 568 | 568 | let oldest: string | null = row.oldest; |
| 569 | 569 | const byCause = new Map<HoldCause, number>(); |
| 570 | // Their wait begins when they could first start: not before a hold | |
| 571 | // released them, nor before their `not_before` (#14597). | |
| 570 | 572 | const holds = row.kind === "agent" ? heldRuns(input.db, now) : new Map<string, HoldCause>(); |
| 571 | if (holds.size > 0) { | |
| 572 | const apart = queuedApart(input.db, "agent", [...holds.keys()], now); | |
| 573 | const lastHeld = row.kind === "agent" ? lastHeldAt(input.db, now) : new Map<string, string>(); | |
| 574 | if (holds.size > 0 || lastHeld.size > 0) { | |
| 575 | const apart = queuedApart(input.db, "agent", [...holds.keys()], now, lastHeld); | |
| 573 | 576 | waiting = apart.waiting; |
| 574 | 577 | oldest = apart.oldest; |
| 575 | 578 | for (const id of apart.held) { |
src/runs/held.ts +28 −4
| @@ -9,6 +9,10 @@ | ||
| 9 | 9 | * one place the dispatcher and vitals share state. A pass older than |
| 10 | 10 | * `HELD_FRESH_MS` is ignored, so a dispatcher that stopped cannot hide its |
| 11 | 11 | * queue behind the last thing it said. |
| 12 | * | |
| 13 | * Each pass also remembers when it last held each run, for a day: a run | |
| 14 | * released from a hold (its issue's sibling ended, the pace turned) has waited | |
| 15 | * for a slot since then, not since it was queued (#14597). | |
| 12 | 16 | */ |
| 13 | 17 | |
| 14 | 18 | import type { Database } from "bun:sqlite"; |
| @@ -31,16 +35,34 @@ export const HOLD_WORDS: Readonly<Record<HoldCause, string>> = { | ||
| 31 | 35 | clone: "a clone backing off", |
| 32 | 36 | }; |
| 33 | 37 | |
| 38 | /** How long a pass remembers when it last held a run; a day bounds the map. */ | |
| 39 | const LAST_HELD_KEPT_MS = 24 * 60 * 60_000; | |
| 40 | ||
| 34 | 41 | interface Pass { |
| 35 | 42 | readonly at: number; |
| 36 | 43 | readonly held: ReadonlyMap<string, HoldCause>; |
| 44 | /** Run id → the instant (ms) a pass last held it, held now or not. */ | |
| 45 | readonly lastHeld: ReadonlyMap<string, number>; | |
| 37 | 46 | } |
| 38 | 47 | |
| 39 | 48 | const passes = new WeakMap<Database, Pass>(); |
| 40 | 49 | |
| 41 | 50 | /** One pass's holds, replacing the last: run id → the policy that held it. */ |
| 42 | 51 | export function noteHeld(db: Database, held: ReadonlyMap<string, HoldCause>, at = Date.now()): void { |
| 43 | passes.set(db, { at, held: new Map(held) }); | |
| 52 | const lastHeld = new Map<string, number>(); | |
| 53 | for (const [id, t] of passes.get(db)?.lastHeld ?? []) if (at - t <= LAST_HELD_KEPT_MS) lastHeld.set(id, t); | |
| 54 | for (const id of held.keys()) lastHeld.set(id, at); | |
| 55 | passes.set(db, { at, held: new Map(held), lastHeld }); | |
| 56 | } | |
| 57 | ||
| 58 | /** | |
| 59 | * When the latest fresh pass last held each run, as ISO instants: a run's wait | |
| 60 | * for a slot begins no earlier. Empty when the pass is stale, as `heldRuns`. | |
| 61 | */ | |
| 62 | export function lastHeldAt(db: Database, now = Date.now()): ReadonlyMap<string, string> { | |
| 63 | const pass = passes.get(db); | |
| 64 | if (pass === undefined || now - pass.at > HELD_FRESH_MS || pass.at - now > HELD_FRESH_MS) return new Map(); | |
| 65 | return new Map([...pass.lastHeld].map(([id, t]) => [id, new Date(t).toISOString()])); | |
| 44 | 66 | } |
| 45 | 67 | |
| 46 | 68 | /** The holds of the latest pass, when it is fresh; empty otherwise. */ |
| @@ -53,12 +75,14 @@ export function heldRuns(db: Database, now = Date.now()): ReadonlyMap<string, Ho | ||
| 53 | 75 | /** The latest pass as plain data (one scope), and its adoption by a thread that does not make passes (`engine/observed.ts`). */ |
| 54 | 76 | export const observedPass = (db: Database): Map<string, unknown> => { |
| 55 | 77 | const pass = passes.get(db); |
| 56 | return pass === undefined ? new Map() : new Map([["", { at: pass.at, held: [...pass.held] }]]); | |
| 78 | return pass === undefined | |
| 79 | ? new Map() | |
| 80 | : new Map([["", { at: pass.at, held: [...pass.held], lastHeld: [...pass.lastHeld] }]]); | |
| 57 | 81 | }; |
| 58 | 82 | export function adoptPass(db: Database, value: unknown): void { |
| 59 | 83 | if (value === undefined) passes.delete(db); |
| 60 | 84 | else { |
| 61 | const v = value as { at: number; held: [string, HoldCause][] }; | |
| 62 | passes.set(db, { at: v.at, held: new Map(v.held) }); | |
| 85 | const v = value as { at: number; held: [string, HoldCause][]; lastHeld?: [string, number][] }; | |
| 86 | passes.set(db, { at: v.at, held: new Map(v.held), lastHeld: new Map(v.lastHeld ?? []) }); | |
| 63 | 87 | } |
| 64 | 88 | } |
test/vitals-held-agent.test.ts +37 −4
| @@ -67,21 +67,54 @@ test("queue:agent leaves out the runs the dispatcher holds, and names them", () | ||
| 67 | 67 | expect(agentQueue(later)?.sentence).toContain("2 queued, 0 of"); |
| 68 | 68 | |
| 69 | 69 | // Every queued run held by policy: no stalled queue to report. |
| 70 | noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "limit"]]), later); | |
| 71 | expect(agentQueue(later)).toBeUndefined(); | |
| 70 | const before = later - QUEUE_WAIT_MAX_MS - 60_000; | |
| 71 | noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "limit"]]), before); | |
| 72 | expect(agentQueue(before)).toBeUndefined(); | |
| 72 | 73 | |
| 73 | // One held, one free: the free one is the queue, the held one is named. | |
| 74 | // One held, one free: the free one is the queue, waiting since its last | |
| 75 | // hold (#14597), and the held one is named. | |
| 74 | 76 | noteHeld(app.db, new Map([[a.id, "pace"]]), later); |
| 75 | 77 | const one = agentQueue(later); |
| 76 | 78 | expect(one?.sentence).toContain("1 queued, 0 of"); |
| 77 | 79 | expect(one?.sentence).toContain("held on purpose: 1 by the week's pace."); |
| 78 | expect(one?.since).toBe(new Date(b.createdAt).toISOString()); | |
| 80 | expect(one?.since).toBe(new Date(before).toISOString()); | |
| 79 | 81 | |
| 80 | 82 | // A dispatcher that stopped cannot hide its queue behind its last pass. |
| 81 | 83 | noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "pace"]]), later - HELD_FRESH_MS - 1); |
| 82 | 84 | expect(agentQueue(later)?.sentence).toContain("2 queued, 0 of"); |
| 83 | 85 | }); |
| 84 | 86 | |
| 87 | test("a run released from a hold waits for a slot from its release, not from when it was queued (#14597)", () => { | |
| 88 | app.db.query("DELETE FROM run").run(); | |
| 89 | const r = createRun(app.db, { repoId, kind: "agent" }); | |
| 90 | const released = Date.now() + 3 * QUEUE_WAIT_MAX_MS; | |
| 91 | // Held behind its own issue's run for over an hour, then released. | |
| 92 | noteHeld(app.db, new Map([[r.id, "issue"]]), released - 60_000); | |
| 93 | noteHeld(app.db, new Map(), released); | |
| 94 | // A minute into its wait for a slot: not a stalled queue. | |
| 95 | expect(agentQueue(released + 60_000)).toBeUndefined(); | |
| 96 | // Past the limit since its release: reported, with the release as its since. | |
| 97 | const late = released + QUEUE_WAIT_MAX_MS + 60_000; | |
| 98 | noteHeld(app.db, new Map(), late); | |
| 99 | const q = agentQueue(late); | |
| 100 | expect(q?.sentence).toContain("1 queued, 0 of"); | |
| 101 | expect(q?.since).toBe(new Date(released - 60_000).toISOString()); | |
| 102 | // A dispatcher gone quiet cannot hide the queue behind an old release. | |
| 103 | expect(agentQueue(late + HELD_FRESH_MS + 1)?.since).toBe(new Date(r.createdAt).toISOString()); | |
| 104 | }); | |
| 105 | ||
| 106 | test("a run deferred by not_before waits from then, not from when it was queued (#14597)", () => { | |
| 107 | app.db.query("DELETE FROM run").run(); | |
| 108 | const r = createRun(app.db, { repoId, kind: "agent" }); | |
| 109 | const due = Date.now() + 3 * QUEUE_WAIT_MAX_MS; | |
| 110 | app.db.query("UPDATE run SET not_before = ?1 WHERE id = ?2").run(new Date(due).toISOString(), r.id); | |
| 111 | noteHeld(app.db, new Map(), due + 60_000); | |
| 112 | expect(agentQueue(due + 60_000)).toBeUndefined(); | |
| 113 | const late = due + QUEUE_WAIT_MAX_MS + 60_000; | |
| 114 | noteHeld(app.db, new Map(), late); | |
| 115 | expect(agentQueue(late)?.since).toBe(new Date(due).toISOString()); | |
| 116 | }); | |
| 117 | ||
| 85 | 118 | test("the held count is grouped by cause, and counts only the held runs still queued", () => { |
| 86 | 119 | app.db.query("DELETE FROM run").run(); |
| 87 | 120 | const runs = Array.from({ length: 5 }, () => createRun(app.db, { repoId, kind: "agent" })); |
new above
offline