Public: anyone can read this; the crew can change it.

Sentinel: queue:agent was degraded 18 minutes today: The agent queue has waited over 30 minutes: 16 queued, 12 of 12 slots running; held on purpose: 3 by its own issue's ru… #14647

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14597-sentinel-queue-agent-was-degraded-18-min → main 1 commit fork

  • Merged. scout/14597-sentinel-queue-agent-was-degraded-18-min was merged into main as 9ce177f.

Review

Sign in to review this change.
4 files changed +82 −15
src/core/runs.ts +11 −4
@@ -1746,7 +1746,7 @@ export function queueDepth(
17461746 ): { kind: RunKind; queued: number; oldest: string }[] {
17471747 return db
17481748 .query<{ kind: RunKind; queued: number; oldest: string }, [string]>(
1749 `SELECT kind, COUNT(*) AS queued, MIN(created_at) AS oldest
1749 `SELECT kind, COUNT(*) AS queued, MIN(MAX(created_at, COALESCE(not_before, created_at))) AS oldest
17501750 FROM run
17511751 WHERE state = 'queued' AND (not_before IS NULL OR not_before <= ?1)
17521752 GROUP BY kind`,
@@ -1760,12 +1760,16 @@ export function queueDepth(
17601760 * count and oldest of the rest. Two queries, however deep the queue, both on
17611761 * `run_queue`: the `state IN` repeats the partial index's own condition so
17621762 * SQLite may use it, and `+kind` keeps it off the per-repository agent index.
1763 * A run waits for a slot from the latest of when it was queued, its
1764 * `not_before`, and `lastHeld` (run id → when a pass last held it, #14597):
1765 * a run just released from a hold has not waited the hours it was held.
17631766 */
17641767 export function queuedApart(
17651768 db: Database,
17661769 kind: RunKind,
17671770 heldIds: readonly string[],
17681771 now = Date.now(),
1772 lastHeld: ReadonlyMap<string, string> = new Map(),
17691773 ): { held: string[]; waiting: number; oldest: string | null } {
17701774 const ids = JSON.stringify(heldIds);
17711775 const at = new Date(now).toISOString();
@@ -1777,11 +1781,14 @@ export function queuedApart(
17771781 .all(kind, at, ids)
17781782 .map((r) => r.id);
17791783 const rest = db
1780 .query<{ waiting: number; oldest: string | null }, [RunKind, string, string]>(
1781 `SELECT COUNT(*) AS waiting, MIN(created_at) AS oldest FROM run
1784 .query<{ waiting: number; oldest: string | null }, [RunKind, string, string, string]>(
1785 `SELECT COUNT(*) AS waiting,
1786 MIN(MAX(created_at, COALESCE(not_before, created_at),
1787 COALESCE((SELECT value FROM json_each(?4) WHERE key = run.id), created_at))) AS oldest
1788 FROM run
17821789 WHERE ${eligible} AND id NOT IN (SELECT value FROM json_each(?3))`,
17831790 )
1784 .get(kind, at, ids)!;
1791 .get(kind, at, ids, JSON.stringify(Object.fromEntries(lastHeld)))!;
17851792 return { held, waiting: rest.waiting, oldest: rest.oldest };
17861793 }
17871794
src/kernel/vitals.ts +6 −3
@@ -26,7 +26,7 @@ import {
2626 queueDepth,
2727 queuedApart,
2828 } from "../core/runs.ts";
29 import { HOLD_WORDS, heldRuns, type HoldCause } from "../runs/held.ts";
29 import { HOLD_WORDS, heldRuns, lastHeldAt, type HoldCause } from "../runs/held.ts";
3030 import { STALE_AFTER_MS } from "../runs/lease.ts";
3131 import { ownRepoPath } from "./own_repo.ts";
3232 import { BUDGETS, limitSinceRaised, type BudgetLimits } from "./budgets.ts";
@@ -567,9 +567,12 @@ export function vitals(input: VitalsInput): Vitals {
567567 let waiting = row.queued;
568568 let oldest: string | null = row.oldest;
569569 const byCause = new Map<HoldCause, number>();
570 // Their wait begins when they could first start: not before a hold
571 // released them, nor before their `not_before` (#14597).
570572 const holds = row.kind === "agent" ? heldRuns(input.db, now) : new Map<string, HoldCause>();
571 if (holds.size > 0) {
572 const apart = queuedApart(input.db, "agent", [...holds.keys()], now);
573 const lastHeld = row.kind === "agent" ? lastHeldAt(input.db, now) : new Map<string, string>();
574 if (holds.size > 0 || lastHeld.size > 0) {
575 const apart = queuedApart(input.db, "agent", [...holds.keys()], now, lastHeld);
573576 waiting = apart.waiting;
574577 oldest = apart.oldest;
575578 for (const id of apart.held) {
src/runs/held.ts +28 −4
@@ -9,6 +9,10 @@
99 * one place the dispatcher and vitals share state. A pass older than
1010 * `HELD_FRESH_MS` is ignored, so a dispatcher that stopped cannot hide its
1111 * queue behind the last thing it said.
12 *
13 * Each pass also remembers when it last held each run, for a day: a run
14 * released from a hold (its issue's sibling ended, the pace turned) has waited
15 * for a slot since then, not since it was queued (#14597).
1216 */
1317
1418 import type { Database } from "bun:sqlite";
@@ -31,16 +35,34 @@ export const HOLD_WORDS: Readonly<Record<HoldCause, string>> = {
3135 clone: "a clone backing off",
3236 };
3337
38 /** How long a pass remembers when it last held a run; a day bounds the map. */
39 const LAST_HELD_KEPT_MS = 24 * 60 * 60_000;
40
3441 interface Pass {
3542 readonly at: number;
3643 readonly held: ReadonlyMap<string, HoldCause>;
44 /** Run id → the instant (ms) a pass last held it, held now or not. */
45 readonly lastHeld: ReadonlyMap<string, number>;
3746 }
3847
3948 const passes = new WeakMap<Database, Pass>();
4049
4150 /** One pass's holds, replacing the last: run id → the policy that held it. */
4251 export function noteHeld(db: Database, held: ReadonlyMap<string, HoldCause>, at = Date.now()): void {
43 passes.set(db, { at, held: new Map(held) });
52 const lastHeld = new Map<string, number>();
53 for (const [id, t] of passes.get(db)?.lastHeld ?? []) if (at - t <= LAST_HELD_KEPT_MS) lastHeld.set(id, t);
54 for (const id of held.keys()) lastHeld.set(id, at);
55 passes.set(db, { at, held: new Map(held), lastHeld });
56 }
57
58 /**
59 * When the latest fresh pass last held each run, as ISO instants: a run's wait
60 * for a slot begins no earlier. Empty when the pass is stale, as `heldRuns`.
61 */
62 export function lastHeldAt(db: Database, now = Date.now()): ReadonlyMap<string, string> {
63 const pass = passes.get(db);
64 if (pass === undefined || now - pass.at > HELD_FRESH_MS || pass.at - now > HELD_FRESH_MS) return new Map();
65 return new Map([...pass.lastHeld].map(([id, t]) => [id, new Date(t).toISOString()]));
4466 }
4567
4668 /** The holds of the latest pass, when it is fresh; empty otherwise. */
@@ -53,12 +75,14 @@ export function heldRuns(db: Database, now = Date.now()): ReadonlyMap<string, Ho
5375 /** The latest pass as plain data (one scope), and its adoption by a thread that does not make passes (`engine/observed.ts`). */
5476 export const observedPass = (db: Database): Map<string, unknown> => {
5577 const pass = passes.get(db);
56 return pass === undefined ? new Map() : new Map([["", { at: pass.at, held: [...pass.held] }]]);
78 return pass === undefined
79 ? new Map()
80 : new Map([["", { at: pass.at, held: [...pass.held], lastHeld: [...pass.lastHeld] }]]);
5781 };
5882 export function adoptPass(db: Database, value: unknown): void {
5983 if (value === undefined) passes.delete(db);
6084 else {
61 const v = value as { at: number; held: [string, HoldCause][] };
62 passes.set(db, { at: v.at, held: new Map(v.held) });
85 const v = value as { at: number; held: [string, HoldCause][]; lastHeld?: [string, number][] };
86 passes.set(db, { at: v.at, held: new Map(v.held), lastHeld: new Map(v.lastHeld ?? []) });
6387 }
6488 }
test/vitals-held-agent.test.ts +37 −4
@@ -67,21 +67,54 @@ test("queue:agent leaves out the runs the dispatcher holds, and names them", ()
6767 expect(agentQueue(later)?.sentence).toContain("2 queued, 0 of");
6868
6969 // Every queued run held by policy: no stalled queue to report.
70 noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "limit"]]), later);
71 expect(agentQueue(later)).toBeUndefined();
70 const before = later - QUEUE_WAIT_MAX_MS - 60_000;
71 noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "limit"]]), before);
72 expect(agentQueue(before)).toBeUndefined();
7273
73 // One held, one free: the free one is the queue, the held one is named.
74 // One held, one free: the free one is the queue, waiting since its last
75 // hold (#14597), and the held one is named.
7476 noteHeld(app.db, new Map([[a.id, "pace"]]), later);
7577 const one = agentQueue(later);
7678 expect(one?.sentence).toContain("1 queued, 0 of");
7779 expect(one?.sentence).toContain("held on purpose: 1 by the week's pace.");
78 expect(one?.since).toBe(new Date(b.createdAt).toISOString());
80 expect(one?.since).toBe(new Date(before).toISOString());
7981
8082 // A dispatcher that stopped cannot hide its queue behind its last pass.
8183 noteHeld(app.db, new Map([[a.id, "pace"], [b.id, "pace"]]), later - HELD_FRESH_MS - 1);
8284 expect(agentQueue(later)?.sentence).toContain("2 queued, 0 of");
8385 });
8486
87 test("a run released from a hold waits for a slot from its release, not from when it was queued (#14597)", () => {
88 app.db.query("DELETE FROM run").run();
89 const r = createRun(app.db, { repoId, kind: "agent" });
90 const released = Date.now() + 3 * QUEUE_WAIT_MAX_MS;
91 // Held behind its own issue's run for over an hour, then released.
92 noteHeld(app.db, new Map([[r.id, "issue"]]), released - 60_000);
93 noteHeld(app.db, new Map(), released);
94 // A minute into its wait for a slot: not a stalled queue.
95 expect(agentQueue(released + 60_000)).toBeUndefined();
96 // Past the limit since its release: reported, with the release as its since.
97 const late = released + QUEUE_WAIT_MAX_MS + 60_000;
98 noteHeld(app.db, new Map(), late);
99 const q = agentQueue(late);
100 expect(q?.sentence).toContain("1 queued, 0 of");
101 expect(q?.since).toBe(new Date(released - 60_000).toISOString());
102 // A dispatcher gone quiet cannot hide the queue behind an old release.
103 expect(agentQueue(late + HELD_FRESH_MS + 1)?.since).toBe(new Date(r.createdAt).toISOString());
104 });
105
106 test("a run deferred by not_before waits from then, not from when it was queued (#14597)", () => {
107 app.db.query("DELETE FROM run").run();
108 const r = createRun(app.db, { repoId, kind: "agent" });
109 const due = Date.now() + 3 * QUEUE_WAIT_MAX_MS;
110 app.db.query("UPDATE run SET not_before = ?1 WHERE id = ?2").run(new Date(due).toISOString(), r.id);
111 noteHeld(app.db, new Map(), due + 60_000);
112 expect(agentQueue(due + 60_000)).toBeUndefined();
113 const late = due + QUEUE_WAIT_MAX_MS + 60_000;
114 noteHeld(app.db, new Map(), late);
115 expect(agentQueue(late)?.since).toBe(new Date(due).toISOString());
116 });
117
85118 test("the held count is grouped by cause, and counts only the held runs still queued", () => {
86119 app.db.query("DELETE FROM run").run();
87120 const runs = Array.from({ length: 5 }, () => createRun(app.db, { repoId, kind: "agent" }));

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs