Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12626
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12625-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 2 commits fork
3 files
changed +29
−2
src/core/runs.ts +16
| @@ -754,6 +754,22 @@ export function budgetFailures(db: Database, since: string): Run[] { | ||
| 754 | 754 | .map(toRun); |
| 755 | 755 | } |
| 756 | 756 | |
| 757 | /** | |
| 758 | * Whether a later run of the same repository, kind and environment passed | |
| 759 | * after `run`: a budget kill it outlived is fixed, not a card waiting out its | |
| 760 | * day (#12625). Run ids sort by creation, so "later" is a greater id. | |
| 761 | */ | |
| 762 | export function passedSince(db: Database, run: Run): boolean { | |
| 763 | return ( | |
| 764 | db | |
| 765 | .query<{ one: number }, [string, string, string, string]>( | |
| 766 | `SELECT 1 AS one FROM run WHERE repo_id = ?1 AND kind = ?2 AND env = ?3 | |
| 767 | AND id > ?4 AND state = 'ok' LIMIT 1`, | |
| 768 | ) | |
| 769 | .get(run.repoId, run.kind, run.env, run.id) !== null | |
| 770 | ); | |
| 771 | } | |
| 772 | ||
| 757 | 773 | /** What a sweep of abandoned checks did, run by run. */ |
| 758 | 774 | export interface AbandonedChecks { |
| 759 | 775 | /** Cancelled and queued again: the cancelled run and its retry. */ |
src/kernel/vitals.ts +5 −1
| @@ -19,6 +19,7 @@ import { refusedInternal, type RefusedLedger } from "./internal_refused.ts"; | ||
| 19 | 19 | import { recentErrors, redactSecrets, type ErrorLedger, type ErrorLine } from "./log.ts"; |
| 20 | 20 | import { |
| 21 | 21 | budgetFailures, |
| 22 | passedSince, | |
| 22 | 23 | leaseLapsed, |
| 23 | 24 | listRunning, |
| 24 | 25 | queueDepth, |
| @@ -592,7 +593,9 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 592 | 593 | for (const run of budgetFailures( |
| 593 | 594 | input.db, |
| 594 | 595 | new Date(now - 24 * 3_600_000).toISOString(), |
| 595 | )) | |
| 596 | )) { | |
| 597 | // A later run of the same kind passed: fixed, not waiting out its day. | |
| 598 | if (passedSince(input.db, run)) continue; | |
| 596 | 599 | need( |
| 597 | 600 | "todo", |
| 598 | 601 | `budget:${run.id}`, |
| @@ -600,6 +603,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 600 | 603 | run.finishedAt ?? undefined, |
| 601 | 604 | "/settings/capacity", |
| 602 | 605 | ); |
| 606 | } | |
| 603 | 607 | |
| 604 | 608 | // A red main holding its pulls' checks is one card, naming the test. |
| 605 | 609 | for (const { red, held } of reds) |
test/check-budget.test.ts +8 −1
| @@ -12,7 +12,7 @@ import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | ||
| 12 | 12 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; |
| 13 | 13 | import { BUDGETS, budgetSummary } from "../src/kernel/budgets.ts"; |
| 14 | 14 | import { vitals } from "../src/kernel/vitals.ts"; |
| 15 | import { listRuns } from "../src/core/runs.ts"; | |
| 15 | import { createRun, listRuns } from "../src/core/runs.ts"; | |
| 16 | 16 | import { createUser, getUserByName } from "../src/core/users.ts"; |
| 17 | 17 | |
| 18 | 18 | process.env["FORGE_CELLS"] = "off"; |
| @@ -93,6 +93,13 @@ test("a check past its budget is re-queued once; the second overrun fails it and | ||
| 93 | 93 | const seen = vitals({ db: app.db, signals: app.signals, dataDir }); |
| 94 | 94 | expect(seen.todo.length).toBeGreaterThan(0); |
| 95 | 95 | expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true); |
| 96 | ||
| 97 | // A later check of the same kind that passes is the fix: the card goes now, | |
| 98 | // not when its day is out (#12625). | |
| 99 | const later = createRun(app.db, { repoId: repo.id, kind: "check", env: second.env }); | |
| 100 | app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(later.id, new Date().toISOString()); | |
| 101 | const well = vitals({ db: app.db, signals: app.signals, dataDir }); | |
| 102 | expect(well.needs.some((n) => n.includes("budget: a full gate"))).toBe(false); | |
| 96 | 103 | }, 60_000); |
| 97 | 104 | |
| 98 | 105 | test("a verification of main is not cut off at the landing gate's limit (#12499)", async () => { |
new above
offline