Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12626

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12625-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 2 commits fork

  • Merged. torque/12625-sentinel-budget-01m7cph78n2493t8as2avbd1 was merged into main as a9ebc3b.

Review

Sign in to review this change.
3 files changed +29 −2
src/core/runs.ts +16
@@ -754,6 +754,22 @@ export function budgetFailures(db: Database, since: string): Run[] {
754754 .map(toRun);
755755 }
756756
757 /**
758 * Whether a later run of the same repository, kind and environment passed
759 * after `run`: a budget kill it outlived is fixed, not a card waiting out its
760 * day (#12625). Run ids sort by creation, so "later" is a greater id.
761 */
762 export function passedSince(db: Database, run: Run): boolean {
763 return (
764 db
765 .query<{ one: number }, [string, string, string, string]>(
766 `SELECT 1 AS one FROM run WHERE repo_id = ?1 AND kind = ?2 AND env = ?3
767 AND id > ?4 AND state = 'ok' LIMIT 1`,
768 )
769 .get(run.repoId, run.kind, run.env, run.id) !== null
770 );
771 }
772
757773 /** What a sweep of abandoned checks did, run by run. */
758774 export interface AbandonedChecks {
759775 /** Cancelled and queued again: the cancelled run and its retry. */
src/kernel/vitals.ts +5 −1
@@ -19,6 +19,7 @@ import { refusedInternal, type RefusedLedger } from "./internal_refused.ts";
1919 import { recentErrors, redactSecrets, type ErrorLedger, type ErrorLine } from "./log.ts";
2020 import {
2121 budgetFailures,
22 passedSince,
2223 leaseLapsed,
2324 listRunning,
2425 queueDepth,
@@ -592,7 +593,9 @@ export function vitals(input: VitalsInput): Vitals {
592593 for (const run of budgetFailures(
593594 input.db,
594595 new Date(now - 24 * 3_600_000).toISOString(),
595 ))
596 )) {
597 // A later run of the same kind passed: fixed, not waiting out its day.
598 if (passedSince(input.db, run)) continue;
596599 need(
597600 "todo",
598601 `budget:${run.id}`,
@@ -600,6 +603,7 @@ export function vitals(input: VitalsInput): Vitals {
600603 run.finishedAt ?? undefined,
601604 "/settings/capacity",
602605 );
606 }
603607
604608 // A red main holding its pulls' checks is one card, naming the test.
605609 for (const { red, held } of reds)
test/check-budget.test.ts +8 −1
@@ -12,7 +12,7 @@ import { commitToBranch, createRepo } from "../src/actions/repos.ts";
1212 import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
1313 import { BUDGETS, budgetSummary } from "../src/kernel/budgets.ts";
1414 import { vitals } from "../src/kernel/vitals.ts";
15 import { listRuns } from "../src/core/runs.ts";
15 import { createRun, listRuns } from "../src/core/runs.ts";
1616 import { createUser, getUserByName } from "../src/core/users.ts";
1717
1818 process.env["FORGE_CELLS"] = "off";
@@ -93,6 +93,13 @@ test("a check past its budget is re-queued once; the second overrun fails it and
9393 const seen = vitals({ db: app.db, signals: app.signals, dataDir });
9494 expect(seen.todo.length).toBeGreaterThan(0);
9595 expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
96
97 // A later check of the same kind that passes is the fix: the card goes now,
98 // not when its day is out (#12625).
99 const later = createRun(app.db, { repoId: repo.id, kind: "check", env: second.env });
100 app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(later.id, new Date().toISOString());
101 const well = vitals({ db: app.db, signals: app.signals, dataDir });
102 expect(well.needs.some((n) => n.includes("budget: a full gate"))).toBe(false);
96103 }, 60_000);
97104
98105 test("a verification of main is not cut off at the landing gate's limit (#12499)", async () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs