Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12823
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12811-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 4 commits fork 2 threads · 2 unresolved
5 files
changed +168
−8
src/checks/service.ts +6 −5
| @@ -98,7 +98,7 @@ import { | ||
| 98 | 98 | verifyDue, |
| 99 | 99 | } from "./verify.ts"; |
| 100 | 100 | import { getIssueById } from "../core/issues.ts"; |
| 101 | import { BUDGETS, budgetSummary, STARVATION_MS } from "../kernel/budgets.ts"; | |
| 101 | import { BUDGETS, budgetSummary, checkLimitMs, STARVATION_MS } from "../kernel/budgets.ts"; | |
| 102 | 102 | import { transact } from "../kernel/db.ts"; |
| 103 | 103 | import { createLog } from "../kernel/log.ts"; |
| 104 | 104 | import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts"; |
| @@ -901,10 +901,11 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 901 | 901 | write(`forge ${options.label} — checked in its parent's lane`); |
| 902 | 902 | const started = Date.now(); |
| 903 | 903 | // The budget: past it the check is aborted, whatever it is doing. |
| 904 | const limit = | |
| 905 | run.env === VERIFY_ENV | |
| 906 | ? (options.verifyBudgetMs ?? BUDGETS.verify) | |
| 907 | : (options.budgetMs ?? BUDGETS.gate); | |
| 904 | const limit = checkLimitMs(run.env, { | |
| 905 | gate: options.budgetMs ?? BUDGETS.gate, | |
| 906 | verify: options.verifyBudgetMs ?? BUDGETS.verify, | |
| 907 | review: BUDGETS.review, | |
| 908 | }); | |
| 908 | 909 | const budget = new AbortController(); |
| 909 | 910 | let overran = false; |
| 910 | 911 | const forward = (): void => budget.abort(); |
src/kernel/budgets.ts +46
| @@ -36,6 +36,13 @@ export const BUDGETS = { | ||
| 36 | 36 | */ |
| 37 | 37 | export const STARVATION_MS = BUDGETS.laneWait / 2; |
| 38 | 38 | |
| 39 | /** The run limits, in ms, a budget kill is measured against today. */ | |
| 40 | export type BudgetLimits = { | |
| 41 | readonly gate: number; | |
| 42 | readonly verify: number; | |
| 43 | readonly review: number; | |
| 44 | }; | |
| 45 | ||
| 39 | 46 | /** The summary prefix every budget kill carries; vitals and retries read it. */ |
| 40 | 47 | export const BUDGET_PREFIX = "budget:"; |
| 41 | 48 | |
| @@ -48,3 +55,42 @@ export function budgetSummary( | ||
| 48 | 55 | const min = (ms: number): number => Math.max(Math.round(ms / 60_000), 1); |
| 49 | 56 | return `${BUDGET_PREFIX} ${what} took ${min(tookMs)} min, limit ${min(limitMs)}`; |
| 50 | 57 | } |
| 58 | ||
| 59 | /** | |
| 60 | * The limit, in minutes, a budget kill's summary names ("… limit 60"), or | |
| 61 | * undefined when it names none. The parser of `budgetSummary`'s format, just | |
| 62 | * above: change one and the round-trip test in budget-raised fails. | |
| 63 | */ | |
| 64 | export function limitInSummary(summary: string): number | undefined { | |
| 65 | const m = /limit (\d+)\s*$/.exec(summary.trim()); | |
| 66 | return m === null ? undefined : Number(m[1]); | |
| 67 | } | |
| 68 | ||
| 69 | /** | |
| 70 | * The limit, in ms, a check run in `env` is killed at: a scheduled | |
| 71 | * verification has its own; everything else — a pull's check, main after a | |
| 72 | * landing, and a bisection step (`verify-bisect`), which runs a few named | |
| 73 | * files, not the whole gate one file to a process — runs under the gate's. | |
| 74 | * The checks service kills by this and vitals judges a kill by it, so the | |
| 75 | * two cannot drift. | |
| 76 | */ | |
| 77 | export function checkLimitMs(env: string, limits: BudgetLimits = BUDGETS): number { | |
| 78 | return env === "verify" ? limits.verify : limits.gate; | |
| 79 | } | |
| 80 | ||
| 81 | /** | |
| 82 | * Whether a budget kill happened under a limit that has since been raised: the | |
| 83 | * run would get more room today, so the kill is history, not a card (#12811). | |
| 84 | * A check is judged by `checkLimitMs`; an agent run's kill is a reviewer's, | |
| 85 | * judged by the review limit. `limits` are the ones the services run under, | |
| 86 | * `BUDGETS` unless they were started with others. | |
| 87 | */ | |
| 88 | export function limitSinceRaised( | |
| 89 | run: { readonly summary: string | null; readonly kind: string; readonly env: string }, | |
| 90 | limits: BudgetLimits = BUDGETS, | |
| 91 | ): boolean { | |
| 92 | const was = limitInSummary(run.summary ?? ""); | |
| 93 | if (was === undefined) return false; | |
| 94 | const now = run.kind === "check" ? checkLimitMs(run.env, limits) : limits.review; | |
| 95 | return was < Math.max(Math.round(now / 60_000), 1); | |
| 96 | } |
src/kernel/vitals.ts +8 −1
| @@ -28,7 +28,7 @@ import { | ||
| 28 | 28 | import { HOLD_WORDS, heldRuns, type HoldCause } from "../runs/held.ts"; |
| 29 | 29 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 30 | 30 | import { ownRepoPath } from "./own_repo.ts"; |
| 31 | import { BUDGETS } from "./budgets.ts"; | |
| 31 | import { BUDGETS, limitSinceRaised, type BudgetLimits } from "./budgets.ts"; | |
| 32 | 32 | import { crashLoop, LOOP_STARTS, LOOP_WINDOW_MS } from "./boots.ts"; |
| 33 | 33 | import { openDbFiles } from "./file_modes.ts"; |
| 34 | 34 | import { dbFile, dbGrowth, dbSize, dbSizeNeed, fileBytes, type DbWalk } from "../system/db_size.ts"; |
| @@ -190,6 +190,11 @@ export interface VitalsInput { | ||
| 190 | 190 | * fires; `dbSize.last` unless a test gives one. |
| 191 | 191 | */ |
| 192 | 192 | readonly dbWalk?: (db: Database, dataDir: string) => DbWalk | null; |
| 193 | /** | |
| 194 | * The limits the checks service runs under, in ms; `BUDGETS` unless the | |
| 195 | * service was started with others. A kill under a lower limit is history. | |
| 196 | */ | |
| 197 | readonly runLimits?: BudgetLimits; | |
| 193 | 198 | /** One forge's own signals (`App.signals`). */ |
| 194 | 199 | readonly signals: { errors: ErrorLedger; refused: RefusedLedger }; |
| 195 | 200 | /** Where snapshots go: the directory, when the store is one, and whether an off-box mirror copies them. */ |
| @@ -596,6 +601,8 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 596 | 601 | )) { |
| 597 | 602 | // A later run of the same kind passed: fixed, not waiting out its day. |
| 598 | 603 | if (passedSince(input.db, run)) continue; |
| 604 | // Killed under a limit since raised: history, not a card. | |
| 605 | if (limitSinceRaised(run, input.runLimits)) continue; | |
outdated
This skip has no test that fails without it. Add a vitals-level case (old limit, default runLimits => no card). |
||
| 599 | 606 | need( |
| 600 | 607 | "todo", |
| 601 | 608 | `budget:${run.id}`, |
test/budget-raised.test.ts added +104
| @@ -0,0 +1,104 @@ | ||
| 1 | /** | |
| 2 | * A budget kill under a limit since raised is history, not a card (#12811): | |
| 3 | * the helper, the format it parses, and vitals carrying it. | |
| 4 | */ | |
| 5 | ||
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { createRun } from "../src/core/runs.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { | |
| 15 | BUDGETS, | |
| 16 | budgetSummary, | |
| 17 | checkLimitMs, | |
| 18 | limitInSummary, | |
| 19 | limitSinceRaised, | |
| 20 | } from "../src/kernel/budgets.ts"; | |
| 21 | import { vitals } from "../src/kernel/vitals.ts"; | |
| 22 | ||
| 23 | const MIN = 60_000; | |
| 24 | const check = (env: string, summary: string) => ({ kind: "check", env, summary }); | |
| 25 | ||
| 26 | test("budgetSummary's limit reads back through limitInSummary", () => { | |
| 27 | for (const minutes of [1, 10, 60, 150]) { | |
| 28 | const said = budgetSummary("a full gate", minutes * MIN + 5_000, minutes * MIN); | |
| 29 | expect(said).toBe(`budget: a full gate took ${minutes} min, limit ${minutes}`); | |
| 30 | expect(limitInSummary(said)).toBe(minutes); | |
| 31 | } | |
| 32 | expect(limitInSummary("budget: something")).toBeUndefined(); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a verification killed under the old 60 min limit is history", () => { | |
| 36 | const old = budgetSummary("a full gate", 60 * MIN, 60 * MIN); | |
| 37 | expect(limitSinceRaised(check("verify", old))).toBe(true); | |
| 38 | }); | |
| 39 | ||
| 40 | test("a kill under today's limit still counts", () => { | |
| 41 | expect(limitSinceRaised(check("verify", budgetSummary("a full gate", BUDGETS.verify, BUDGETS.verify)))).toBe(false); | |
| 42 | expect(limitSinceRaised(check("main", budgetSummary("a full gate", BUDGETS.gate, BUDGETS.gate)))).toBe(false); | |
| 43 | }); | |
| 44 | ||
| 45 | test("a bisection step runs under the gate's limit, so its kill there still counts", () => { | |
| 46 | expect(checkLimitMs("verify-bisect")).toBe(BUDGETS.gate); | |
| 47 | expect(checkLimitMs("verify")).toBe(BUDGETS.verify); | |
| 48 | const atGate = budgetSummary("a full gate", BUDGETS.gate, BUDGETS.gate); | |
| 49 | expect(limitSinceRaised(check("verify-bisect", atGate))).toBe(false); | |
| 50 | }); | |
| 51 | ||
| 52 | test("a reviewer's kill is judged by the review limit, not the gate's", () => { | |
| 53 | const review = budgetSummary("a review run", BUDGETS.review, BUDGETS.review); | |
| 54 | expect(limitSinceRaised({ kind: "agent", env: "", summary: review })).toBe(false); | |
| 55 | expect(limitSinceRaised({ kind: "agent", env: "", summary: budgetSummary("a review run", MIN, MIN) })).toBe(true); | |
| 56 | }); | |
| 57 | ||
| 58 | test("a summary naming no limit is not history", () => { | |
| 59 | expect(limitSinceRaised(check("verify", "budget: something"))).toBe(false); | |
| 60 | }); | |
| 61 | ||
| 62 | test("a kill is measured against the limits the service runs under", () => { | |
| 63 | const short = check("main", budgetSummary("a full gate", 1, 1)); | |
| 64 | expect(limitSinceRaised(short)).toBe(true); | |
| 65 | expect(limitSinceRaised(short, { gate: 1, verify: MIN, review: MIN })).toBe(false); | |
| 66 | }); | |
| 67 | ||
| 68 | let dataDir: string; | |
| 69 | let app: App; | |
| 70 | ||
| 71 | beforeAll(async () => { | |
| 72 | dataDir = await mkdtemp(join(tmpdir(), "forge-budget-raised-")); | |
| 73 | app = createApp({ dataDir, port: 0 }); | |
| 74 | }); | |
| 75 | ||
| 76 | afterAll(async () => { | |
| 77 | app.close(); | |
| 78 | await rm(dataDir, { recursive: true, force: true }); | |
| 79 | }); | |
| 80 | ||
| 81 | test("vitals: a verification killed at the old limit is no card; under limits that match it, it is", async () => { | |
| 82 | const u = await createUser(app.db, { | |
| 83 | origin: "seed", | |
| 84 | trusted: true, | |
| 85 | username: "alice", | |
| 86 | email: "[email protected]", | |
| 87 | password: "a-good-passphrase", | |
| 88 | adminIfFirst: true, | |
| 89 | }); | |
| 90 | if (!u.ok) throw new Error(u.error); | |
| 91 | const made = await createRepo(app, u.value, { name: "slow", visibility: "public" }); | |
| 92 | if (!made.ok) throw new Error(String(made.error)); | |
| 93 | const summary = "budget: a full gate took 60 min, limit 60"; | |
| 94 | const run = createRun(app.db, { repoId: made.value.id, kind: "check", env: "verify", summary }); | |
| 95 | app.db | |
| 96 | .query("UPDATE run SET state = 'failed', finished_at = ?2 WHERE id = ?1") | |
| 97 | .run(run.id, new Date().toISOString()); | |
| 98 | const card = (limits?: Parameters<typeof vitals>[0]["runLimits"]) => | |
| 99 | vitals({ db: app.db, signals: app.signals, dataDir, ...(limits ? { runLimits: limits } : {}) }) | |
| 100 | .needs.some((n) => n.includes(summary)); | |
| 101 | ||
| 102 | expect(card()).toBe(false); | |
| 103 | expect(card({ gate: 60 * MIN, verify: 60 * MIN, review: BUDGETS.review })).toBe(true); | |
| 104 | }); |
test/check-budget.test.ts +4 −2
| @@ -90,7 +90,9 @@ test("a check past its budget is re-queued once; the second overrun fails it and | ||
| 90 | 90 | expect(second.state).toBe("failed"); |
| 91 | 91 | expect(second.summary).toMatch(/^budget: a full gate took \d+ min, limit \d+$/); |
| 92 | 92 | |
| 93 | const seen = vitals({ db: app.db, signals: app.signals, dataDir }); | |
| 93 | // The limits this service was started with: the kill is under them, a card. | |
| 94 | const runLimits = { gate: 400, verify: 60_000, review: BUDGETS.review }; | |
| 95 | const seen = vitals({ db: app.db, signals: app.signals, dataDir, runLimits }); | |
| 94 | 96 | expect(seen.todo.length).toBeGreaterThan(0); |
| 95 | 97 | expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true); |
| 96 | 98 | |
| @@ -98,7 +100,7 @@ test("a check past its budget is re-queued once; the second overrun fails it and | ||
| 98 | 100 | // not when its day is out (#12625). |
| 99 | 101 | const later = createRun(app.db, { repoId: repo.id, kind: "check", env: second.env }); |
| 100 | 102 | app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(later.id, new Date().toISOString()); |
| 101 | const well = vitals({ db: app.db, signals: app.signals, dataDir }); | |
| 103 | const well = vitals({ db: app.db, signals: app.signals, dataDir, runLimits }); | |
| 102 | 104 | expect(well.needs.some((n) => n.includes("budget: a full gate"))).toBe(false); |
| 103 | 105 | }, 60_000); |
| 104 | 106 |
new above
offline
Treat
verify-bisectlikeverifyhere (see lanes_read.ts:424) and test it.