Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12823

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12811-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 4 commits fork 2 threads · 2 unresolved

  • Merged. torque/12811-sentinel-budget-01m7cph78n2493t8as2avbd1 was merged into main as 1d071fd.

Review

Sign in to review this change.
5 files changed +168 −8
src/checks/service.ts +6 −5
@@ -98,7 +98,7 @@ import {
9898 verifyDue,
9999 } from "./verify.ts";
100100 import { getIssueById } from "../core/issues.ts";
101 import { BUDGETS, budgetSummary, STARVATION_MS } from "../kernel/budgets.ts";
101 import { BUDGETS, budgetSummary, checkLimitMs, STARVATION_MS } from "../kernel/budgets.ts";
102102 import { transact } from "../kernel/db.ts";
103103 import { createLog } from "../kernel/log.ts";
104104 import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts";
@@ -901,10 +901,11 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
901901 write(`forge ${options.label} — checked in its parent's lane`);
902902 const started = Date.now();
903903 // The budget: past it the check is aborted, whatever it is doing.
904 const limit =
905 run.env === VERIFY_ENV
906 ? (options.verifyBudgetMs ?? BUDGETS.verify)
907 : (options.budgetMs ?? BUDGETS.gate);
904 const limit = checkLimitMs(run.env, {
905 gate: options.budgetMs ?? BUDGETS.gate,
906 verify: options.verifyBudgetMs ?? BUDGETS.verify,
907 review: BUDGETS.review,
908 });
908909 const budget = new AbortController();
909910 let overran = false;
910911 const forward = (): void => budget.abort();
src/kernel/budgets.ts +46
@@ -36,6 +36,13 @@ export const BUDGETS = {
3636 */
3737 export const STARVATION_MS = BUDGETS.laneWait / 2;
3838
39 /** The run limits, in ms, a budget kill is measured against today. */
40 export type BudgetLimits = {
41 readonly gate: number;
42 readonly verify: number;
43 readonly review: number;
44 };
45
3946 /** The summary prefix every budget kill carries; vitals and retries read it. */
4047 export const BUDGET_PREFIX = "budget:";
4148
@@ -48,3 +55,42 @@ export function budgetSummary(
4855 const min = (ms: number): number => Math.max(Math.round(ms / 60_000), 1);
4956 return `${BUDGET_PREFIX} ${what} took ${min(tookMs)} min, limit ${min(limitMs)}`;
5057 }
58
59 /**
60 * The limit, in minutes, a budget kill's summary names ("… limit 60"), or
61 * undefined when it names none. The parser of `budgetSummary`'s format, just
62 * above: change one and the round-trip test in budget-raised fails.
63 */
64 export function limitInSummary(summary: string): number | undefined {
65 const m = /limit (\d+)\s*$/.exec(summary.trim());
66 return m === null ? undefined : Number(m[1]);
67 }
68
69 /**
70 * The limit, in ms, a check run in `env` is killed at: a scheduled
71 * verification has its own; everything else — a pull's check, main after a
72 * landing, and a bisection step (`verify-bisect`), which runs a few named
73 * files, not the whole gate one file to a process — runs under the gate's.
74 * The checks service kills by this and vitals judges a kill by it, so the
75 * two cannot drift.
76 */
77 export function checkLimitMs(env: string, limits: BudgetLimits = BUDGETS): number {
78 return env === "verify" ? limits.verify : limits.gate;
outdated

Treat verify-bisect like verify here (see lanes_read.ts:424) and test it.

79 }
80
81 /**
82 * Whether a budget kill happened under a limit that has since been raised: the
83 * run would get more room today, so the kill is history, not a card (#12811).
84 * A check is judged by `checkLimitMs`; an agent run's kill is a reviewer's,
85 * judged by the review limit. `limits` are the ones the services run under,
86 * `BUDGETS` unless they were started with others.
87 */
88 export function limitSinceRaised(
89 run: { readonly summary: string | null; readonly kind: string; readonly env: string },
90 limits: BudgetLimits = BUDGETS,
91 ): boolean {
92 const was = limitInSummary(run.summary ?? "");
93 if (was === undefined) return false;
94 const now = run.kind === "check" ? checkLimitMs(run.env, limits) : limits.review;
95 return was < Math.max(Math.round(now / 60_000), 1);
96 }
src/kernel/vitals.ts +8 −1
@@ -28,7 +28,7 @@ import {
2828 import { HOLD_WORDS, heldRuns, type HoldCause } from "../runs/held.ts";
2929 import { STALE_AFTER_MS } from "../runs/lease.ts";
3030 import { ownRepoPath } from "./own_repo.ts";
31 import { BUDGETS } from "./budgets.ts";
31 import { BUDGETS, limitSinceRaised, type BudgetLimits } from "./budgets.ts";
3232 import { crashLoop, LOOP_STARTS, LOOP_WINDOW_MS } from "./boots.ts";
3333 import { openDbFiles } from "./file_modes.ts";
3434 import { dbFile, dbGrowth, dbSize, dbSizeNeed, fileBytes, type DbWalk } from "../system/db_size.ts";
@@ -190,6 +190,11 @@ export interface VitalsInput {
190190 * fires; `dbSize.last` unless a test gives one.
191191 */
192192 readonly dbWalk?: (db: Database, dataDir: string) => DbWalk | null;
193 /**
194 * The limits the checks service runs under, in ms; `BUDGETS` unless the
195 * service was started with others. A kill under a lower limit is history.
196 */
197 readonly runLimits?: BudgetLimits;
193198 /** One forge's own signals (`App.signals`). */
194199 readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
195200 /** Where snapshots go: the directory, when the store is one, and whether an off-box mirror copies them. */
@@ -596,6 +601,8 @@ export function vitals(input: VitalsInput): Vitals {
596601 )) {
597602 // A later run of the same kind passed: fixed, not waiting out its day.
598603 if (passedSince(input.db, run)) continue;
604 // Killed under a limit since raised: history, not a card.
605 if (limitSinceRaised(run, input.runLimits)) continue;
outdated

This skip has no test that fails without it. Add a vitals-level case (old limit, default runLimits => no card).

599606 need(
600607 "todo",
601608 `budget:${run.id}`,
test/budget-raised.test.ts added +104
@@ -0,0 +1,104 @@
1 /**
2 * A budget kill under a limit since raised is history, not a card (#12811):
3 * the helper, the format it parses, and vitals carrying it.
4 */
5
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { createRun } from "../src/core/runs.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { createRepo } from "../src/actions/repos.ts";
14 import {
15 BUDGETS,
16 budgetSummary,
17 checkLimitMs,
18 limitInSummary,
19 limitSinceRaised,
20 } from "../src/kernel/budgets.ts";
21 import { vitals } from "../src/kernel/vitals.ts";
22
23 const MIN = 60_000;
24 const check = (env: string, summary: string) => ({ kind: "check", env, summary });
25
26 test("budgetSummary's limit reads back through limitInSummary", () => {
27 for (const minutes of [1, 10, 60, 150]) {
28 const said = budgetSummary("a full gate", minutes * MIN + 5_000, minutes * MIN);
29 expect(said).toBe(`budget: a full gate took ${minutes} min, limit ${minutes}`);
30 expect(limitInSummary(said)).toBe(minutes);
31 }
32 expect(limitInSummary("budget: something")).toBeUndefined();
33 });
34
35 test("a verification killed under the old 60 min limit is history", () => {
36 const old = budgetSummary("a full gate", 60 * MIN, 60 * MIN);
37 expect(limitSinceRaised(check("verify", old))).toBe(true);
38 });
39
40 test("a kill under today's limit still counts", () => {
41 expect(limitSinceRaised(check("verify", budgetSummary("a full gate", BUDGETS.verify, BUDGETS.verify)))).toBe(false);
42 expect(limitSinceRaised(check("main", budgetSummary("a full gate", BUDGETS.gate, BUDGETS.gate)))).toBe(false);
43 });
44
45 test("a bisection step runs under the gate's limit, so its kill there still counts", () => {
46 expect(checkLimitMs("verify-bisect")).toBe(BUDGETS.gate);
47 expect(checkLimitMs("verify")).toBe(BUDGETS.verify);
48 const atGate = budgetSummary("a full gate", BUDGETS.gate, BUDGETS.gate);
49 expect(limitSinceRaised(check("verify-bisect", atGate))).toBe(false);
50 });
51
52 test("a reviewer's kill is judged by the review limit, not the gate's", () => {
53 const review = budgetSummary("a review run", BUDGETS.review, BUDGETS.review);
54 expect(limitSinceRaised({ kind: "agent", env: "", summary: review })).toBe(false);
55 expect(limitSinceRaised({ kind: "agent", env: "", summary: budgetSummary("a review run", MIN, MIN) })).toBe(true);
56 });
57
58 test("a summary naming no limit is not history", () => {
59 expect(limitSinceRaised(check("verify", "budget: something"))).toBe(false);
60 });
61
62 test("a kill is measured against the limits the service runs under", () => {
63 const short = check("main", budgetSummary("a full gate", 1, 1));
64 expect(limitSinceRaised(short)).toBe(true);
65 expect(limitSinceRaised(short, { gate: 1, verify: MIN, review: MIN })).toBe(false);
66 });
67
68 let dataDir: string;
69 let app: App;
70
71 beforeAll(async () => {
72 dataDir = await mkdtemp(join(tmpdir(), "forge-budget-raised-"));
73 app = createApp({ dataDir, port: 0 });
74 });
75
76 afterAll(async () => {
77 app.close();
78 await rm(dataDir, { recursive: true, force: true });
79 });
80
81 test("vitals: a verification killed at the old limit is no card; under limits that match it, it is", async () => {
82 const u = await createUser(app.db, {
83 origin: "seed",
84 trusted: true,
85 username: "alice",
86 email: "[email protected]",
87 password: "a-good-passphrase",
88 adminIfFirst: true,
89 });
90 if (!u.ok) throw new Error(u.error);
91 const made = await createRepo(app, u.value, { name: "slow", visibility: "public" });
92 if (!made.ok) throw new Error(String(made.error));
93 const summary = "budget: a full gate took 60 min, limit 60";
94 const run = createRun(app.db, { repoId: made.value.id, kind: "check", env: "verify", summary });
95 app.db
96 .query("UPDATE run SET state = 'failed', finished_at = ?2 WHERE id = ?1")
97 .run(run.id, new Date().toISOString());
98 const card = (limits?: Parameters<typeof vitals>[0]["runLimits"]) =>
99 vitals({ db: app.db, signals: app.signals, dataDir, ...(limits ? { runLimits: limits } : {}) })
100 .needs.some((n) => n.includes(summary));
101
102 expect(card()).toBe(false);
103 expect(card({ gate: 60 * MIN, verify: 60 * MIN, review: BUDGETS.review })).toBe(true);
104 });
test/check-budget.test.ts +4 −2
@@ -90,7 +90,9 @@ test("a check past its budget is re-queued once; the second overrun fails it and
9090 expect(second.state).toBe("failed");
9191 expect(second.summary).toMatch(/^budget: a full gate took \d+ min, limit \d+$/);
9292
93 const seen = vitals({ db: app.db, signals: app.signals, dataDir });
93 // The limits this service was started with: the kill is under them, a card.
94 const runLimits = { gate: 400, verify: 60_000, review: BUDGETS.review };
95 const seen = vitals({ db: app.db, signals: app.signals, dataDir, runLimits });
9496 expect(seen.todo.length).toBeGreaterThan(0);
9597 expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
9698
@@ -98,7 +100,7 @@ test("a check past its budget is re-queued once; the second overrun fails it and
98100 // not when its day is out (#12625).
99101 const later = createRun(app.db, { repoId: repo.id, kind: "check", env: second.env });
100102 app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(later.id, new Date().toISOString());
101 const well = vitals({ db: app.db, signals: app.signals, dataDir });
103 const well = vitals({ db: app.db, signals: app.signals, dataDir, runLimits });
102104 expect(well.needs.some((n) => n.includes("budget: a full gate"))).toBe(false);
103105 }, 60_000);
104106

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs