Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12603

Closed opened this
Closed without merging

scout/forge:scout/12601-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
2 files changed +53 −2
src/core/runs.ts +10 −2
@@ -743,12 +743,20 @@ function cutOffBefore(db: Database, run: Run, prefix: string): boolean {
743743 );
744744 }
745745
746 /** Failed runs the budget killed since `since`: each is a card in the vitals. */
746 /**
747 * Failed runs the budget killed since `since`: each is a card in the vitals,
748 * until a later run of the same kind, repo and environment has passed: the
749 * cause is fixed then, and a day's wait is not the cure.
750 */
747751 export function budgetFailures(db: Database, since: string): Run[] {
748752 return db
749753 .query<RunRow, [string, string]>(
750754 `SELECT * FROM run INDEXED BY run_failed_finished WHERE state = 'failed' AND summary LIKE ?2 || '%'
751 AND finished_at >= ?1 ORDER BY finished_at DESC LIMIT 20`,
755 AND finished_at >= ?1
756 AND NOT EXISTS (SELECT 1 FROM run later WHERE later.kind = run.kind
757 AND later.repo_id = run.repo_id AND later.env = run.env
758 AND later.state = 'ok' AND later.finished_at > run.finished_at)
759 ORDER BY finished_at DESC LIMIT 20`,
752760 )
753761 .all(since, BUDGET_PREFIX)
754762 .map(toRun);
test/budget-card-cleared.test.ts added +43
@@ -0,0 +1,43 @@
1 /** A run killed for its budget is a card until a later run of its kind passes. */
2
3 import { expect, test } from "bun:test";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp } from "../src/app.ts";
8 import { createRepo } from "../src/actions/repos.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { budgetFailures, claimRun, createRun, finishRun } from "../src/core/runs.ts";
11
12 test("a later passing run of the same kind clears the budget card", async () => {
13 const dataDir = await mkdtemp(join(tmpdir(), "forge-budget-card-"));
14 const app = createApp({ dataDir, port: 0 });
15 try {
16 const u = await createUser(app.db, {
17 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
18 password: "a-good-passphrase", adminIfFirst: true,
19 });
20 if (!u.ok) throw new Error(u.error);
21 const made = await createRepo(app, u.value, { name: "slow", visibility: "public" });
22 if (!made.ok) throw new Error(String(made.error));
23 const repoId = made.value.id;
24 const since = "1970-01-01T00:00:00.000Z";
25 const killed = createRun(app.db, { repoId, kind: "check", env: "verify" });
26 claimRun(app.db, killed.id);
27 finishRun(app.db, killed.id, { state: "failed", summary: "budget: a full gate took 60 min, limit 60" });
28 expect(budgetFailures(app.db, since).map((r) => r.id)).toEqual([killed.id]);
29 // A pass of another environment does not clear it.
30 const other = createRun(app.db, { repoId, kind: "check", env: "" });
31 claimRun(app.db, other.id);
32 finishRun(app.db, other.id, { state: "ok", summary: "ok" });
33 expect(budgetFailures(app.db, since)).toHaveLength(1);
34 await Bun.sleep(5); // real-timer: finished_at is stamped by SQLite at ms resolution, so the later pass must land in a later millisecond
35 const again = createRun(app.db, { repoId, kind: "check", env: "verify" });
36 claimRun(app.db, again.id);
37 finishRun(app.db, again.id, { state: "ok", summary: "ok" });
38 expect(budgetFailures(app.db, since)).toHaveLength(0);
39 } finally {
40 app.close();
41 await rm(dataDir, { recursive: true, force: true });
42 }
43 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs