Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12603
scout/forge:scout/12601-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 3 commits fork
2 files
changed +53
−2
src/core/runs.ts +10 −2
| @@ -743,12 +743,20 @@ function cutOffBefore(db: Database, run: Run, prefix: string): boolean { | ||
| 743 | 743 | ); |
| 744 | 744 | } |
| 745 | 745 | |
| 746 | /** Failed runs the budget killed since `since`: each is a card in the vitals. */ | |
| 746 | /** | |
| 747 | * Failed runs the budget killed since `since`: each is a card in the vitals, | |
| 748 | * until a later run of the same kind, repo and environment has passed: the | |
| 749 | * cause is fixed then, and a day's wait is not the cure. | |
| 750 | */ | |
| 747 | 751 | export function budgetFailures(db: Database, since: string): Run[] { |
| 748 | 752 | return db |
| 749 | 753 | .query<RunRow, [string, string]>( |
| 750 | 754 | `SELECT * FROM run INDEXED BY run_failed_finished WHERE state = 'failed' AND summary LIKE ?2 || '%' |
| 751 | AND finished_at >= ?1 ORDER BY finished_at DESC LIMIT 20`, | |
| 755 | AND finished_at >= ?1 | |
| 756 | AND NOT EXISTS (SELECT 1 FROM run later WHERE later.kind = run.kind | |
| 757 | AND later.repo_id = run.repo_id AND later.env = run.env | |
| 758 | AND later.state = 'ok' AND later.finished_at > run.finished_at) | |
| 759 | ORDER BY finished_at DESC LIMIT 20`, | |
| 752 | 760 | ) |
| 753 | 761 | .all(since, BUDGET_PREFIX) |
| 754 | 762 | .map(toRun); |
test/budget-card-cleared.test.ts added +43
| @@ -0,0 +1,43 @@ | ||
| 1 | /** A run killed for its budget is a card until a later run of its kind passes. */ | |
| 2 | ||
| 3 | import { expect, test } from "bun:test"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp } from "../src/app.ts"; | |
| 8 | import { createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { budgetFailures, claimRun, createRun, finishRun } from "../src/core/runs.ts"; | |
| 11 | ||
| 12 | test("a later passing run of the same kind clears the budget card", async () => { | |
| 13 | const dataDir = await mkdtemp(join(tmpdir(), "forge-budget-card-")); | |
| 14 | const app = createApp({ dataDir, port: 0 }); | |
| 15 | try { | |
| 16 | const u = await createUser(app.db, { | |
| 17 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 18 | password: "a-good-passphrase", adminIfFirst: true, | |
| 19 | }); | |
| 20 | if (!u.ok) throw new Error(u.error); | |
| 21 | const made = await createRepo(app, u.value, { name: "slow", visibility: "public" }); | |
| 22 | if (!made.ok) throw new Error(String(made.error)); | |
| 23 | const repoId = made.value.id; | |
| 24 | const since = "1970-01-01T00:00:00.000Z"; | |
| 25 | const killed = createRun(app.db, { repoId, kind: "check", env: "verify" }); | |
| 26 | claimRun(app.db, killed.id); | |
| 27 | finishRun(app.db, killed.id, { state: "failed", summary: "budget: a full gate took 60 min, limit 60" }); | |
| 28 | expect(budgetFailures(app.db, since).map((r) => r.id)).toEqual([killed.id]); | |
| 29 | // A pass of another environment does not clear it. | |
| 30 | const other = createRun(app.db, { repoId, kind: "check", env: "" }); | |
| 31 | claimRun(app.db, other.id); | |
| 32 | finishRun(app.db, other.id, { state: "ok", summary: "ok" }); | |
| 33 | expect(budgetFailures(app.db, since)).toHaveLength(1); | |
| 34 | await Bun.sleep(5); // real-timer: finished_at is stamped by SQLite at ms resolution, so the later pass must land in a later millisecond | |
| 35 | const again = createRun(app.db, { repoId, kind: "check", env: "verify" }); | |
| 36 | claimRun(app.db, again.id); | |
| 37 | finishRun(app.db, again.id, { state: "ok", summary: "ok" }); | |
| 38 | expect(budgetFailures(app.db, since)).toHaveLength(0); | |
| 39 | } finally { | |
| 40 | app.close(); | |
| 41 | await rm(dataDir, { recursive: true, force: true }); | |
| 42 | } | |
| 43 | }); |
new above
offline