Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12503
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12499-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 6 commits fork 2 threads · 2 unresolved
5 files
changed +101
−26
src/checks/service.ts +6 −1
| @@ -165,6 +165,8 @@ export interface ChecksOptions { | ||
| 165 | 165 | readonly label?: string | undefined; |
| 166 | 166 | /** How long a check may run before it is killed. @default BUDGETS.gate */ |
| 167 | 167 | readonly budgetMs?: number | undefined; |
| 168 | /** How long a verification of the default branch may run. @default BUDGETS.verify */ | |
| 169 | readonly verifyBudgetMs?: number | undefined; | |
| 168 | 170 | /** |
| 169 | 171 | * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0, |
| 170 | 172 | * nothing held; the forge says `RED_HOLD_MS`. |
| @@ -882,7 +884,10 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 882 | 884 | write(`forge ${options.label} — checked in its parent's lane`); |
| 883 | 885 | const started = Date.now(); |
| 884 | 886 | // The budget: past it the check is aborted, whatever it is doing. |
| 885 | const limit = options.budgetMs ?? BUDGETS.gate; | |
| 887 | const limit = | |
| 888 | run.env === VERIFY_ENV | |
| 889 | ? (options.verifyBudgetMs ?? BUDGETS.verify) | |
| 890 | : (options.budgetMs ?? BUDGETS.gate); | |
| 886 | 891 | const budget = new AbortController(); |
| 887 | 892 | let overran = false; |
| 888 | 893 | const forward = (): void => budget.abort(); |
src/kernel/budgets.ts +8
| @@ -13,6 +13,14 @@ export const BUDGETS = { | ||
| 13 | 13 | * fast tier exists to measure. |
| 14 | 14 | */ |
| 15 | 15 | gate: 60 * 60_000, |
| 16 | /** | |
| 17 | * A verification of the default branch, in ms: the full gate run one file | |
| 18 | * to a process, at the lowest priority, beside whatever else the lane | |
| 19 | * holds, so the impact map is measured. It got through 2061 of 3153 test | |
| 20 | * files in the 60 min of `gate`, twice, and failed on 2026-11-14 (#12499). | |
| 21 | * Landing never waits on it, so it has its own limit, with room to grow. | |
| 22 | */ | |
| 23 | verify: 150 * 60_000, | |
| 16 | 24 | /** A reviewer's run on a pull, in ms. */ |
| 17 | 25 | review: 10 * 60_000, |
| 18 | 26 | /** A check waiting in the lane for its turn, in ms; past it the lane is a cause. */ |
test/check-budget.test.ts +44 −1
| @@ -13,7 +13,7 @@ import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | ||
| 13 | 13 | import { BUDGETS, budgetSummary } from "../src/kernel/budgets.ts"; |
| 14 | 14 | import { vitals } from "../src/kernel/vitals.ts"; |
| 15 | 15 | import { listRuns } from "../src/core/runs.ts"; |
| 16 | import { createUser } from "../src/core/users.ts"; | |
| 16 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 17 | 17 | |
| 18 | 18 | process.env["FORGE_CELLS"] = "off"; |
| 19 | 19 | |
| @@ -28,6 +28,7 @@ beforeAll(async () => { | ||
| 28 | 28 | forgeUrl: "http://forge.test", |
| 29 | 29 | concurrency: 1, |
| 30 | 30 | budgetMs: 400, |
| 31 | verifyBudgetMs: 60_000, | |
| 31 | 32 | }); |
| 32 | 33 | }); |
| 33 | 34 | |
| @@ -93,3 +94,45 @@ test("a check past its budget is re-queued once; the second overrun fails it and | ||
| 93 | 94 | expect(seen.todo.length).toBeGreaterThan(0); |
| 94 | 95 | expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true); |
| 95 | 96 | }, 60_000); |
| 97 | ||
outdated
This passes even if service.ts still uses BUDGETS.gate for verify runs. Drive a verify-env run through startChecks without budgetMs and assert the abort timer uses the verify limit. Keep it fast. |
||
| 98 | test("a verification of main is not cut off at the landing gate's limit (#12499)", async () => { | |
| 99 | const alice = getUserByName(app.db, "alice"); | |
| 100 | if (alice === undefined) throw new Error("alice"); | |
| 101 | const u = { value: alice }; | |
| 102 | const made = await createRepo(app, u.value, { name: "verified", visibility: "public" }); | |
| 103 | if (!made.ok) throw new Error(String(made.error)); | |
| 104 | const repo = made.value; | |
| 105 | // The landing check is quick; the verification outlasts the 400 ms gate limit. | |
| 106 | const pushed = await commitToBranch(app, u.value, repo, { | |
| 107 | message: "A verification slower than the gate limit", | |
| 108 | files: [ | |
| 109 | { | |
| 110 | path: "forge.json", | |
| 111 | content: JSON.stringify({ check: { run: "true", verify: "sleep 1.5", install: "" } }), | |
| 112 | }, | |
| 113 | ], | |
| 114 | }); | |
| 115 | if (!pushed.ok) throw new Error(JSON.stringify(pushed.error)); | |
| 116 | const oid = pushed.value.commitOid; | |
| 117 | const mine = () => | |
| 118 | listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.commitOid === oid); | |
| 119 | const until = async (done: () => boolean, what: string): Promise<void> => { | |
| 120 | const deadline = Date.now() + 30_000; | |
| 121 | while (!done() && Date.now() < deadline) { | |
| 122 | await Bun.sleep(100); // real-timer: polls a real check child | |
| 123 | } | |
| 124 | if (!done()) throw new Error(`timed out waiting for ${what}`); | |
| 125 | }; | |
| 126 | await until(() => mine().some((r) => r.env === repo.defaultBranch && r.state === "ok"), "main's check"); | |
| 127 | const asked = await checks.verifyDue(); | |
| 128 | expect(asked.map((r) => r.env)).toEqual(["verify"]); | |
| 129 | await until(() => mine().some((r) => r.env === "verify" && (r.state === "ok" || r.state === "failed" || r.state === "cancelled")), "the verification"); | |
| 130 | const verify = mine().filter((r) => r.env === "verify"); | |
| 131 | expect(verify).toHaveLength(1); | |
| 132 | expect(verify[0]!.summary).not.toMatch(/^budget:/); | |
| 133 | expect(verify[0]!.state).toBe("ok"); | |
| 134 | }, 60_000); | |
| 135 | ||
| 136 | test("the verify budget is longer than the landing gate's", () => { | |
| 137 | expect(BUDGETS.verify).toBeGreaterThan(BUDGETS.gate); | |
| 138 | }); |
test/scaffold.test.ts +41 −22
| @@ -24,6 +24,7 @@ import { createSession } from "../src/core/sessions.ts"; | ||
| 24 | 24 | import { createUser } from "../src/core/users.ts"; |
| 25 | 25 | import { serve } from "../src/http/server.ts"; |
| 26 | 26 | import { bakedFiles as scaffoldFiles } from "../src/apps/template.ts"; |
| 27 | import { lostPort, onFreshPort } from "../src/kernel/port.ts"; | |
| 27 | 28 | |
| 28 | 29 | /** How long the written-out app's `bun test` may take, wall clock. */ |
| 29 | 30 | const SUITE_BUDGET_MS = 3_000; |
| @@ -208,30 +209,48 @@ describe("a scaffolded app signs a person in through the forge", () => { | ||
| 208 | 209 | }); |
| 209 | 210 | if (!made.ok) throw new Error(made.error); |
| 210 | 211 | const person = `forge_session=${createSession(forge.db, made.value.id).token}`; |
| 211 | const probe = Bun.serve({ port: 0, fetch: () => new Response("") }); | |
| 212 | const port = probe.port; | |
| 213 | probe.stop(true); | |
| 214 | const base = `http://127.0.0.1:${port}`; | |
| 215 | const child = Bun.spawn([process.execPath, "run", "src/server.ts"], { | |
| 216 | cwd: appDir, | |
| 217 | env: { | |
| 218 | PATH: process.env.PATH ?? "/usr/bin:/bin", | |
| 219 | HOME: root, | |
| 220 | PORT: String(port), | |
| 221 | HOST: "127.0.0.1", | |
| 222 | FORGE_APP_DATA: dataDir, | |
| 223 | FORGE_APP_URL: base, | |
| 224 | FORGE_URL: servers.url, | |
| 225 | TMPDIR: temp, | |
| 226 | }, | |
| 227 | stdout: "ignore", | |
| 228 | stderr: "ignore", | |
| 212 | // The port is probed, then bound by the child: on a loaded machine another | |
| 213 | // process can take it in between, and its /healthz is not this app's. So | |
| 214 | // the app is up only when its own child is alive and has registered, and | |
| 215 | // a child that lost its port is started again on a fresh one. | |
| 216 | const registryPath = join(dataDir, "oidc-client.json"); | |
| 217 | const { child, base } = await onFreshPort(async (port) => { | |
| 218 | const base = `http://127.0.0.1:${port}`; | |
| 219 | const child = Bun.spawn([process.execPath, "run", "src/server.ts"], { | |
| 220 | cwd: appDir, | |
| 221 | env: { | |
| 222 | PATH: process.env.PATH ?? "/usr/bin:/bin", | |
| 223 | HOME: root, | |
| 224 | PORT: String(port), | |
| 225 | HOST: "127.0.0.1", | |
| 226 | FORGE_APP_DATA: dataDir, | |
| 227 | FORGE_APP_URL: base, | |
| 228 | FORGE_URL: servers.url, | |
| 229 | TMPDIR: temp, | |
| 230 | }, | |
| 231 | stdout: "ignore", | |
| 232 | stderr: "pipe", | |
| 233 | }); | |
| 234 | let stderr = ""; | |
| 235 | const drained = (async () => { | |
| 236 | for await (const c of child.stderr) stderr = (stderr + new TextDecoder().decode(c)).slice(-4096); | |
| 237 | })(); | |
| 238 | const until = Date.now() + 20_000; | |
| 239 | let up = false; | |
| 240 | while (Date.now() < until && child.exitCode === null && !up) { | |
| 241 | const healthy = await fetch(`${base}/healthz`, { signal: AbortSignal.timeout(2000) }).then((r) => r.ok, () => false); | |
| 242 | up = healthy && child.exitCode === null && (await Bun.file(registryPath).exists()); | |
| 243 | if (!up) await Bun.sleep(100); // real-timer: polls a real child process booting on a real port | |
| 244 | } | |
| 245 | if (!up && child.exitCode !== null) await drained; | |
| 246 | const portLost = !up && lostPort(stderr); | |
| 247 | if (portLost) { | |
| 248 | child.kill(); | |
| 249 | await rm(registryPath, { force: true }); | |
| 250 | } | |
| 251 | return { result: { child, base }, portLost }; | |
| 229 | 252 | }); |
| 230 | 253 | try { |
| 231 | for (let i = 0; i < 100; i++) { | |
| 232 | if ((await fetch(`${base}/healthz`).catch(() => undefined))?.ok) break; | |
| 233 | await Bun.sleep(100); | |
| 234 | } | |
| 235 | 254 | const manual = { redirect: "manual" as const }; |
| 236 | 255 | // It registered itself on first boot, and kept the client id. |
| 237 | 256 | const registry = JSON.parse( |
test/verify-requeued-at-tip.test.ts +2 −2
| @@ -58,8 +58,8 @@ describe("a verification a restart cut off", () => { | ||
| 58 | 58 | if (!third.ok) throw new Error(JSON.stringify(third.error)); |
| 59 | 59 | // No service was up to ask main's own check of it: asked here, as the push would have. |
| 60 | 60 | createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "main", commitOid: third.value.commitOid }); |
| 61 | // The forge that carries on gives a check a third of a second: the held verification overruns. | |
| 62 | f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1, budgetMs: 300 }); | |
| 61 | // The forge that carries on gives a verification a third of a second: the held one overruns. | |
| 62 | f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1, budgetMs: 300, verifyBudgetMs: 300 }); | |
| 63 | 63 | const verifications = (): Run[] => |
| 64 | 64 | listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter((run) => run.env === VERIFY_ENV); |
| 65 | 65 | await f.until(() => verifications().some((run) => run.summary.startsWith(BUDGET_PREFIX)), "the overrun said"); |
new above
offline
Correct wiring, but nothing tests it. See the comment on the test file.