Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7cph78n2493t8as2avbd1re is past its budget: A run was killed twice: budget: a full gate took 60 min, limit 60. #12503

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12499-sentinel-budget-01m7cph78n2493t8as2avbd1 → main 6 commits fork 2 threads · 2 unresolved

  • Merged. mason/12499-sentinel-budget-01m7cph78n2493t8as2avbd1 was merged into main as 176bd8e.

Review

Sign in to review this change.
5 files changed +101 −26
src/checks/service.ts +6 −1
@@ -165,6 +165,8 @@ export interface ChecksOptions {
165165 readonly label?: string | undefined;
166166 /** How long a check may run before it is killed. @default BUDGETS.gate */
167167 readonly budgetMs?: number | undefined;
168 /** How long a verification of the default branch may run. @default BUDGETS.verify */
169 readonly verifyBudgetMs?: number | undefined;
168170 /**
169171 * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0,
170172 * nothing held; the forge says `RED_HOLD_MS`.
@@ -882,7 +884,10 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
882884 write(`forge ${options.label} — checked in its parent's lane`);
883885 const started = Date.now();
884886 // The budget: past it the check is aborted, whatever it is doing.
outdated

Correct wiring, but nothing tests it. See the comment on the test file.

885 const limit = options.budgetMs ?? BUDGETS.gate;
887 const limit =
888 run.env === VERIFY_ENV
889 ? (options.verifyBudgetMs ?? BUDGETS.verify)
890 : (options.budgetMs ?? BUDGETS.gate);
886891 const budget = new AbortController();
887892 let overran = false;
888893 const forward = (): void => budget.abort();
src/kernel/budgets.ts +8
@@ -13,6 +13,14 @@ export const BUDGETS = {
1313 * fast tier exists to measure.
1414 */
1515 gate: 60 * 60_000,
16 /**
17 * A verification of the default branch, in ms: the full gate run one file
18 * to a process, at the lowest priority, beside whatever else the lane
19 * holds, so the impact map is measured. It got through 2061 of 3153 test
20 * files in the 60 min of `gate`, twice, and failed on 2026-11-14 (#12499).
21 * Landing never waits on it, so it has its own limit, with room to grow.
22 */
23 verify: 150 * 60_000,
1624 /** A reviewer's run on a pull, in ms. */
1725 review: 10 * 60_000,
1826 /** A check waiting in the lane for its turn, in ms; past it the lane is a cause. */
test/check-budget.test.ts +44 −1
@@ -13,7 +13,7 @@ import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
1313 import { BUDGETS, budgetSummary } from "../src/kernel/budgets.ts";
1414 import { vitals } from "../src/kernel/vitals.ts";
1515 import { listRuns } from "../src/core/runs.ts";
16 import { createUser } from "../src/core/users.ts";
16 import { createUser, getUserByName } from "../src/core/users.ts";
1717
1818 process.env["FORGE_CELLS"] = "off";
1919
@@ -28,6 +28,7 @@ beforeAll(async () => {
2828 forgeUrl: "http://forge.test",
2929 concurrency: 1,
3030 budgetMs: 400,
31 verifyBudgetMs: 60_000,
3132 });
3233 });
3334
@@ -93,3 +94,45 @@ test("a check past its budget is re-queued once; the second overrun fails it and
9394 expect(seen.todo.length).toBeGreaterThan(0);
9495 expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
9596 }, 60_000);
97
outdated

This passes even if service.ts still uses BUDGETS.gate for verify runs. Drive a verify-env run through startChecks without budgetMs and assert the abort timer uses the verify limit. Keep it fast.

98 test("a verification of main is not cut off at the landing gate's limit (#12499)", async () => {
99 const alice = getUserByName(app.db, "alice");
100 if (alice === undefined) throw new Error("alice");
101 const u = { value: alice };
102 const made = await createRepo(app, u.value, { name: "verified", visibility: "public" });
103 if (!made.ok) throw new Error(String(made.error));
104 const repo = made.value;
105 // The landing check is quick; the verification outlasts the 400 ms gate limit.
106 const pushed = await commitToBranch(app, u.value, repo, {
107 message: "A verification slower than the gate limit",
108 files: [
109 {
110 path: "forge.json",
111 content: JSON.stringify({ check: { run: "true", verify: "sleep 1.5", install: "" } }),
112 },
113 ],
114 });
115 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
116 const oid = pushed.value.commitOid;
117 const mine = () =>
118 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.commitOid === oid);
119 const until = async (done: () => boolean, what: string): Promise<void> => {
120 const deadline = Date.now() + 30_000;
121 while (!done() && Date.now() < deadline) {
122 await Bun.sleep(100); // real-timer: polls a real check child
123 }
124 if (!done()) throw new Error(`timed out waiting for ${what}`);
125 };
126 await until(() => mine().some((r) => r.env === repo.defaultBranch && r.state === "ok"), "main's check");
127 const asked = await checks.verifyDue();
128 expect(asked.map((r) => r.env)).toEqual(["verify"]);
129 await until(() => mine().some((r) => r.env === "verify" && (r.state === "ok" || r.state === "failed" || r.state === "cancelled")), "the verification");
130 const verify = mine().filter((r) => r.env === "verify");
131 expect(verify).toHaveLength(1);
132 expect(verify[0]!.summary).not.toMatch(/^budget:/);
133 expect(verify[0]!.state).toBe("ok");
134 }, 60_000);
135
136 test("the verify budget is longer than the landing gate's", () => {
137 expect(BUDGETS.verify).toBeGreaterThan(BUDGETS.gate);
138 });
test/scaffold.test.ts +41 −22
@@ -24,6 +24,7 @@ import { createSession } from "../src/core/sessions.ts";
2424 import { createUser } from "../src/core/users.ts";
2525 import { serve } from "../src/http/server.ts";
2626 import { bakedFiles as scaffoldFiles } from "../src/apps/template.ts";
27 import { lostPort, onFreshPort } from "../src/kernel/port.ts";
2728
2829 /** How long the written-out app's `bun test` may take, wall clock. */
2930 const SUITE_BUDGET_MS = 3_000;
@@ -208,30 +209,48 @@ describe("a scaffolded app signs a person in through the forge", () => {
208209 });
209210 if (!made.ok) throw new Error(made.error);
210211 const person = `forge_session=${createSession(forge.db, made.value.id).token}`;
211 const probe = Bun.serve({ port: 0, fetch: () => new Response("") });
212 const port = probe.port;
213 probe.stop(true);
214 const base = `http://127.0.0.1:${port}`;
215 const child = Bun.spawn([process.execPath, "run", "src/server.ts"], {
216 cwd: appDir,
217 env: {
218 PATH: process.env.PATH ?? "/usr/bin:/bin",
219 HOME: root,
220 PORT: String(port),
221 HOST: "127.0.0.1",
222 FORGE_APP_DATA: dataDir,
223 FORGE_APP_URL: base,
224 FORGE_URL: servers.url,
225 TMPDIR: temp,
226 },
227 stdout: "ignore",
228 stderr: "ignore",
212 // The port is probed, then bound by the child: on a loaded machine another
213 // process can take it in between, and its /healthz is not this app's. So
214 // the app is up only when its own child is alive and has registered, and
215 // a child that lost its port is started again on a fresh one.
216 const registryPath = join(dataDir, "oidc-client.json");
217 const { child, base } = await onFreshPort(async (port) => {
218 const base = `http://127.0.0.1:${port}`;
219 const child = Bun.spawn([process.execPath, "run", "src/server.ts"], {
220 cwd: appDir,
221 env: {
222 PATH: process.env.PATH ?? "/usr/bin:/bin",
223 HOME: root,
224 PORT: String(port),
225 HOST: "127.0.0.1",
226 FORGE_APP_DATA: dataDir,
227 FORGE_APP_URL: base,
228 FORGE_URL: servers.url,
229 TMPDIR: temp,
230 },
231 stdout: "ignore",
232 stderr: "pipe",
233 });
234 let stderr = "";
235 const drained = (async () => {
236 for await (const c of child.stderr) stderr = (stderr + new TextDecoder().decode(c)).slice(-4096);
237 })();
238 const until = Date.now() + 20_000;
239 let up = false;
240 while (Date.now() < until && child.exitCode === null && !up) {
241 const healthy = await fetch(`${base}/healthz`, { signal: AbortSignal.timeout(2000) }).then((r) => r.ok, () => false);
242 up = healthy && child.exitCode === null && (await Bun.file(registryPath).exists());
243 if (!up) await Bun.sleep(100); // real-timer: polls a real child process booting on a real port
244 }
245 if (!up && child.exitCode !== null) await drained;
246 const portLost = !up && lostPort(stderr);
247 if (portLost) {
248 child.kill();
249 await rm(registryPath, { force: true });
250 }
251 return { result: { child, base }, portLost };
229252 });
230253 try {
231 for (let i = 0; i < 100; i++) {
232 if ((await fetch(`${base}/healthz`).catch(() => undefined))?.ok) break;
233 await Bun.sleep(100);
234 }
235254 const manual = { redirect: "manual" as const };
236255 // It registered itself on first boot, and kept the client id.
237256 const registry = JSON.parse(
test/verify-requeued-at-tip.test.ts +2 −2
@@ -58,8 +58,8 @@ describe("a verification a restart cut off", () => {
5858 if (!third.ok) throw new Error(JSON.stringify(third.error));
5959 // No service was up to ask main's own check of it: asked here, as the push would have.
6060 createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "main", commitOid: third.value.commitOid });
61 // The forge that carries on gives a check a third of a second: the held verification overruns.
62 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1, budgetMs: 300 });
61 // The forge that carries on gives a verification a third of a second: the held one overruns.
62 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1, budgetMs: 300, verifyBudgetMs: 300 });
6363 const verifications = (): Run[] =>
6464 listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter((run) => run.env === VERIFY_ENV);
6565 await f.until(() => verifications().some((run) => run.summary.startsWith(BUDGET_PREFIX)), "the overrun said");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs