Public: anyone can read this; the crew can change it.

Sentinel: One failure is stopping many pulls: the forge's, not theirs #12588

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12585-sentinel-one-failure-is-stopping-many-pu → main 3 commits fork 3 threads · 3 unresolved

  • Merged. torque/12585-sentinel-one-failure-is-stopping-many-pu was merged into main as ad943ed.

Review

Sign in to review this change.
4 files changed +132 −10
src/checks/service.ts +14 −2
@@ -51,6 +51,9 @@ import {
5151 CPU_STOP_SUMMARY,
5252 DISK_USED_SUMMARY,
5353 requeueCheck,
54 chainCutOffs,
55 CHECK_RELEASE_LIMIT,
56 RELEASE_SUMMARY,
5457 REQUEUED_SUMMARY,
5558 sweepAbandonedChecks,
5659 type Run,
@@ -965,9 +968,18 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
965968 if (!outcome.ok && stopped && signal.aborted) {
966969 // Cut off by this forge stopping, not by the check: the same check
967970 // goes back on the queue, and the forge that carries on runs it.
968 const moved = requeueCheck(app.db, run.id, undefined, REQUEUED_SUMMARY, againAt());
971 // Past the release ceiling the shutdown is what this check hangs or
972 // slows: it fails, saying so, instead of riding every release.
973 if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) {
974 const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`;
975 write(summary);
976 const failed = finishRun(app.db, run.id, { state: "failed", summary });
977 if (failed) announceRun(app, failed);
978 return;
979 }
980 const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt());
969981 if (!moved) return;
970 write(REQUEUED_SUMMARY);
982 write(RELEASE_SUMMARY);
971983 announceRun(app, moved.again, run.actorId ?? undefined);
972984 return;
973985 }
src/core/runs.ts +29 −4
@@ -626,9 +626,19 @@ export const CPU_STOP_SUMMARY = "failed: infrastructure — stopped for CPU by t
626626 /** What a check says as it is queued again because the disk was 80% used (#11799): the machine's wait, not the pull's. */
627627 export const DISK_USED_SUMMARY = "waiting: the forge's disk is 80% used; queued again when there is room";
628628
629 /** What a check cut off by its forge being stopped on purpose (a release) says when requeued. */
630 export const RELEASE_SUMMARY = "interrupted by a release; queued again";
631
629632 /**
630 * How many times one check may be cut off by its forge going away and be
631 * queued again. Past it the check fails once, saying so: a check that kills
633 * How many releases one check may be cut off by, across its chain, before
634 * it fails: a release is not the check's fault, but one that outlasts this
635 * many is hanging or slowing the shutdown itself.
636 */
637 export const CHECK_RELEASE_LIMIT = 20;
638
639 /**
640 * How many times one check may be cut off by its forge dying (a lapsed lease)
641 * and be queued again. Past it the check fails once, saying so: a check that kills
632642 * its forge must not loop forever.
633643 */
634644 export const CHECK_RESTART_LIMIT = 3;
@@ -636,7 +646,7 @@ export const CHECK_RESTART_LIMIT = 3;
636646 /**
outdated

Over-long line, and the issue number does not belong in a doc comment. Also update the earlier 'attempt + 1 — the one way' sentence, which is now false.

637647 * Cancel a check the forge's own going-away cut off and queue the same
638648 * check again, `attempt + 1` — the one way a check is retried, from a
639 * graceful stop or a lapsed lease alike. Only a run still `running` is
649 * release or a lapsed lease alike; the summary says which. Only a run still `running` is
640650 * moved, and `lapsedBefore`, when given, must still be true of its lease:
641651 * the UPDATE's WHERE is what decides between two forges, so only one of
642652 * them ever queues the retry. Returns the cancelled run and its retry, or
@@ -676,6 +686,18 @@ export function requeueCheck(
676686 })();
677687 }
678688
689 /** How many earlier attempts of this check's chain were cut off with `summary`
690 * (same commit, reason and issue). */
691 export function chainCutOffs(db: Database, run: Run, summary: string): number {
692 return db
693 .query<{ n: number }, [string, string, string, string, number, string]>(
694 `SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND repo_id = ?1 AND commit_oid = ?2
695 AND reason = ?3 AND COALESCE(issue_id, '') = ?4 AND attempt < ?5
696 AND state = 'cancelled' AND summary = ?6`,
697 )
698 .get(run.repoId, run.commitOid, run.reason, run.issueId ?? "", run.attempt, summary)!.n;
699 }
700
679701 /**
680702 * Whether the run before this one in its chain (the attempt just below, same
681703 * kind, repository, reason and issue) was killed for its budget: the second
@@ -791,7 +813,10 @@ export function sweepAbandonedChecks(
791813 "the forge went away, and this commit no longer heads an open pull",
792814 );
793815 if (done) out.cancelled.push(done);
794 } else if (row.attempt >= CHECK_RESTART_LIMIT) {
816 } else if (
817 // Releases are the forge's doing: only deaths spend the allowance.
818 row.attempt - chainCutOffs(db, toRun(row), RELEASE_SUMMARY) >= CHECK_RESTART_LIMIT
819 ) {
795820 const done = settle(
796821 row.id,
797822 "failed",
test/checks-restart.test.ts +87 −2
@@ -5,23 +5,31 @@
55 * (`requeueAbandonedChecks`), and the boot sweep of checkouts nobody runs.
66 */
77
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
99 import { existsSync, mkdirSync } from "node:fs";
1010 import { mkdtemp, rm } from "node:fs/promises";
1111 import { tmpdir } from "node:os";
1212 import { join } from "node:path";
13 import { BUDGET_PREFIX } from "../src/kernel/budgets.ts";
1314 import { createApp, type App } from "../src/app.ts";
1415 import { openPull } from "../src/actions/pulls.ts";
1516 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
1617 import { requeueAbandonedChecks, startChecks, sweepSoon } from "../src/checks/service.ts";
1718 import {
1819 cancelRun,
20 CHECK_RELEASE_LIMIT,
1921 CHECK_RESTART_LIMIT,
22 chainCutOffs,
23 overranBefore,
24 RELEASE_SUMMARY,
25 vanishedBefore,
26 VANISHED_SUMMARY,
2027 claimRun,
2128 createRun,
2229 getRun,
2330 listRuns,
2431 REQUEUED_SUMMARY,
32 requeueCheck,
2533 type Run,
2634 } from "../src/core/runs.ts";
2735 import type { Pull } from "../src/core/pulls.ts";
@@ -52,7 +60,7 @@ describe("checks", () => {
5260 const runs = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 });
5361 const interrupted = runs.find((r) => r.id === cut!.id);
5462 expect(interrupted?.state).toBe("cancelled");
55 expect(interrupted?.summary).toContain("queued again");
63 expect(interrupted?.summary).toBe(RELEASE_SUMMARY);
5664 const again = runs.find((r) => r.commitOid === oid && r.state === "queued");
5765 expect(again?.attempt).toBe(1);
5866
@@ -107,6 +115,28 @@ describe("checks", () => {
107115 const done = await f.settled((r) => r.id === again?.id);
108116 expect(done.state).toBe("ok");
109117 });
118 test("a check that outlasts the release ceiling fails once, saying the forge was stopped under it", async () => {
119 // Invariant: releases are counted apart, but not without end: past CHECK_RELEASE_LIMIT a
120 // stop fails the check.
121 const id = await f.slowCheck(f.held("ceiling"), "A check every release cuts");
122 const run = getRun(f.app.db, id)!;
123 for (let i = 0; i < CHECK_RELEASE_LIMIT; i++) {
124 const old = createRun(f.app.db, {
125 repoId: f.repo.id, kind: "check", env: run.env, commitOid: run.commitOid,
126 reason: run.reason, attempt: i, ...(run.issueId === null ? {} : { issueId: run.issueId }),
127 });
128 f.app.db.query("UPDATE run SET state = 'cancelled', summary = ?2 WHERE id = ?1").run(old.id, RELEASE_SUMMARY);
129 }
130 f.app.db.query("UPDATE run SET attempt = ?2 WHERE id = ?1").run(id, CHECK_RELEASE_LIMIT);
131 await f.checks.stop({ drain: false });
132 const done = getRun(f.app.db, id)!;
133 expect(done.state).toBe("failed");
134 expect(done.summary).toContain("the forge was stopped under this check");
135 expect(listRuns(f.app.db, f.repo.id, { kind: "check", limit: 100 }).some((r) => r.state === "queued")).toBe(false);
136 f.release("ceiling");
137 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
138 });
139
110140 test("an open pull whose head was never checked is checked when the service boots", async () => {
111141 // Invariant: a booting service checks the head of an open pull that has no check, with reason boot.
112142 await f.checks.stop();
@@ -204,6 +234,11 @@ describe("a check its forge died under", () => {
204234 await rm(dataDir, { recursive: true, force: true });
205235 });
206236
237 // Each test's chain of attempts stands alone.
238 beforeEach(() => {
239 app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repo.id);
240 });
241
207242 /** A check that was running when its forge died: claimed, lease long lapsed. */
208243 const abandoned = (commitOid: string, attempt = 0): Run => {
209244 const run = createRun(app.db, {
@@ -258,6 +293,56 @@ describe("a check its forge died under", () => {
258293 cancelRun(app.db, again[0]!.id);
259294 });
260295
296 /** `n` earlier attempts of this pull's check, each cut off with `summary`. */
297 const history = (n: number, summary: string): void => {
298 for (let i = 0; i < n; i++) {
299 const old = createRun(app.db, {
300 repoId: repo.id, kind: "check", env: `pull-${pull.number}`, commitOid: pull.headOid,
301 reason: "push", issueId: pull.id, attempt: i,
302 });
303 app.db.query("UPDATE run SET state = 'cancelled', summary = ?2 WHERE id = ?1").run(old.id, summary);
304 }
305 };
306
307 test("releases do not spend the allowance: a check cut by many releases and one death is queued again", () => {
308 history(CHECK_RESTART_LIMIT + 5, RELEASE_SUMMARY);
309 const run = abandoned(pull.headOid, CHECK_RESTART_LIMIT + 5);
310 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
311 expect(getRun(app.db, run.id)?.state).toBe("cancelled");
312 expect(queuedFor(pull.headOid)).toHaveLength(1);
313 for (const q of queuedFor(pull.headOid)) cancelRun(app.db, q.id);
314 });
315
316 test("deaths still spend it: after releases, CHECK_RESTART_LIMIT lapses fail the check", () => {
317 history(4, RELEASE_SUMMARY);
318 const deaths = CHECK_RESTART_LIMIT;
319 // attempts 4..6 were deaths (summary REQUEUED_SUMMARY), this is attempt 7.
320 const run = abandoned(pull.headOid, 4 + deaths);
321 expect(chainCutOffs(app.db, run, RELEASE_SUMMARY)).toBeGreaterThanOrEqual(4);
322 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
323 expect(getRun(app.db, run.id)?.state).toBe("failed");
324 });
325
326 test("the once-only lookups read a release's retry as a different attempt from its parent", () => {
327 // A budget cut at attempt 0 makes attempt 1 the second overrun; a release at attempt 0 does
328 // not.
329 const cut = (summary: string) => {
330 app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repo.id);
331 const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "x", commitOid: pull.headOid, reason: "push", issueId: pull.id });
332 const moved = requeueCheck(app.db, claimRun(app.db, a.id)!.id, undefined, summary);
333 return moved!;
334 };
335 const overrun = cut(`${BUDGET_PREFIX} a full gate took 61 min, limit 60`);
336 expect(overrun.again.attempt).toBe(overrun.cancelled.attempt + 1);
337 expect(overranBefore(app.db, overrun.again)).toBe(true);
338 const released = cut(RELEASE_SUMMARY);
339 expect(released.again.attempt).toBe(released.cancelled.attempt + 1);
340 expect(overranBefore(app.db, released.again)).toBe(false);
341 expect(vanishedBefore(app.db, released.again)).toBe(false);
342 const vanished = cut(VANISHED_SUMMARY);
343 expect(vanishedBefore(app.db, vanished.again)).toBe(true);
344 });
345
261346 test("a check whose lease is still held is left alone", () => {
262347 const run = createRun(app.db, {
263348 repoId: repo.id,
test/verify-requeued-at-tip.test.ts +2 −2
@@ -10,7 +10,7 @@ import { join } from "node:path";
1010 import { commitToBranch } from "../src/actions/repos.ts";
1111 import { startChecks } from "../src/checks/service.ts";
1212 import { VERIFY_ENV } from "../src/checks/verify.ts";
13 import { claimRun, createRun, getRun, listRuns, requeueCheck, REQUEUED_SUMMARY, type Run } from "../src/core/runs.ts";
13 import { claimRun, createRun, getRun, listRuns, requeueCheck, RELEASE_SUMMARY, type Run } from "../src/core/runs.ts";
1414 import { BUDGET_PREFIX } from "../src/kernel/budgets.ts";
1515 import { checksForge } from "./helpers/checks-forge.ts";
1616
@@ -41,7 +41,7 @@ describe("a verification a restart cut off", () => {
4141
4242 // The forge stops: a release.
4343 await f.checks.stop();
44 expect(getRun(f.app.db, asked!.id)).toMatchObject({ state: "cancelled", summary: REQUEUED_SUMMARY });
44 expect(getRun(f.app.db, asked!.id)).toMatchObject({ state: "cancelled", summary: RELEASE_SUMMARY });
4545 const again = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter(
4646 (run) => run.env === VERIFY_ENV && run.state === "queued",
4747 );

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs