Public: anyone can read this; the crew can change it.
Sentinel: One failure is stopping many pulls: the forge's, not theirs #12588
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12585-sentinel-one-failure-is-stopping-many-pu → main 3 commits fork 3 threads · 3 unresolved
4 files
changed +132
−10
src/checks/service.ts +14 −2
| @@ -51,6 +51,9 @@ import { | ||
| 51 | 51 | CPU_STOP_SUMMARY, |
| 52 | 52 | DISK_USED_SUMMARY, |
| 53 | 53 | requeueCheck, |
| 54 | chainCutOffs, | |
| 55 | CHECK_RELEASE_LIMIT, | |
| 56 | RELEASE_SUMMARY, | |
| 54 | 57 | REQUEUED_SUMMARY, |
| 55 | 58 | sweepAbandonedChecks, |
| 56 | 59 | type Run, |
| @@ -965,9 +968,18 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 965 | 968 | if (!outcome.ok && stopped && signal.aborted) { |
| 966 | 969 | // Cut off by this forge stopping, not by the check: the same check |
| 967 | 970 | // goes back on the queue, and the forge that carries on runs it. |
| 968 | const moved = requeueCheck(app.db, run.id, undefined, REQUEUED_SUMMARY, againAt()); | |
| 971 | // Past the release ceiling the shutdown is what this check hangs or | |
| 972 | // slows: it fails, saying so, instead of riding every release. | |
| 973 | if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) { | |
| 974 | const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`; | |
| 975 | write(summary); | |
| 976 | const failed = finishRun(app.db, run.id, { state: "failed", summary }); | |
| 977 | if (failed) announceRun(app, failed); | |
| 978 | return; | |
| 979 | } | |
| 980 | const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt()); | |
| 969 | 981 | if (!moved) return; |
| 970 | write(REQUEUED_SUMMARY); | |
| 982 | write(RELEASE_SUMMARY); | |
| 971 | 983 | announceRun(app, moved.again, run.actorId ?? undefined); |
| 972 | 984 | return; |
| 973 | 985 | } |
src/core/runs.ts +29 −4
| @@ -626,9 +626,19 @@ export const CPU_STOP_SUMMARY = "failed: infrastructure — stopped for CPU by t | ||
| 626 | 626 | /** What a check says as it is queued again because the disk was 80% used (#11799): the machine's wait, not the pull's. */ |
| 627 | 627 | export const DISK_USED_SUMMARY = "waiting: the forge's disk is 80% used; queued again when there is room"; |
| 628 | 628 | |
| 629 | /** What a check cut off by its forge being stopped on purpose (a release) says when requeued. */ | |
| 630 | export const RELEASE_SUMMARY = "interrupted by a release; queued again"; | |
| 631 | ||
| 629 | 632 | /** |
| 630 | * How many times one check may be cut off by its forge going away and be | |
| 631 | * queued again. Past it the check fails once, saying so: a check that kills | |
| 633 | * How many releases one check may be cut off by, across its chain, before | |
| 634 | * it fails: a release is not the check's fault, but one that outlasts this | |
| 635 | * many is hanging or slowing the shutdown itself. | |
| 636 | */ | |
| 637 | export const CHECK_RELEASE_LIMIT = 20; | |
| 638 | ||
| 639 | /** | |
| 640 | * How many times one check may be cut off by its forge dying (a lapsed lease) | |
| 641 | * and be queued again. Past it the check fails once, saying so: a check that kills | |
| 632 | 642 | * its forge must not loop forever. |
| 633 | 643 | */ |
| 634 | 644 | export const CHECK_RESTART_LIMIT = 3; |
| @@ -636,7 +646,7 @@ export const CHECK_RESTART_LIMIT = 3; | ||
| 636 | 646 | /** |
| 637 | 647 | * Cancel a check the forge's own going-away cut off and queue the same |
| 638 | 648 | * check again, `attempt + 1` — the one way a check is retried, from a |
| 639 | * graceful stop or a lapsed lease alike. Only a run still `running` is | |
| 649 | * release or a lapsed lease alike; the summary says which. Only a run still `running` is | |
| 640 | 650 | * moved, and `lapsedBefore`, when given, must still be true of its lease: |
| 641 | 651 | * the UPDATE's WHERE is what decides between two forges, so only one of |
| 642 | 652 | * them ever queues the retry. Returns the cancelled run and its retry, or |
| @@ -676,6 +686,18 @@ export function requeueCheck( | ||
| 676 | 686 | })(); |
| 677 | 687 | } |
| 678 | 688 | |
| 689 | /** How many earlier attempts of this check's chain were cut off with `summary` | |
| 690 | * (same commit, reason and issue). */ | |
| 691 | export function chainCutOffs(db: Database, run: Run, summary: string): number { | |
| 692 | return db | |
| 693 | .query<{ n: number }, [string, string, string, string, number, string]>( | |
| 694 | `SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND repo_id = ?1 AND commit_oid = ?2 | |
| 695 | AND reason = ?3 AND COALESCE(issue_id, '') = ?4 AND attempt < ?5 | |
| 696 | AND state = 'cancelled' AND summary = ?6`, | |
| 697 | ) | |
| 698 | .get(run.repoId, run.commitOid, run.reason, run.issueId ?? "", run.attempt, summary)!.n; | |
| 699 | } | |
| 700 | ||
| 679 | 701 | /** |
| 680 | 702 | * Whether the run before this one in its chain (the attempt just below, same |
| 681 | 703 | * kind, repository, reason and issue) was killed for its budget: the second |
| @@ -791,7 +813,10 @@ export function sweepAbandonedChecks( | ||
| 791 | 813 | "the forge went away, and this commit no longer heads an open pull", |
| 792 | 814 | ); |
| 793 | 815 | if (done) out.cancelled.push(done); |
| 794 | } else if (row.attempt >= CHECK_RESTART_LIMIT) { | |
| 816 | } else if ( | |
| 817 | // Releases are the forge's doing: only deaths spend the allowance. | |
| 818 | row.attempt - chainCutOffs(db, toRun(row), RELEASE_SUMMARY) >= CHECK_RESTART_LIMIT | |
| 819 | ) { | |
| 795 | 820 | const done = settle( |
| 796 | 821 | row.id, |
| 797 | 822 | "failed", |
test/checks-restart.test.ts +87 −2
| @@ -5,23 +5,31 @@ | ||
| 5 | 5 | * (`requeueAbandonedChecks`), and the boot sweep of checkouts nobody runs. |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 8 | import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; | |
| 9 | 9 | import { existsSync, mkdirSync } from "node:fs"; |
| 10 | 10 | import { mkdtemp, rm } from "node:fs/promises"; |
| 11 | 11 | import { tmpdir } from "node:os"; |
| 12 | 12 | import { join } from "node:path"; |
| 13 | import { BUDGET_PREFIX } from "../src/kernel/budgets.ts"; | |
| 13 | 14 | import { createApp, type App } from "../src/app.ts"; |
| 14 | 15 | import { openPull } from "../src/actions/pulls.ts"; |
| 15 | 16 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 16 | 17 | import { requeueAbandonedChecks, startChecks, sweepSoon } from "../src/checks/service.ts"; |
| 17 | 18 | import { |
| 18 | 19 | cancelRun, |
| 20 | CHECK_RELEASE_LIMIT, | |
| 19 | 21 | CHECK_RESTART_LIMIT, |
| 22 | chainCutOffs, | |
| 23 | overranBefore, | |
| 24 | RELEASE_SUMMARY, | |
| 25 | vanishedBefore, | |
| 26 | VANISHED_SUMMARY, | |
| 20 | 27 | claimRun, |
| 21 | 28 | createRun, |
| 22 | 29 | getRun, |
| 23 | 30 | listRuns, |
| 24 | 31 | REQUEUED_SUMMARY, |
| 32 | requeueCheck, | |
| 25 | 33 | type Run, |
| 26 | 34 | } from "../src/core/runs.ts"; |
| 27 | 35 | import type { Pull } from "../src/core/pulls.ts"; |
| @@ -52,7 +60,7 @@ describe("checks", () => { | ||
| 52 | 60 | const runs = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }); |
| 53 | 61 | const interrupted = runs.find((r) => r.id === cut!.id); |
| 54 | 62 | expect(interrupted?.state).toBe("cancelled"); |
| 55 | expect(interrupted?.summary).toContain("queued again"); | |
| 63 | expect(interrupted?.summary).toBe(RELEASE_SUMMARY); | |
| 56 | 64 | const again = runs.find((r) => r.commitOid === oid && r.state === "queued"); |
| 57 | 65 | expect(again?.attempt).toBe(1); |
| 58 | 66 | |
| @@ -107,6 +115,28 @@ describe("checks", () => { | ||
| 107 | 115 | const done = await f.settled((r) => r.id === again?.id); |
| 108 | 116 | expect(done.state).toBe("ok"); |
| 109 | 117 | }); |
| 118 | test("a check that outlasts the release ceiling fails once, saying the forge was stopped under it", async () => { | |
| 119 | // Invariant: releases are counted apart, but not without end: past CHECK_RELEASE_LIMIT a | |
| 120 | // stop fails the check. | |
| 121 | const id = await f.slowCheck(f.held("ceiling"), "A check every release cuts"); | |
| 122 | const run = getRun(f.app.db, id)!; | |
| 123 | for (let i = 0; i < CHECK_RELEASE_LIMIT; i++) { | |
| 124 | const old = createRun(f.app.db, { | |
| 125 | repoId: f.repo.id, kind: "check", env: run.env, commitOid: run.commitOid, | |
| 126 | reason: run.reason, attempt: i, ...(run.issueId === null ? {} : { issueId: run.issueId }), | |
| 127 | }); | |
| 128 | f.app.db.query("UPDATE run SET state = 'cancelled', summary = ?2 WHERE id = ?1").run(old.id, RELEASE_SUMMARY); | |
| 129 | } | |
| 130 | f.app.db.query("UPDATE run SET attempt = ?2 WHERE id = ?1").run(id, CHECK_RELEASE_LIMIT); | |
| 131 | await f.checks.stop({ drain: false }); | |
| 132 | const done = getRun(f.app.db, id)!; | |
| 133 | expect(done.state).toBe("failed"); | |
| 134 | expect(done.summary).toContain("the forge was stopped under this check"); | |
| 135 | expect(listRuns(f.app.db, f.repo.id, { kind: "check", limit: 100 }).some((r) => r.state === "queued")).toBe(false); | |
| 136 | f.release("ceiling"); | |
| 137 | f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 }); | |
| 138 | }); | |
| 139 | ||
| 110 | 140 | test("an open pull whose head was never checked is checked when the service boots", async () => { |
| 111 | 141 | // Invariant: a booting service checks the head of an open pull that has no check, with reason boot. |
| 112 | 142 | await f.checks.stop(); |
| @@ -204,6 +234,11 @@ describe("a check its forge died under", () => { | ||
| 204 | 234 | await rm(dataDir, { recursive: true, force: true }); |
| 205 | 235 | }); |
| 206 | 236 | |
| 237 | // Each test's chain of attempts stands alone. | |
| 238 | beforeEach(() => { | |
| 239 | app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repo.id); | |
| 240 | }); | |
| 241 | ||
| 207 | 242 | /** A check that was running when its forge died: claimed, lease long lapsed. */ |
| 208 | 243 | const abandoned = (commitOid: string, attempt = 0): Run => { |
| 209 | 244 | const run = createRun(app.db, { |
| @@ -258,6 +293,56 @@ describe("a check its forge died under", () => { | ||
| 258 | 293 | cancelRun(app.db, again[0]!.id); |
| 259 | 294 | }); |
| 260 | 295 | |
| 296 | /** `n` earlier attempts of this pull's check, each cut off with `summary`. */ | |
| 297 | const history = (n: number, summary: string): void => { | |
| 298 | for (let i = 0; i < n; i++) { | |
| 299 | const old = createRun(app.db, { | |
| 300 | repoId: repo.id, kind: "check", env: `pull-${pull.number}`, commitOid: pull.headOid, | |
| 301 | reason: "push", issueId: pull.id, attempt: i, | |
| 302 | }); | |
| 303 | app.db.query("UPDATE run SET state = 'cancelled', summary = ?2 WHERE id = ?1").run(old.id, summary); | |
| 304 | } | |
| 305 | }; | |
| 306 | ||
| 307 | test("releases do not spend the allowance: a check cut by many releases and one death is queued again", () => { | |
| 308 | history(CHECK_RESTART_LIMIT + 5, RELEASE_SUMMARY); | |
| 309 | const run = abandoned(pull.headOid, CHECK_RESTART_LIMIT + 5); | |
| 310 | expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1); | |
| 311 | expect(getRun(app.db, run.id)?.state).toBe("cancelled"); | |
| 312 | expect(queuedFor(pull.headOid)).toHaveLength(1); | |
| 313 | for (const q of queuedFor(pull.headOid)) cancelRun(app.db, q.id); | |
| 314 | }); | |
| 315 | ||
| 316 | test("deaths still spend it: after releases, CHECK_RESTART_LIMIT lapses fail the check", () => { | |
| 317 | history(4, RELEASE_SUMMARY); | |
| 318 | const deaths = CHECK_RESTART_LIMIT; | |
| 319 | // attempts 4..6 were deaths (summary REQUEUED_SUMMARY), this is attempt 7. | |
| 320 | const run = abandoned(pull.headOid, 4 + deaths); | |
| 321 | expect(chainCutOffs(app.db, run, RELEASE_SUMMARY)).toBeGreaterThanOrEqual(4); | |
| 322 | expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1); | |
| 323 | expect(getRun(app.db, run.id)?.state).toBe("failed"); | |
| 324 | }); | |
| 325 | ||
| 326 | test("the once-only lookups read a release's retry as a different attempt from its parent", () => { | |
| 327 | // A budget cut at attempt 0 makes attempt 1 the second overrun; a release at attempt 0 does | |
| 328 | // not. | |
| 329 | const cut = (summary: string) => { | |
| 330 | app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repo.id); | |
| 331 | const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "x", commitOid: pull.headOid, reason: "push", issueId: pull.id }); | |
| 332 | const moved = requeueCheck(app.db, claimRun(app.db, a.id)!.id, undefined, summary); | |
| 333 | return moved!; | |
| 334 | }; | |
| 335 | const overrun = cut(`${BUDGET_PREFIX} a full gate took 61 min, limit 60`); | |
| 336 | expect(overrun.again.attempt).toBe(overrun.cancelled.attempt + 1); | |
| 337 | expect(overranBefore(app.db, overrun.again)).toBe(true); | |
| 338 | const released = cut(RELEASE_SUMMARY); | |
| 339 | expect(released.again.attempt).toBe(released.cancelled.attempt + 1); | |
| 340 | expect(overranBefore(app.db, released.again)).toBe(false); | |
| 341 | expect(vanishedBefore(app.db, released.again)).toBe(false); | |
| 342 | const vanished = cut(VANISHED_SUMMARY); | |
| 343 | expect(vanishedBefore(app.db, vanished.again)).toBe(true); | |
| 344 | }); | |
| 345 | ||
| 261 | 346 | test("a check whose lease is still held is left alone", () => { |
| 262 | 347 | const run = createRun(app.db, { |
| 263 | 348 | repoId: repo.id, |
test/verify-requeued-at-tip.test.ts +2 −2
| @@ -10,7 +10,7 @@ import { join } from "node:path"; | ||
| 10 | 10 | import { commitToBranch } from "../src/actions/repos.ts"; |
| 11 | 11 | import { startChecks } from "../src/checks/service.ts"; |
| 12 | 12 | import { VERIFY_ENV } from "../src/checks/verify.ts"; |
| 13 | import { claimRun, createRun, getRun, listRuns, requeueCheck, REQUEUED_SUMMARY, type Run } from "../src/core/runs.ts"; | |
| 13 | import { claimRun, createRun, getRun, listRuns, requeueCheck, RELEASE_SUMMARY, type Run } from "../src/core/runs.ts"; | |
| 14 | 14 | import { BUDGET_PREFIX } from "../src/kernel/budgets.ts"; |
| 15 | 15 | import { checksForge } from "./helpers/checks-forge.ts"; |
| 16 | 16 | |
| @@ -41,7 +41,7 @@ describe("a verification a restart cut off", () => { | ||
| 41 | 41 | |
| 42 | 42 | // The forge stops: a release. |
| 43 | 43 | await f.checks.stop(); |
| 44 | expect(getRun(f.app.db, asked!.id)).toMatchObject({ state: "cancelled", summary: REQUEUED_SUMMARY }); | |
| 44 | expect(getRun(f.app.db, asked!.id)).toMatchObject({ state: "cancelled", summary: RELEASE_SUMMARY }); | |
| 45 | 45 | const again = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter( |
| 46 | 46 | (run) => run.env === VERIFY_ENV && run.state === "queued", |
| 47 | 47 | ); |
new above
offline
Over-long line, and the issue number does not belong in a doc comment. Also update the earlier 'attempt + 1 — the one way' sentence, which is now false.