Public: anyone can read this; the crew can change it.
Vitals raise a broken need on a crash loop #12294
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12285-vitals-raise-a-broken-need-on-a-crash-lo → main 2 commits fork
3 files
changed +86
−0
src/kernel/boots.ts +19
| @@ -154,3 +154,22 @@ export function uncleanInFirstMinutes(dataDir: string, windowMs = FIRST_MINUTES_ | ||
| 154 | 154 | if (firstAt === undefined) return null; |
| 155 | 155 | return ledger.starts.slice(0, -1).filter((s) => !s.clean && s.at - firstAt <= windowMs && now - s.at < memoryMs).length; |
| 156 | 156 | } |
| 157 | ||
| 158 | /** A crash loop: this many unclean starts … */ | |
| 159 | export const LOOP_STARTS = 3; | |
| 160 | ||
| 161 | /** | |
| 162 | * … within this long (#12285). A release leaves one unclean old process for | |
| 163 | * a moment, never three. | |
| 164 | */ | |
| 165 | export const LOOP_WINDOW_MS = 10 * 60_000; | |
| 166 | ||
| 167 | /** | |
| 168 | * Starts that never exited cleanly within the last {@link LOOP_WINDOW_MS} | |
| 169 | * when there are {@link LOOP_STARTS} or more, else null. The newest start is | |
| 170 | * left out: it is the one running. | |
| 171 | */ | |
| 172 | export function crashLoop(dataDir: string, now = Date.now()): { unclean: number; since: number } | null { | |
| 173 | const bad = read(dataDir).starts.slice(0, -1).filter((s) => !s.clean && now - s.at <= LOOP_WINDOW_MS); | |
| 174 | return bad.length >= LOOP_STARTS ? { unclean: bad.length, since: bad[0]!.at } : null; | |
| 175 | } |
src/kernel/vitals.ts +10
| @@ -27,6 +27,7 @@ import { HOLD_WORDS, heldRuns, type HoldCause } from "../runs/held.ts"; | ||
| 27 | 27 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 28 | 28 | import { ownRepoPath } from "./own_repo.ts"; |
| 29 | 29 | import { BUDGETS } from "./budgets.ts"; |
| 30 | import { crashLoop, LOOP_STARTS, LOOP_WINDOW_MS } from "./boots.ts"; | |
| 30 | 31 | import { openDbFiles } from "./file_modes.ts"; |
| 31 | 32 | import { dbFile, dbGrowth, dbSize, dbSizeNeed, fileBytes, type DbWalk } from "../system/db_size.ts"; |
| 32 | 33 | import { backupVerdict, isWatched } from "../snapshot/bucket.ts"; |
| @@ -586,6 +587,15 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 586 | 587 | "/settings/capacity#errors", |
| 587 | 588 | ); |
| 588 | 589 | |
| 590 | const looping = crashLoop(input.dataDir, now); | |
| 591 | if (looping !== null) | |
| 592 | need( | |
| 593 | "crash-loop", | |
| 594 | `The forge restarted ${looping.unclean} times without exiting cleanly in the last ${LOOP_WINDOW_MS / 60_000} minutes (${LOOP_STARTS} or more is a crash loop).`, | |
| 595 | new Date(looping.since).toISOString(), | |
| 596 | "/settings/capacity", | |
| 597 | ); | |
| 598 | ||
| 589 | 599 | const refused = refusedInternal(now, input.signals.refused); |
| 590 | 600 | if (refused.lastHour > 0) |
| 591 | 601 | need( |
test/health-broken.test.ts added +57
| @@ -0,0 +1,57 @@ | ||
| 1 | /** A crash loop is a need: three unclean starts in ten minutes (#12285). */ | |
| 2 | import { afterAll, expect, test } from "bun:test"; | |
| 3 | import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { recordBoot, recordCleanExit, crashLoop } from "../src/kernel/boots.ts"; | |
| 8 | import { vitals } from "../src/kernel/vitals.ts"; | |
| 9 | import { quietLogs } from "./helpers/globals.ts"; | |
| 10 | ||
| 11 | quietLogs(); | |
| 12 | const tmp = mkdtempSync(join(tmpdir(), "broken-")); | |
| 13 | afterAll(() => rmSync(tmp, { recursive: true, force: true })); | |
| 14 | const min = 60_000; | |
| 15 | ||
| 16 | function ledger(name: string) { | |
| 17 | const dir = join(tmp, name); | |
| 18 | mkdirSync(dir); | |
| 19 | return dir; | |
| 20 | } | |
| 21 | ||
| 22 | test("three unclean starts in ten minutes raise a crash-loop need; two do not", () => { | |
| 23 | const dir = ledger("loop"); | |
| 24 | recordBoot(dir, "a", 0); | |
| 25 | recordBoot(dir, "b", min); | |
| 26 | recordBoot(dir, "c", 2 * min); | |
| 27 | expect(crashLoop(dir, 3 * min)).toBeNull(); // a and b: two | |
| 28 | recordBoot(dir, "d", 3 * min); | |
| 29 | expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 }); | |
| 30 | }); | |
| 31 | ||
| 32 | test("clean exits and old crashes are not a loop", () => { | |
| 33 | const dir = ledger("calm"); | |
| 34 | for (const [i, id] of ["a", "b", "c", "d"].entries()) { | |
| 35 | recordBoot(dir, id, i * min); | |
| 36 | recordCleanExit(dir, id); | |
| 37 | } | |
| 38 | recordBoot(dir, "e", 4 * min); | |
| 39 | expect(crashLoop(dir, 4 * min)).toBeNull(); | |
| 40 | const old = ledger("old"); | |
| 41 | for (const id of ["a", "b", "c", "d"]) recordBoot(old, id, 0); | |
| 42 | expect(crashLoop(old, 30 * min)).toBeNull(); | |
| 43 | }); | |
| 44 | ||
| 45 | test("vitals carries the loop as a need that clears with the window", () => { | |
| 46 | const dir = ledger("vitals"); | |
| 47 | const app = createApp({ dataDir: dir, url: "https://f.test" }); | |
| 48 | const now = Date.now(); | |
| 49 | for (const id of ["a", "b", "c", "d"]) recordBoot(dir, id, now - 60_000); | |
| 50 | const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, now, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) }); | |
| 51 | const d = v.degraded.find((x) => x.cause === "crash-loop"); | |
| 52 | expect(d?.sentence).toContain("restarted 3 times"); | |
| 53 | expect(v.ok).toBe(false); | |
| 54 | const later = vitals({ db: app.db, signals: app.signals, dataDir: dir, now: now + 20 * min, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) }); | |
| 55 | expect(later.degraded.find((x) => x.cause === "crash-loop")).toBeUndefined(); | |
| 56 | app.close(); | |
| 57 | }); |
new above
offline