Public: anyone can read this; the crew can change it.
Sentinel: disk-red was degraded 18 minutes today: The check machine's disk was full when 8 checks failed: 10% free, under the 10% floor — the machine's red, not the pull… #5796
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/5792-sentinel-disk-red-was-degraded-18-minute → main 1 commit fork 1 thread · 1 unresolved
3 files
changed +15
−6
src/kernel/vitals.ts +6 −2
| @@ -423,8 +423,12 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 423 | 423 | for (const shared of sharedTests(input.db, now)) |
| 424 | 424 | need(sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared)); |
| 425 | 425 | |
| 426 | // Checks that failed on a full disk are one card, not a park per pull. | |
| 427 | const full = diskReds(input.db, now); | |
| 426 | // Checks that failed on a full disk are one card, not a park per pull, and | |
| 427 | // only while the disk is still under the floor: once it is back the machine | |
| 428 | // is well, and the hour's reds are history, not a degraded state. | |
| 429 | const diskStillFull = | |
| 430 | disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN); | |
| 431 | const full = diskStillFull ? diskReds(input.db, now) : undefined; | |
| 428 | 432 | if (full !== undefined) need("disk-red", diskRedSentence(full), full.since, "/settings/capacity"); |
| 429 | 433 | |
| 430 | 434 | // A red restore drill, or none in eight days, is a card until a drill passes. |
test/checks-disk-red.test.ts +7 −2
| @@ -83,11 +83,11 @@ describe("a red on a full check machine", () => { | ||
| 83 | 83 | })!; |
| 84 | 84 | }; |
| 85 | 85 | const open = (pull: Pull) => getIssueById(app.db, pull.id)?.state === "open"; |
| 86 | const cards = () => | |
| 86 | const cards = (freeNow = 30) => | |
| 87 | 87 | vitals({ |
| 88 | 88 | db: app.db, |
| 89 | 89 | dataDir: app.config.dataDir, |
| 90 | disk: reading(500), | |
| 90 | disk: reading(freeNow), | |
| 91 | 91 | signals: { errors: new ErrorLedger(), refused: new RefusedLedger() }, |
| 92 | 92 | }).degraded.filter((d) => d.cause === "disk-red"); |
| 93 | 93 | |
| @@ -118,6 +118,11 @@ describe("a red on a full check machine", () => { | ||
| 118 | 118 | expect(requested).toHaveLength(3); |
| 119 | 119 | }); |
| 120 | 120 | |
| 121 | test("the card ends when the disk is back above the floor, not an hour later", () => { | |
| 122 | expect(cards(30)).toHaveLength(1); | |
| 123 | expect(cards(500)).toHaveLength(0); | |
| 124 | }); | |
| 125 | ||
| 121 | 126 | test("above the floor: a red is the pull's own, parked as ever, and no card", async () => { |
| 122 | 127 | const before = cards()[0]!.sentence; |
| 123 | 128 | const own = await spentPull(); |
test/vitals.test.ts +2 −2
| @@ -168,8 +168,8 @@ test("the number of queries does not grow with runs", () => { | ||
| 168 | 168 | if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id); |
| 169 | 169 | } |
| 170 | 170 | expect(count()).toBe(before); |
| 171 | // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, the checks red on a full disk, and the newest drill of every domain, and the checks no red main holds. | |
| 172 | expect(before).toBe(10); | |
| 171 | // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, (the checks red on a full disk only while the disk is full), and the newest drill of every domain, and the checks no red main holds. | |
| 172 | expect(before).toBe(9); | |
| 173 | 173 | }); |
| 174 | 174 | |
| 175 | 175 | test("errors are counted, bounded, redacted, and a burst is said", async () => { |
new above
offline
Duplicates the floor comparison from the disk card above; compute it once and reuse.