Public: anyone can read this; the crew can change it.
Sentinel: disk-80 was degraded 16 minutes today: Disk under the forge's data is 83% used, past the 80% mark. #11941
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/11940-sentinel-disk-80-was-degraded-16-minutes → main 3 commits fork 1 thread · 1 unresolved
2 files
changed +18
−2
src/kernel/vitals.ts +5 −2
| @@ -335,6 +335,9 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 335 | 335 | }; |
| 336 | 336 | |
| 337 | 337 | const disk = (input.disk ?? readDisk)(input.dataDir); |
| 338 | // Room to spare (#11940): at 100 GiB free no share-of-the-disk guard speaks, | |
| 339 | // the floor and the marks alike. | |
| 340 | const room = roomy(disk); | |
| 338 | 341 | if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN)) |
| 339 | 342 | need( |
| 340 | 343 | "disk", |
| @@ -342,14 +345,14 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 342 | 345 | undefined, |
| 343 | 346 | "/settings/capacity", |
| 344 | 347 | ); |
| 345 | else if (disk.totalBytes > 0 && !roomy(disk) && disk.freeBytes / disk.totalBytes < DISK_FLOOR) | |
| 348 | else if (disk.totalBytes > 0 && !room && disk.freeBytes / disk.totalBytes < DISK_FLOOR) | |
| 346 | 349 | need( |
| 347 | 350 | "disk-floor", |
| 348 | 351 | `Disk is under the ${DISK_FLOOR * 100}% the forge keeps for itself: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free. Child forges sleep, and no child forge, app or check starts, until room is made.`, |
| 349 | 352 | undefined, |
| 350 | 353 | "/settings/capacity", |
| 351 | 354 | ); |
| 352 | else if (disk.totalBytes > 0) { | |
| 355 | else if (disk.totalBytes > 0 && !room) { | |
| 353 | 356 | // The owner hears well before the floor (#11800): one need for the highest mark passed. |
| 354 | 357 | const used = 1 - disk.freeBytes / disk.totalBytes; |
| 355 | 358 | const mark = DISK_USED_MARKS.find((m) => used >= m); |
test/disk-marks.test.ts +13
| @@ -22,6 +22,19 @@ describe("vitals names the highest mark passed", () => { | ||
| 22 | 22 | const at = (usedPct: number) => |
| 23 | 23 | vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir, disk: () => ({ freeBytes: 100 - usedPct, totalBytes: 100 }) }); |
| 24 | 24 | |
| 25 | const GiB = 1024 ** 3; | |
| 26 | const causes = (freeBytes: number, totalBytes: number) => | |
| 27 | vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir, disk: () => ({ freeBytes, totalBytes }) }).degraded.map((d) => d.cause); | |
| 28 | ||
| 29 | test("a roomy disk is not a mark: 83% used with 362 GiB free says nothing (#11940)", () => { | |
| 30 | expect(causes(362 * GiB, 2200 * GiB)).toEqual([]); | |
| 31 | }); | |
| 32 | ||
| 33 | test("at exactly 100 GiB free the marks are quiet, one byte under they speak", () => { | |
| 34 | expect(causes(100 * GiB, 500 * GiB)).toEqual([]); | |
| 35 | expect(causes(100 * GiB - 1, 500 * GiB)).toEqual(["disk-80"]); | |
| 36 | }); | |
| 37 | ||
| 25 | 38 | test("69% says nothing, 70% and 80% say their mark, the floor says only the floor", () => { |
| 26 | 39 | expect(at(69).degraded.map((d) => d.cause)).toEqual([]); |
| 27 | 40 | expect(at(70).degraded.map((d) => d.cause)).toEqual(["disk-70"]); |
new above
offline
Add a test in test/disk-marks.test.ts for this guard: 362 GiB free of 2.2 TiB gives no disk-80 or disk-70 cause, and a disk just under the roomy threshold still does. Nothing in this diff fails if the guard is removed.