trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns1g rEventsFlows

Disk health rules: a spec catalog and alerts only on what the drive itself reports #10616

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/10464-disk-health-rules-a-spec-catalog-and-ale → main 4 commits fork

  • Merged. torque/10464-disk-health-rules-a-spec-catalog-and-ale was merged into main as cf39091.

Review

Sign in to review this change.
9 files changed +471 −0
scripts/unreached-allow.json +2
@@ -15,6 +15,8 @@
1515 "src/flow/review_rounds.ts",
1616 "src/flow/shared_red.ts",
1717 "src/flow/stale_fold.ts",
18 "src/machines/disk_catalog.ts",
19 "src/machines/disk_health.ts",
1820 "src/spawn/pod.ts",
1921 "src/tui/gallery.ts",
2022 "src/tui/ui/qr.ts",
src/machines/disk_catalog.ts added +28
@@ -0,0 +1,28 @@
1 /**
2 * Vendor ratings for the disks the forge knows (#10464). Keyed by the exact
3 * model string smartctl reports; a model not listed returns `null` — nothing
4 * is guessed from a similar name. Ratings are the vendor's published figures
5 * (Samsung 980 PRO data sheet: 1 TB, 600 TBW endurance, 5-year limited
6 * warranty, composite temperature warning 82 °C, critical 85 °C).
7 */
8
9 export type DiskSpec = {
10 model: string;
11 /** Rated endurance, terabytes written (1 TB = 10^12 bytes, as vendors state it). */
12 ratedTbw: number;
13 warrantyYears: number;
14 /** Composite temperature, degrees Celsius. */
15 warnTempC: number;
16 critTempC: number;
17 };
18
19 const CATALOG: readonly DiskSpec[] = [
20 { model: "Samsung SSD 980 PRO 1TB", ratedTbw: 600, warrantyYears: 5, warnTempC: 82, critTempC: 85 },
21 ];
22
23 /** The spec for exactly this model, or null when it is not in the catalog. */
24 export function diskSpec(model: unknown): DiskSpec | null {
25 if (typeof model !== "string") return null;
26 const found = CATALOG.find((s) => s.model === model);
27 return found ? { ...found } : null;
28 }
src/machines/disk_health.ts added +142
@@ -0,0 +1,142 @@
1 /**
2 * Disk health from `smartctl --json -a` (#10464): parse one reading, and
3 * decide whether the drive itself reports something bad. Pure: text in, plain
4 * data out.
5 *
6 * NVMe health log fields follow the NVMe base spec, "SMART / Health
7 * Information" (Log Page 02h): a data unit is 1000 × 512 bytes; critical
8 * warning is a bit field; temperature is composite, in °C (smartctl converts
9 * from kelvin); warning/critical composite time are in minutes.
10 *
11 * Alerts only on what the drive says is bad (see `assessDisk`). Everything
12 * else — unsafe shutdowns, temperature now, written vs rated — is a quiet
13 * number for the page, never an alert.
14 */
15
16 import type { DiskSpec } from "./disk_catalog.ts";
17
18 /** Bytes per NVMe "data unit": 1000 sectors of 512 bytes (NVMe base spec, log 02h). */
19 export const BYTES_PER_DATA_UNIT = 512_000;
20
21 const MAX_TEXT = 4_000_000;
22
23 export type DiskReading = {
24 kind: "nvme";
25 model: string;
26 firmware: string;
27 capacityBytes: number;
28 dataUnitsWritten: number;
29 bytesWritten: number;
30 percentageUsed: number;
31 availableSpare: number;
32 spareThreshold: number;
33 mediaErrors: number;
34 criticalWarning: number;
35 temperatureC: number;
36 warnTempMinutes: number;
37 critTempMinutes: number;
38 powerOnHours: number;
39 unsafeShutdowns: number;
40 };
41
42 export type BehindController = { kind: "behind-controller" };
43 export type Refusal = { kind: "refused"; reason: "not-json" | "not-smartctl-output" | "no-nvme-health-log" | "bad-field"; field?: string };
44 export type ParsedDisk = DiskReading | BehindController | Refusal;
45
46 type Obj = Record<string, unknown>;
47 const isObj = (v: unknown): v is Obj => typeof v === "object" && v !== null && !Array.isArray(v);
48
49 /** A non-negative safe integer, or undefined. */
50 function count(v: unknown): number | undefined {
51 return typeof v === "number" && Number.isSafeInteger(v) && v >= 0 ? v : undefined;
52 }
53
54 export function parseSmartJson(text: unknown): ParsedDisk {
55 if (typeof text !== "string" || text.length === 0 || text.length > MAX_TEXT) return { kind: "refused", reason: "not-json" };
56 let doc: unknown;
57 try {
58 doc = JSON.parse(text);
59 } catch {
60 return { kind: "refused", reason: "not-json" };
61 }
62 if (!isObj(doc) || !isObj(doc.smartctl) || !isObj(doc.device)) return { kind: "refused", reason: "not-smartctl-output" };
63
64 const log = doc.nvme_smart_health_information_log;
65 if (!isObj(log)) {
66 const dev = doc.device as Obj;
67 const ident = [dev.type, dev.protocol, doc.model_name, doc.product, doc.scsi_product, doc.scsi_vendor]
68 .filter((x): x is string => typeof x === "string")
69 .join(" ");
70 if (/megaraid|perc|raid/i.test(ident)) return { kind: "behind-controller" };
71 return { kind: "refused", reason: "no-nvme-health-log" };
72 }
73
74 const bad = (field: string): Refusal => ({ kind: "refused", reason: "bad-field", field });
75 const model = doc.model_name;
76 if (typeof model !== "string" || model.length === 0 || model.length > 200) return bad("model_name");
77 const fw = doc.firmware_version;
78 if (typeof fw !== "string" || fw.length > 100) return bad("firmware_version");
79 const cap = isObj(doc.user_capacity) ? count(doc.user_capacity.bytes) : undefined;
80 if (cap === undefined) return bad("user_capacity.bytes");
81
82 const names = [
83 "data_units_written",
84 "percentage_used",
85 "available_spare",
86 "available_spare_threshold",
87 "media_errors",
88 "critical_warning",
89 "temperature",
90 "warning_temp_time",
91 "critical_comp_time",
92 "power_on_hours",
93 "unsafe_shutdowns",
94 ] as const;
95 const n: Record<string, number> = {};
96 for (const name of names) {
97 const v = count(log[name]);
98 if (v === undefined) return bad(name);
99 n[name] = v;
100 }
101
102 return {
103 kind: "nvme",
104 model,
105 firmware: fw,
106 capacityBytes: cap,
107 dataUnitsWritten: n.data_units_written!,
108 bytesWritten: n.data_units_written! * BYTES_PER_DATA_UNIT,
109 percentageUsed: n.percentage_used!,
110 availableSpare: n.available_spare!,
111 spareThreshold: n.available_spare_threshold!,
112 mediaErrors: n.media_errors!,
113 criticalWarning: n.critical_warning!,
114 temperatureC: n.temperature!,
115 warnTempMinutes: n.warning_temp_time!,
116 critTempMinutes: n.critical_comp_time!,
117 powerOnHours: n.power_on_hours!,
118 unsafeShutdowns: n.unsafe_shutdowns!,
119 };
120 }
121
122 export type Assessment = { level: "ok" | "alert"; reasons: string[] };
123
124 /** Percentage used at or above this is an alert (the drive's own wear estimate). */
125 export const WEAR_ALERT_PERCENT = 80;
126
127 /**
128 * Alerts only on: a critical-warning bit set, available spare below its
129 * threshold, percentage used ≥ 80, media errors higher than the previous
130 * reading, minutes above the critical temperature higher than the previous
131 * reading. `spec` (the vendor ratings, or null) is accepted so callers pass
132 * one shape, but no rating raises an alert: written-vs-rated is a quiet number.
133 */
134 export function assessDisk(reading: DiskReading, previous?: DiskReading | null, _spec?: DiskSpec | null): Assessment {
135 const reasons: string[] = [];
136 if (reading.criticalWarning !== 0) reasons.push(`critical warning bits set (0x${reading.criticalWarning.toString(16).padStart(2, "0")})`);
137 if (reading.availableSpare < reading.spareThreshold) reasons.push(`available spare ${reading.availableSpare}% is below its threshold ${reading.spareThreshold}%`);
138 if (reading.percentageUsed >= WEAR_ALERT_PERCENT) reasons.push(`percentage used ${reading.percentageUsed}% is at or above ${WEAR_ALERT_PERCENT}%`);
139 if (previous && reading.mediaErrors > previous.mediaErrors) reasons.push(`media errors rose from ${previous.mediaErrors} to ${reading.mediaErrors}`);
140 if (previous && reading.critTempMinutes > previous.critTempMinutes) reasons.push(`minutes above critical temperature rose from ${previous.critTempMinutes} to ${reading.critTempMinutes}`);
141 return { level: reasons.length ? "alert" : "ok", reasons };
142 }
test/disk-health.test.ts added +98
@@ -0,0 +1,98 @@
1 import { describe, expect, test } from "bun:test";
2 import { readFileSync } from "node:fs";
3 import { diskSpec } from "../src/machines/disk_catalog.ts";
4 import { assessDisk, parseSmartJson, type DiskReading } from "../src/machines/disk_health.ts";
5
6 const raw = (n: string) => readFileSync(new URL(`./fixtures/smart/${n}.json`, import.meta.url), "utf8");
7 const read = (n: string): DiskReading => {
8 const r = parseSmartJson(raw(n));
9 if (r.kind !== "nvme") throw new Error(`fixture ${n} parsed as ${r.kind}`);
10 return r;
11 };
12 const spec = diskSpec("Samsung SSD 980 PRO 1TB");
13
14 describe("catalog", () => {
15 test("980 PRO 1TB ratings", () => {
16 expect(spec).toEqual({ model: "Samsung SSD 980 PRO 1TB", ratedTbw: 600, warrantyYears: 5, warnTempC: 82, critTempC: 85 });
17 });
18 test("an unknown model is null, not a guess", () => {
19 expect(diskSpec("Samsung SSD 980 PRO 2TB")).toBeNull();
20 expect(diskSpec("samsung ssd 980 pro 1tb")).toBeNull();
21 expect(diskSpec(undefined)).toBeNull();
22 });
23 });
24
25 describe("parseSmartJson", () => {
26 test("the 11-09 reading", () => {
27 const r = read("healthy");
28 expect(r).toMatchObject({ model: "Samsung SSD 980 PRO 1TB", firmware: "5B2QGXA7", capacityBytes: 1000204886016, percentageUsed: 0, availableSpare: 100, spareThreshold: 10, mediaErrors: 0, criticalWarning: 0, temperatureC: 39, warnTempMinutes: 0, critTempMinutes: 0, powerOnHours: 2575, unsafeShutdowns: 41 });
29 // 1,074,219 data units × 512,000 B = 550.0 GB
30 expect(r.bytesWritten).toBe(1074219 * 512000);
31 expect(Math.round(r.bytesWritten / 1e9)).toBe(550);
32 });
33 test("a PERC virtual disk is behind a controller, never an SSD", () => {
34 expect(parseSmartJson(raw("perc"))).toEqual({ kind: "behind-controller" });
35 });
36 test("bad input is a named refusal", () => {
37 expect(parseSmartJson("nope")).toEqual({ kind: "refused", reason: "not-json" });
38 expect(parseSmartJson("")).toEqual({ kind: "refused", reason: "not-json" });
39 expect(parseSmartJson("[1]")).toEqual({ kind: "refused", reason: "not-smartctl-output" });
40 expect(parseSmartJson('{"smartctl":{},"device":{"type":"sat"}}')).toEqual({ kind: "refused", reason: "no-nvme-health-log" });
41 const doc = JSON.parse(raw("healthy"));
42 doc.nvme_smart_health_information_log.media_errors = -1;
43 expect(parseSmartJson(JSON.stringify(doc))).toEqual({ kind: "refused", reason: "bad-field", field: "media_errors" });
44 });
45 });
46
47 describe("assessDisk", () => {
48 test("healthy is ok, with or without a previous reading", () => {
49 expect(assessDisk(read("healthy"), undefined, spec)).toEqual({ level: "ok", reasons: [] });
50 expect(assessDisk(read("healthy"), read("healthy"), spec)).toEqual({ level: "ok", reasons: [] });
51 });
52 test("41 unsafe shutdowns alone is not an alert", () => {
53 const r = { ...read("healthy"), unsafeShutdowns: 4100 };
54 expect(assessDisk(r, { ...r, unsafeShutdowns: 41 }, spec).level).toBe("ok");
55 });
56 test("high temperature now, or time above warning, is not an alert", () => {
57 const r = { ...read("healthy"), temperatureC: 84, warnTempMinutes: 900 };
58 expect(assessDisk(r, read("healthy"), spec).level).toBe("ok");
59 });
60 test("a critical-warning bit", () => {
61 const a = assessDisk({ ...read("healthy"), criticalWarning: 1 }, undefined, spec);
62 expect(a.level).toBe("alert");
63 expect(a.reasons.join()).toMatch(/critical warning/);
64 });
65 test("spare below threshold, but not at it", () => {
66 const h = read("healthy");
67 expect(assessDisk({ ...h, availableSpare: 9 }, undefined, spec).reasons.join()).toMatch(/spare/);
68 expect(assessDisk({ ...h, availableSpare: 10 }, undefined, spec).level).toBe("ok");
69 });
70 test("percentage used ≥ 80, not 79", () => {
71 const h = read("healthy");
72 expect(assessDisk({ ...h, percentageUsed: 79 }, undefined, spec).level).toBe("ok");
73 expect(assessDisk({ ...h, percentageUsed: 80 }, undefined, spec).level).toBe("alert");
74 expect(assessDisk(read("worn"), undefined, spec).reasons.join()).toMatch(/percentage used 82%/);
75 });
76 test("media errors only when higher than the previous reading", () => {
77 const h = read("healthy");
78 const e = { ...h, mediaErrors: 3 };
79 expect(assessDisk(e, h, spec).reasons.join()).toMatch(/media errors rose from 0 to 3/);
80 expect(assessDisk(e, e, spec).level).toBe("ok");
81 expect(assessDisk(e, undefined, spec).level).toBe("ok");
82 });
83 test("critical-temperature minutes only when rising", () => {
84 const h = read("healthy");
85 const hot = read("hot");
86 expect(assessDisk(hot, h, spec).reasons.join()).toMatch(/critical temperature rose from 0 to 57/);
87 expect(assessDisk(hot, hot, spec).level).toBe("ok");
88 expect(assessDisk(hot, undefined, spec).level).toBe("ok");
89 });
90 test("failing: warning bit and spare below threshold both named", () => {
91 const a = assessDisk(read("failing"), read("healthy"), spec);
92 expect(a.level).toBe("alert");
93 expect(a.reasons.length).toBe(3); // bits, spare, media errors 0 → 12
94 });
95 test("an unknown model assesses the same", () => {
96 expect(assessDisk(read("worn"), undefined, null).level).toBe("alert");
97 });
98 });
test/fixtures/smart/failing.json added +44
@@ -0,0 +1,44 @@
1 {
2 "json_format_version": [
3 1,
4 0
5 ],
6 "smartctl": {
7 "version": [
8 7,
9 4
10 ],
11 "exit_status": 0
12 },
13 "device": {
14 "name": "/dev/nvme0",
15 "type": "nvme",
16 "protocol": "NVMe"
17 },
18 "model_name": "Samsung SSD 980 PRO 1TB",
19 "serial_number": "REDACTED",
20 "firmware_version": "5B2QGXA7",
21 "user_capacity": {
22 "blocks": 1953525168,
23 "bytes": 1000204886016
24 },
25 "nvme_smart_health_information_log": {
26 "critical_warning": 5,
27 "temperature": 39,
28 "available_spare": 7,
29 "available_spare_threshold": 10,
30 "percentage_used": 40,
31 "data_units_read": 1000000,
32 "data_units_written": 1074219,
33 "host_reads": 1,
34 "host_writes": 1,
35 "controller_busy_time": 10,
36 "power_cycles": 100,
37 "power_on_hours": 2575,
38 "unsafe_shutdowns": 41,
39 "media_errors": 12,
40 "num_err_log_entries": 0,
41 "warning_temp_time": 0,
42 "critical_comp_time": 0
43 }
44 }↵
test/fixtures/smart/healthy.json added +44
@@ -0,0 +1,44 @@
1 {
2 "json_format_version": [
3 1,
4 0
5 ],
6 "smartctl": {
7 "version": [
8 7,
9 4
10 ],
11 "exit_status": 0
12 },
13 "device": {
14 "name": "/dev/nvme0",
15 "type": "nvme",
16 "protocol": "NVMe"
17 },
18 "model_name": "Samsung SSD 980 PRO 1TB",
19 "serial_number": "REDACTED",
20 "firmware_version": "5B2QGXA7",
21 "user_capacity": {
22 "blocks": 1953525168,
23 "bytes": 1000204886016
24 },
25 "nvme_smart_health_information_log": {
26 "critical_warning": 0,
27 "temperature": 39,
28 "available_spare": 100,
29 "available_spare_threshold": 10,
30 "percentage_used": 0,
31 "data_units_read": 1000000,
32 "data_units_written": 1074219,
33 "host_reads": 1,
34 "host_writes": 1,
35 "controller_busy_time": 10,
36 "power_cycles": 100,
37 "power_on_hours": 2575,
38 "unsafe_shutdowns": 41,
39 "media_errors": 0,
40 "num_err_log_entries": 0,
41 "warning_temp_time": 0,
42 "critical_comp_time": 0
43 }
44 }↵
test/fixtures/smart/hot.json added +44
@@ -0,0 +1,44 @@
1 {
2 "json_format_version": [
3 1,
4 0
5 ],
6 "smartctl": {
7 "version": [
8 7,
9 4
10 ],
11 "exit_status": 0
12 },
13 "device": {
14 "name": "/dev/nvme0",
15 "type": "nvme",
16 "protocol": "NVMe"
17 },
18 "model_name": "Samsung SSD 980 PRO 1TB",
19 "serial_number": "REDACTED",
20 "firmware_version": "5B2QGXA7",
21 "user_capacity": {
22 "blocks": 1953525168,
23 "bytes": 1000204886016
24 },
25 "nvme_smart_health_information_log": {
26 "critical_warning": 0,
27 "temperature": 86,
28 "available_spare": 100,
29 "available_spare_threshold": 10,
30 "percentage_used": 0,
31 "data_units_read": 1000000,
32 "data_units_written": 1074219,
33 "host_reads": 1,
34 "host_writes": 1,
35 "controller_busy_time": 10,
36 "power_cycles": 100,
37 "power_on_hours": 2575,
38 "unsafe_shutdowns": 41,
39 "media_errors": 0,
40 "num_err_log_entries": 0,
41 "warning_temp_time": 340,
42 "critical_comp_time": 57
43 }
44 }↵
test/fixtures/smart/perc.json added +25
@@ -0,0 +1,25 @@
1 {
2 "json_format_version": [
3 1,
4 0
5 ],
6 "smartctl": {
7 "version": [
8 7,
9 4
10 ],
11 "exit_status": 0
12 },
13 "device": {
14 "name": "/dev/bus/0",
15 "type": "megaraid,0",
16 "info_name": "/dev/bus/0 [megaraid_disk_00]",
17 "protocol": "SCSI"
18 },
19 "scsi_vendor": "DELL",
20 "scsi_product": "PERC H730P",
21 "user_capacity": {
22 "blocks": 1,
23 "bytes": 480103981056
24 }
25 }↵
test/fixtures/smart/worn.json added +44
@@ -0,0 +1,44 @@
1 {
2 "json_format_version": [
3 1,
4 0
5 ],
6 "smartctl": {
7 "version": [
8 7,
9 4
10 ],
11 "exit_status": 0
12 },
13 "device": {
14 "name": "/dev/nvme0",
15 "type": "nvme",
16 "protocol": "NVMe"
17 },
18 "model_name": "Samsung SSD 980 PRO 1TB",
19 "serial_number": "REDACTED",
20 "firmware_version": "5B2QGXA7",
21 "user_capacity": {
22 "blocks": 1953525168,
23 "bytes": 1000204886016
24 },
25 "nvme_smart_health_information_log": {
26 "critical_warning": 0,
27 "temperature": 39,
28 "available_spare": 100,
29 "available_spare_threshold": 10,
30 "percentage_used": 82,
31 "data_units_read": 1000000,
32 "data_units_written": 1000000000,
33 "host_reads": 1,
34 "host_writes": 1,
35 "controller_busy_time": 10,
36 "power_cycles": 100,
37 "power_on_hours": 21000,
38 "unsafe_shutdowns": 41,
39 "media_errors": 0,
40 "num_err_log_entries": 0,
41 "warning_temp_time": 0,
42 "critical_comp_time": 0
43 }
44 }↵

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs