Public: anyone can read this; the crew can change it.
Hardware health: know each machine, its spec sheet, and real wear against the ratings #10363
torque/forge:torque/10352-hardware-health-know-each-machine-its-sp → main 3 commits fork
18 files
changed +776
−0
docs/design/audit.md +2
| @@ -105,6 +105,8 @@ Machine routes are not pages and have no entry: `/audit.csv`, | ||
| 105 | 105 | - `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen. |
| 106 | 106 | - `/admin/flows` — `pages/flows-all.ts` · every repository's flow lines with state and last moved, by kind and stale · `/:owner/:name/flows` · none seen. |
| 107 | 107 | - `/admin/data` — `pages/data.ts` · the forge's own database: tables, rows, read-only SQL, secrets masked · `/data` (the Forge data button) · none seen; shares its body (`pages/data_view.ts`) with `/data` and `/:owner/:name/data`. |
| 108 | - `/admin/hardware` — `pages/hardware.ts` · each machine's CPU, memory and disks, the vendor's ratings and NVMe wear against them · `/system`, `/admin/data` · none seen; paste-a-reading form for hosts without smartctl. | |
| 109 | - `/admin/hardware.json` — `pages/hardware.ts` · the same answer as JSON · `/admin/hardware` (linked from it) · none seen. | |
| 108 | 110 | - `/settings/economics` — `pages/economics.ts` · the profit slider's position and the margin, with the policy in words · `/admin/limits`, `/settings/capacity` · none seen. |
| 109 | 111 | - `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen. |
| 110 | 112 | - `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name. |
docs/features/hardware.md added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | # Hardware health | |
| 2 | ||
| 3 | `/admin/hardware` (site admins; `/admin/hardware.json` is the same answer) shows | |
| 4 | the machine the forge runs on: CPU, memory, and one line per disk. A healthy | |
| 5 | NVMe drive is one quiet line (written against rated TBW, percentage used); | |
| 6 | opening it shows every reading beside the vendor's rating. | |
| 7 | ||
| 8 | - **Inventory** (`src/hardware/inventory.ts`): read from `/sys` and `/proc`. A RAID controller's virtual disk (Dell PERC, MegaRAID) is shown as "behind a controller; SMART not visible", never as an SSD. | |
| 9 | - **Spec sheet** (`src/hardware/catalog.ts`): checked in, keyed by model (980 PRO 1TB: 600 TBW, 5 years, 82/85 °C). A model not in it has no rating and the page says so. | |
| 10 | - **Readings** (`src/hardware/service.ts`, `collect.ts`): once an hour, `smartctl -j -a /dev/nvmeN` for each NVMe controller the kernel lists, stored in `hardware_reading` (90 days). Without smartctl on the host, paste its output under "Add a reading". | |
| 11 | - **Alerts** (`src/hardware/rules.ts`) fire only on what the drive itself reports as bad: a critical-warning bit, available spare below its threshold, percentage used ≥ 80, media errors rising, time above the critical temperature rising. Everything else is a number with its rating beside it. "Rising" compares with the reading before, so one reading alone never alerts on a count. | |
| 12 | - **Projection**: the earliest and latest readings of the last 30 days give bytes per day; remaining rated TBW over that is the date. It needs a day of readings. | |
| 13 | ||
| 14 | A data unit is 512,000 bytes (NVMe spec); TBW is 10^12 bytes. No dependency: smartctl is run by fixed argv on a name matching `nvme\d+`. |
src/core/migration-pins/20261109210000-hardware-readings.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 85b95a06833f869e07ec288401fbdfc16461075a6c7d9794f95c1d0d5ffed392 |
src/core/migrations/20261109210000-hardware-readings.ts added +13
| @@ -0,0 +1,13 @@ | ||
| 1 | // One NVMe health reading per row, for the hardware page: wear over time and | |
| 2 | // the "rising" alert rules need the reading before. Additive. | |
| 3 | export default ` | |
| 4 | CREATE TABLE hardware_reading ( | |
| 5 | id INTEGER PRIMARY KEY, | |
| 6 | at INTEGER NOT NULL, | |
| 7 | device TEXT NOT NULL, | |
| 8 | model TEXT NOT NULL, | |
| 9 | serial TEXT NOT NULL, | |
| 10 | log TEXT NOT NULL | |
| 11 | ); | |
| 12 | CREATE INDEX hardware_reading_device_at ON hardware_reading (device, at); | |
| 13 | `; |
src/hardware/catalog.ts added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | /** | |
| 2 | * The vendor's rated figures, keyed by the model string the drive reports. | |
| 3 | * Checked in and small on purpose: a model not listed has no ratings, and the | |
| 4 | * page says so rather than guess. | |
| 5 | * | |
| 6 | * Samsung 980 PRO 1TB (MZ-V8P1T0BW) data sheet: 600 TBW, 5-year limited | |
| 7 | * warranty, composite temperature warning 82 °C and critical 85 °C. | |
| 8 | * TBW is terabytes (10^12 bytes) written. | |
| 9 | */ | |
| 10 | export interface Spec { | |
| 11 | /** Rated endurance in bytes written (10^12 per TBW). */ | |
| 12 | ratedBytes: number; | |
| 13 | warrantyYears: number; | |
| 14 | /** Composite temperature limits, °C. */ | |
| 15 | tempWarnC: number; | |
| 16 | tempCritC: number; | |
| 17 | source: string; | |
| 18 | } | |
| 19 | ||
| 20 | const TB = 1e12; | |
| 21 | ||
| 22 | const CATALOG: Readonly<Record<string, Spec>> = { | |
| 23 | "Samsung SSD 980 PRO 1TB": { ratedBytes: 600 * TB, warrantyYears: 5, tempWarnC: 82, tempCritC: 85, source: "Samsung 980 PRO data sheet" }, | |
| 24 | }; | |
| 25 | ||
| 26 | /** Whitespace-insensitive, case-insensitive match; undefined when the model is not catalogued. */ | |
| 27 | export function specFor(model: string): Spec | undefined { | |
| 28 | const key = model.trim().replace(/\s+/g, " ").toLowerCase(); | |
| 29 | for (const [name, spec] of Object.entries(CATALOG)) if (name.toLowerCase() === key) return spec; | |
| 30 | return undefined; | |
| 31 | } |
src/hardware/collect.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | /** | |
| 2 | * Reads a controller's health log by running `smartctl -j -a /dev/<name>`. | |
| 3 | * The name comes from the kernel's own list (`nvme0`), checked against a | |
| 4 | * pattern, never from a request. No smartctl on the host is an answer, not an | |
| 5 | * error: the page says so. | |
| 6 | */ | |
| 7 | import { type NvmeLog, parseSmart } from "./smart.ts"; | |
| 8 | ||
| 9 | export type Read = { ok: true; log: NvmeLog } | { ok: false; why: string }; | |
| 10 | ||
| 11 | export async function readSmart(controller: string, run: (argv: string[]) => Promise<string | null> = spawnText): Promise<Read> { | |
| 12 | if (!/^nvme\d{1,3}$/.test(controller)) return { ok: false, why: "not an NVMe controller" }; | |
| 13 | const out = await run(["smartctl", "-j", "-a", `/dev/${controller}`]); | |
| 14 | if (out === null) return { ok: false, why: "smartctl is not available on this host" }; | |
| 15 | const log = parseSmart(out); | |
| 16 | return log === null ? { ok: false, why: "smartctl gave no NVMe health log (needs permission to open the device)" } : { ok: true, log }; | |
| 17 | } | |
| 18 | ||
| 19 | async function spawnText(argv: string[]): Promise<string | null> { | |
| 20 | if (Bun.which(argv[0]!) === null) return null; | |
| 21 | try { | |
| 22 | const p = Bun.spawn(argv, { stdout: "pipe", stderr: "ignore", stdin: "ignore", env: { PATH: process.env.PATH ?? "" } }); | |
| 23 | const timer = setTimeout(() => p.kill(), 10_000); | |
| 24 | const text = await new Response(p.stdout).text(); | |
| 25 | clearTimeout(timer); | |
| 26 | return text; | |
| 27 | } catch { | |
| 28 | return null; | |
| 29 | } | |
| 30 | } |
src/hardware/inventory.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | /** | |
| 2 | * What the machine is made of, read from /sys and /proc and nothing else: its | |
| 3 | * CPU, memory and block devices. A RAID controller's virtual disk (Dell PERC, | |
| 4 | * LSI MegaRAID) is not an SSD whatever it sits on, so it is named as behind a | |
| 5 | * controller. Paths are a parameter so a test can hand it a tree it wrote. | |
| 6 | */ | |
| 7 | import { existsSync, readdirSync, readFileSync } from "node:fs"; | |
| 8 | import { join } from "node:path"; | |
| 9 | ||
| 10 | export type DiskKind = "nvme" | "controller" | "ssd" | "hdd" | "unknown"; | |
| 11 | ||
| 12 | export interface DiskInfo { | |
| 13 | /** Kernel name: `nvme0n1`, `sda`. */ | |
| 14 | name: string; | |
| 15 | kind: DiskKind; | |
| 16 | model: string; | |
| 17 | firmware: string; | |
| 18 | capacityBytes: number; | |
| 19 | /** The controller device (`nvme0`) whose health log holds this namespace's wear. */ | |
| 20 | controller: string | null; | |
| 21 | } | |
| 22 | ||
| 23 | export interface Inventory { | |
| 24 | cpu: string; | |
| 25 | threads: number; | |
| 26 | memoryBytes: number | null; | |
| 27 | disks: DiskInfo[]; | |
| 28 | } | |
| 29 | ||
| 30 | export const HOST = { proc: "/proc", sys: "/sys" }; | |
| 31 | ||
| 32 | const read = (path: string): string => { | |
| 33 | try { | |
| 34 | return readFileSync(path, "utf8").trim().replace(/[\u0000-\u001f]/g, " ").slice(0, 120); | |
| 35 | } catch { | |
| 36 | return ""; | |
| 37 | } | |
| 38 | }; | |
| 39 | ||
| 40 | /** Models that are a RAID controller's logical volume, not a drive. */ | |
| 41 | const RAID = /PERC|MegaRAID|Virtual Disk|LOGICAL VOLUME|\bRAID\b/i; | |
| 42 | ||
| 43 | export function classify(name: string, vendor: string, model: string, rotational: boolean | null): DiskKind { | |
| 44 | if (name.startsWith("nvme")) return "nvme"; | |
| 45 | if (RAID.test(`${vendor} ${model}`)) return "controller"; | |
| 46 | if (rotational === true) return "hdd"; | |
| 47 | if (rotational === false) return "ssd"; | |
| 48 | return "unknown"; | |
| 49 | } | |
| 50 | ||
| 51 | const SKIP = /^(loop|ram|zram|dm-|md|sr|fd|nbd|rbd)/; | |
| 52 | ||
| 53 | export function readInventory(paths = HOST): Inventory { | |
| 54 | let cpu = ""; | |
| 55 | let threads = 0; | |
| 56 | try { | |
| 57 | const text = readFileSync(join(paths.proc, "cpuinfo"), "utf8"); | |
| 58 | for (const line of text.split("\n")) { | |
| 59 | if (line.startsWith("processor")) threads++; | |
| 60 | else if (cpu === "" && line.startsWith("model name")) cpu = line.slice(line.indexOf(":") + 1).trim().slice(0, 120); | |
| 61 | } | |
| 62 | } catch { | |
| 63 | /* left empty: unknown, not guessed */ | |
| 64 | } | |
| 65 | const mem = /MemTotal:\s+(\d+) kB/.exec(read(join(paths.proc, "meminfo"))); | |
| 66 | const disks: DiskInfo[] = []; | |
| 67 | let names: string[] = []; | |
| 68 | try { | |
| 69 | names = readdirSync(join(paths.sys, "block")).sort(); | |
| 70 | } catch { | |
| 71 | /* no block devices visible */ | |
| 72 | } | |
| 73 | for (const name of names) { | |
| 74 | if (SKIP.test(name) || !/^[a-z0-9]+$/.test(name)) continue; | |
| 75 | const dir = join(paths.sys, "block", name); | |
| 76 | const sectors = Number(read(join(dir, "size"))); | |
| 77 | if (!(sectors > 0)) continue; | |
| 78 | const rot = read(join(dir, "queue", "rotational")); | |
| 79 | const nvme = /^(nvme\d+)n\d+$/.exec(name); | |
| 80 | // An NVMe namespace's model and firmware live on its controller under /sys/class/nvme. | |
| 81 | const ctl = nvme ? join(paths.sys, "class", "nvme", nvme[1]!) : undefined; | |
| 82 | const vendor = read(join(dir, "device", "vendor")); | |
| 83 | const model = ctl !== undefined && existsSync(ctl) ? read(join(ctl, "model")) : read(join(dir, "device", "model")); | |
| 84 | const firmware = ctl !== undefined && existsSync(ctl) ? read(join(ctl, "firmware_rev")) : read(join(dir, "device", "rev")); | |
| 85 | disks.push({ | |
| 86 | name, | |
| 87 | kind: classify(name, vendor, model, rot === "" ? null : rot === "1"), | |
| 88 | model: vendor !== "" && !nvme ? `${vendor} ${model}` : model, | |
| 89 | firmware, | |
| 90 | capacityBytes: sectors * 512, | |
| 91 | controller: nvme ? nvme[1]! : null, | |
| 92 | }); | |
| 93 | } | |
| 94 | return { cpu, threads, memoryBytes: mem ? Number(mem[1]) * 1024 : null, disks }; | |
| 95 | } |
src/hardware/rules.ts added +63
| @@ -0,0 +1,63 @@ | ||
| 1 | /** | |
| 2 | * When a drive is in trouble, and when it is just a number. Only what the | |
| 3 | * drive itself reports as bad raises an alert; everything else stays a quiet | |
| 4 | * figure beside its rating. Pure: logs in, words out. | |
| 5 | */ | |
| 6 | import { type Spec } from "./catalog.ts"; | |
| 7 | import { type NvmeLog, UNIT_BYTES, warningBits } from "./smart.ts"; | |
| 8 | ||
| 9 | /** Percentage used at or above this is worn (the owner's line; the drive's own estimate). */ | |
| 10 | export const WORN_PERCENT = 80; | |
| 11 | ||
| 12 | /** What the previous stored reading says, for the "rising" rules. */ | |
| 13 | export interface Before { | |
| 14 | mediaErrors: number; | |
| 15 | minutesCrit: number; | |
| 16 | } | |
| 17 | ||
| 18 | export function alerts(log: NvmeLog, before: Before | undefined): string[] { | |
| 19 | const out: string[] = []; | |
| 20 | const bits = warningBits(log.criticalWarning); | |
| 21 | if (log.criticalWarning !== 0) out.push(`The drive sets critical warning: ${bits.length > 0 ? bits.join(", ") : `0x${log.criticalWarning.toString(16)}`}.`); | |
| 22 | if (log.spare < log.spareThreshold) out.push(`Available spare ${log.spare}% is below the drive's threshold ${log.spareThreshold}%.`); | |
| 23 | if (log.percentUsed >= WORN_PERCENT) out.push(`Percentage used is ${log.percentUsed}% (alert from ${WORN_PERCENT}%).`); | |
| 24 | if (before !== undefined && log.mediaErrors > before.mediaErrors) out.push(`Media errors rose from ${before.mediaErrors} to ${log.mediaErrors}.`); | |
| 25 | if (before !== undefined && log.minutesCrit > before.minutesCrit) out.push(`Time above the critical temperature rose from ${before.minutesCrit} to ${log.minutesCrit} minutes.`); | |
| 26 | return out; | |
| 27 | } | |
| 28 | ||
| 29 | export function bytesWritten(log: NvmeLog): number { | |
| 30 | return log.unitsWritten * UNIT_BYTES; | |
| 31 | } | |
| 32 | ||
| 33 | export interface Point { | |
| 34 | at: number; | |
| 35 | unitsWritten: number; | |
| 36 | } | |
| 37 | ||
| 38 | export const WINDOW_DAYS = 30; | |
| 39 | /** Under this span two readings say nothing about a rate. */ | |
| 40 | const MIN_SPAN_MS = 86_400_000; | |
| 41 | ||
| 42 | export type Projection = | |
| 43 | | { kind: "none"; why: string } | |
| 44 | | { kind: "idle"; bytesPerDay: 0 } | |
| 45 | | { kind: "date"; bytesPerDay: number; at: number }; | |
| 46 | ||
| 47 | /** | |
| 48 | * When rated TBW is reached at the last 30 days' write rate: the earliest and | |
| 49 | * latest readings inside the window give bytes per day; the remaining rated | |
| 50 | * bytes divided by it give the days left. Needs a day of history. | |
| 51 | */ | |
| 52 | export function project(points: readonly Point[], now: number, spec: Spec | undefined, written: number): Projection { | |
| 53 | if (spec === undefined) return { kind: "none", why: "no rating for this model" }; | |
| 54 | const from = now - WINDOW_DAYS * 86_400_000; | |
| 55 | const inside = points.filter((p) => p.at >= from && p.at <= now).sort((a, b) => a.at - b.at); | |
| 56 | const first = inside[0]; | |
| 57 | const last = inside[inside.length - 1]; | |
| 58 | if (first === undefined || last === undefined || last.at - first.at < MIN_SPAN_MS) return { kind: "none", why: "needs a day of readings" }; | |
| 59 | const perMs = ((last.unitsWritten - first.unitsWritten) * UNIT_BYTES) / (last.at - first.at); | |
| 60 | if (!(perMs > 0)) return { kind: "idle", bytesPerDay: 0 }; | |
| 61 | const left = Math.max(0, spec.ratedBytes - written); | |
| 62 | return { kind: "date", bytesPerDay: perMs * 86_400_000, at: now + left / perMs }; | |
| 63 | } |
src/hardware/service.ts added +36
| @@ -0,0 +1,36 @@ | ||
| 1 | /** | |
| 2 | * Reads every NVMe controller's health log once an hour and stores it, so the | |
| 3 | * hardware page is a read of the database and never waits on smartctl. | |
| 4 | */ | |
| 5 | import type { App } from "../app.ts"; | |
| 6 | import { createLog } from "../kernel/log.ts"; | |
| 7 | import { readSmart } from "./collect.ts"; | |
| 8 | import { HOST, readInventory } from "./inventory.ts"; | |
| 9 | import { record } from "./store.ts"; | |
| 10 | ||
| 11 | const log = createLog("hardware"); | |
| 12 | ||
| 13 | const EVERY = 3_600_000; | |
| 14 | ||
| 15 | export async function sweep(app: App, now = Date.now()): Promise<void> { | |
| 16 | const seen = new Set<string>(); | |
| 17 | for (const d of readInventory(HOST).disks) { | |
| 18 | if (d.controller === null || seen.has(d.controller)) continue; | |
| 19 | seen.add(d.controller); | |
| 20 | const read = await readSmart(d.controller); | |
| 21 | if (read.ok) record(app.db, d.controller, read.log, now); | |
| 22 | else log.info("hardware: no reading", { controller: d.controller, why: read.why }); | |
| 23 | } | |
| 24 | } | |
| 25 | ||
| 26 | export function startHardware(app: App): () => void { | |
| 27 | const run = () => void sweep(app).catch((error) => log.warn("hardware sweep failed", { error: String(error) })); | |
| 28 | const first = setTimeout(run, 30_000); | |
| 29 | const every = setInterval(run, EVERY); | |
| 30 | first.unref?.(); | |
| 31 | every.unref?.(); | |
| 32 | return () => { | |
| 33 | clearTimeout(first); | |
| 34 | clearInterval(every); | |
| 35 | }; | |
| 36 | } |
src/hardware/smart.ts added +74
| @@ -0,0 +1,74 @@ | ||
| 1 | /** | |
| 2 | * An NVMe health log read out of `smartctl -j -a` output (smartmontools' | |
| 3 | * JSON), as plain numbers. Anything the drive did not report is a failure to | |
| 4 | * parse, not a zero: a missing field must never read as "no errors". | |
| 5 | * Spec: NVM Express base spec, "SMART / Health Information" log page 02h. | |
| 6 | */ | |
| 7 | export interface NvmeLog { | |
| 8 | model: string; | |
| 9 | firmware: string; | |
| 10 | serial: string; | |
| 11 | capacityBytes: number | null; | |
| 12 | /** Critical warning bit field: bit0 spare below threshold, 1 temperature, 2 reliability, 3 read-only, 4 volatile backup, 5 persistent memory. */ | |
| 13 | criticalWarning: number; | |
| 14 | tempC: number; | |
| 15 | spare: number; | |
| 16 | spareThreshold: number; | |
| 17 | percentUsed: number; | |
| 18 | /** Data units written; one unit is 1000 x 512 bytes (NVMe spec). */ | |
| 19 | unitsWritten: number; | |
| 20 | mediaErrors: number; | |
| 21 | /** Minutes the composite temperature spent at or over the warning / critical limit. */ | |
| 22 | minutesWarn: number; | |
| 23 | minutesCrit: number; | |
| 24 | powerOnHours: number; | |
| 25 | unsafeShutdowns: number; | |
| 26 | } | |
| 27 | ||
| 28 | export const UNIT_BYTES = 512_000; | |
| 29 | ||
| 30 | const BITS = ["available spare below threshold", "temperature outside range", "reliability degraded", "media read-only", "volatile memory backup failed", "persistent memory region unreliable"]; | |
| 31 | ||
| 32 | export function warningBits(field: number): string[] { | |
| 33 | return BITS.filter((_, i) => (field & (1 << i)) !== 0); | |
| 34 | } | |
| 35 | ||
| 36 | const rec = (v: unknown): Record<string, unknown> | undefined => (typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : undefined); | |
| 37 | const count = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined); | |
| 38 | const text = (v: unknown): string => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, "").slice(0, 80).trim() : ""); | |
| 39 | ||
| 40 | export function parseSmart(input: string): NvmeLog | null { | |
| 41 | if (input.length > 200_000) return null; | |
| 42 | let root: Record<string, unknown> | undefined; | |
| 43 | try { | |
| 44 | root = rec(JSON.parse(input)); | |
| 45 | } catch { | |
| 46 | return null; | |
| 47 | } | |
| 48 | const h = rec(root?.nvme_smart_health_information_log); | |
| 49 | if (root === undefined || h === undefined) return null; | |
| 50 | const temp = count(h.temperature) ?? (typeof h.temperature === "number" && Number.isFinite(h.temperature) ? h.temperature : undefined); | |
| 51 | const n = { | |
| 52 | criticalWarning: count(h.critical_warning), | |
| 53 | spare: count(h.available_spare), | |
| 54 | spareThreshold: count(h.available_spare_threshold), | |
| 55 | percentUsed: count(h.percentage_used), | |
| 56 | unitsWritten: count(h.data_units_written), | |
| 57 | mediaErrors: count(h.media_errors), | |
| 58 | powerOnHours: count(h.power_on_hours), | |
| 59 | unsafeShutdowns: count(h.unsafe_shutdowns), | |
| 60 | }; | |
| 61 | for (const v of Object.values(n)) if (v === undefined) return null; | |
| 62 | const model = text(root.model_name); | |
| 63 | if (temp === undefined || model === "") return null; | |
| 64 | return { | |
| 65 | model, | |
| 66 | firmware: text(root.firmware_version), | |
| 67 | serial: text(root.serial_number), | |
| 68 | capacityBytes: count(rec(root.user_capacity)?.bytes) ?? count(root.nvme_total_capacity) ?? null, | |
| 69 | tempC: temp, | |
| 70 | minutesWarn: count(h.warning_temp_time) ?? 0, | |
| 71 | minutesCrit: count(h.critical_comp_time) ?? 0, | |
| 72 | ...(n as Record<keyof typeof n, number>), | |
| 73 | }; | |
| 74 | } |
src/hardware/store.ts added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | /** | |
| 2 | * Stored NVMe readings (`hardware_reading`): kept at most hourly per device, | |
| 3 | * 90 days. The log is stored as the parsed numbers, and read back through the | |
| 4 | * same shape check as an import. | |
| 5 | */ | |
| 6 | import type { Database } from "bun:sqlite"; | |
| 7 | import { type NvmeLog } from "./smart.ts"; | |
| 8 | ||
| 9 | export const KEEP_MS = 90 * 86_400_000; | |
| 10 | export const EVERY_MS = 3_600_000; | |
| 11 | ||
| 12 | export interface Stored { | |
| 13 | at: number; | |
| 14 | device: string; | |
| 15 | log: NvmeLog; | |
| 16 | } | |
| 17 | ||
| 18 | export function latest(db: Database, device: string): Stored | undefined { | |
| 19 | const r = db.query("SELECT at, device, log FROM hardware_reading WHERE device = ? ORDER BY at DESC LIMIT 1").get(device) as { at: number; device: string; log: string } | null; | |
| 20 | return r ? { at: r.at, device: r.device, log: JSON.parse(r.log) as NvmeLog } : undefined; | |
| 21 | } | |
| 22 | ||
| 23 | /** The reading before the latest one, for "rising" rules. */ | |
| 24 | export function previous(db: Database, device: string): Stored | undefined { | |
| 25 | const r = db.query("SELECT at, device, log FROM hardware_reading WHERE device = ? ORDER BY at DESC LIMIT 1 OFFSET 1").get(device) as { at: number; device: string; log: string } | null; | |
| 26 | return r ? { at: r.at, device: r.device, log: JSON.parse(r.log) as NvmeLog } : undefined; | |
| 27 | } | |
| 28 | ||
| 29 | export function since(db: Database, device: string, from: number): { at: number; unitsWritten: number }[] { | |
| 30 | const rows = db.query("SELECT at, log FROM hardware_reading WHERE device = ? AND at >= ? ORDER BY at").all(device, from) as { at: number; log: string }[]; | |
| 31 | return rows.map((r) => ({ at: r.at, unitsWritten: (JSON.parse(r.log) as NvmeLog).unitsWritten })); | |
| 32 | } | |
| 33 | ||
| 34 | /** Records a reading unless one under an hour old exists (`force` for an import). Returns whether it did. */ | |
| 35 | export function record(db: Database, device: string, log: NvmeLog, at: number, force = false): boolean { | |
| 36 | const last = latest(db, device); | |
| 37 | if (!force && last !== undefined && at - last.at < EVERY_MS) return false; | |
| 38 | db.query("INSERT INTO hardware_reading (at, device, model, serial, log) VALUES (?, ?, ?, ?, ?)").run(at, device, log.model, log.serial, JSON.stringify(log)); | |
| 39 | db.query("DELETE FROM hardware_reading WHERE at < ?").run(at - KEEP_MS); | |
| 40 | return true; | |
| 41 | } | |
| 42 | ||
| 43 | export function devices(db: Database): string[] { | |
| 44 | return (db.query("SELECT DISTINCT device FROM hardware_reading ORDER BY device").all() as { device: string }[]).map((r) => r.device); | |
| 45 | } |
src/hardware/view.ts added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | /** | |
| 2 | * The hardware answer, as plain data for the page and the JSON: the machine's | |
| 3 | * inventory, and for each NVMe drive its stored reading set beside its | |
| 4 | * rating, its alerts and its projection. A disk behind a RAID controller is | |
| 5 | * named as such and given no wear. | |
| 6 | */ | |
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { specFor, type Spec } from "./catalog.ts"; | |
| 9 | import { type DiskInfo, type Inventory } from "./inventory.ts"; | |
| 10 | import { alerts, bytesWritten, project, type Projection, WINDOW_DAYS } from "./rules.ts"; | |
| 11 | import { latest, previous, since, type Stored } from "./store.ts"; | |
| 12 | ||
| 13 | export interface DiskView { | |
| 14 | disk: DiskInfo; | |
| 15 | /** Present for an NVMe drive with a stored reading. */ | |
| 16 | reading: Stored | undefined; | |
| 17 | spec: Spec | undefined; | |
| 18 | alerts: string[]; | |
| 19 | writtenBytes: number | null; | |
| 20 | projection: Projection | undefined; | |
| 21 | } | |
| 22 | ||
| 23 | export interface HardwareView { | |
| 24 | inventory: Inventory; | |
| 25 | disks: DiskView[]; | |
| 26 | } | |
| 27 | ||
| 28 | export function hardwareView(db: Database, inventory: Inventory, now: number): HardwareView { | |
| 29 | const disks = inventory.disks.map((disk): DiskView => { | |
| 30 | const reading = disk.kind === "nvme" && disk.controller !== null ? latest(db, disk.controller) : undefined; | |
| 31 | if (reading === undefined || disk.controller === null) return { disk, reading: undefined, spec: undefined, alerts: [], writtenBytes: null, projection: undefined }; | |
| 32 | const spec = specFor(reading.log.model); | |
| 33 | const before = previous(db, disk.controller); | |
| 34 | const written = bytesWritten(reading.log); | |
| 35 | return { | |
| 36 | disk, | |
| 37 | reading, | |
| 38 | spec, | |
| 39 | alerts: alerts(reading.log, before ? before.log : undefined), | |
| 40 | writtenBytes: written, | |
| 41 | projection: project(since(db, disk.controller, now - WINDOW_DAYS * 86_400_000), now, spec, written), | |
| 42 | }; | |
| 43 | }); | |
| 44 | return { inventory, disks }; | |
| 45 | } |
src/http/server.ts +4
| @@ -27,6 +27,7 @@ import { prerenderComments } from "../web/prerender.ts"; | ||
| 27 | 27 | import { systemPage, systemStream } from "../web/pages/system.ts"; |
| 28 | 28 | import { contributePage } from "../web/pages/contribute.ts"; |
| 29 | 29 | import { pipelinePage } from "../web/pages/pipeline.ts"; |
| 30 | import { hardwareImport, hardwareJson, hardwarePage } from "../web/pages/hardware.ts"; | |
| 30 | 31 | import { adminDataPage, appDataPage, dataPage } from "../web/pages/data.ts"; |
| 31 | 32 | import { fleetAct, fleetReceipt, fleetUndo } from "../web/pages/fleet-ops.ts"; |
| 32 | 33 | import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts"; |
| @@ -998,6 +999,9 @@ export function buildRouter(): Router<Handler> { | ||
| 998 | 999 | router.add("GET", "/boards/:slug/live", boardLive); |
| 999 | 1000 | router.add("POST", "/boards/:slug/:op", boardPost); |
| 1000 | 1001 | router.add("GET", "/admin/data", adminDataPage); |
| 1002 | router.add("GET", "/admin/hardware", hardwarePage); | |
| 1003 | router.add("GET", "/admin/hardware.json", hardwareJson); | |
| 1004 | router.add("POST", "/admin/hardware/import", hardwareImport); | |
| 1001 | 1005 | router.add("GET", "/admin/limits", limitsPage); |
| 1002 | 1006 | router.add("GET", "/admin/flows", allFlowsPage); |
| 1003 | 1007 | router.add("POST", "/admin/limits", limitsSet); |
src/main.ts +4
| @@ -53,6 +53,7 @@ import { VERIFY_EVERY_MS } from "./checks/verify.ts"; | ||
| 53 | 53 | import { RED_HOLD_MS } from "./checks/red_main.ts"; |
| 54 | 54 | import { stopQueue } from "./checks/queue.ts"; |
| 55 | 55 | import { startLooks } from "./looks/service.ts"; |
| 56 | import { startHardware } from "./hardware/service.ts"; | |
| 56 | 57 | import { startMirrorOut } from "./origins/mirror-out.ts"; |
| 57 | 58 | import { startMirrorPulls } from "./origins/mirror-pulls.ts"; |
| 58 | 59 | import { startProbes } from "./contract/probes.ts"; |
| @@ -478,6 +479,8 @@ const mirrorPulls = startMirrorPulls(app); | ||
| 478 | 479 | // The look: a picture of every environment that comes up green, taken |
| 479 | 480 | // through this forge's own listener. |
| 480 | 481 | const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined; |
| 482 | // The machine's NVMe health logs, once an hour, for /admin/hardware. | |
| 483 | const hardware = startHardware(app); | |
| 481 | 484 | // Is it so: the forge knocks on every door of an app with a contract after |
| 482 | 485 | // each production deploy and once a night, and compares with the table. |
| 483 | 486 | const probes = apps === undefined ? undefined : startProbes(app, apps); |
| @@ -904,6 +907,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 904 | 907 | mirrorOut(), |
| 905 | 908 | mirrorPulls(), |
| 906 | 909 | looks?.stop(), |
| 910 | hardware(), | |
| 907 | 911 | probes?.stop(), |
| 908 | 912 | nightly?.stop(), |
| 909 | 913 | snapshots.stop(), |
src/web/pages/hardware.ts added +128
| @@ -0,0 +1,128 @@ | ||
| 1 | /** | |
| 2 | * The machine's hardware, for a forge admin: | |
| 3 | * - GET /admin/hardware: CPU and memory, then one line per disk. A healthy | |
| 4 | * NVMe drive is one quiet line; opening it shows every reading beside the | |
| 5 | * vendor's rating. Only what the drive itself reports as bad is an alert. | |
| 6 | * A RAID controller's virtual disk is "behind a controller; SMART not visible". | |
| 7 | * - GET /admin/hardware.json: the same answer as JSON. | |
| 8 | * - POST /admin/hardware/import: a pasted `smartctl -j -a` output stored as a | |
| 9 | * reading, for a host with no smartctl of its own. | |
| 10 | * Rules and ratings: `src/hardware/`. | |
| 11 | */ | |
| 12 | import type { Ctx } from "../../http/context.ts"; | |
| 13 | import { json } from "../../http/respond.ts"; | |
| 14 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; | |
| 15 | import { readInventory } from "../../hardware/inventory.ts"; | |
| 16 | import { parseSmart, warningBits } from "../../hardware/smart.ts"; | |
| 17 | import { record } from "../../hardware/store.ts"; | |
| 18 | import { hardwareView, type DiskView } from "../../hardware/view.ts"; | |
| 19 | import { notFoundPage, pageHead, settingsLayout } from "../chrome.ts"; | |
| 20 | import { field, readForm } from "../form.ts"; | |
| 21 | import { html, type Html } from "../html.ts"; | |
| 22 | import { flashTo, page } from "../layout.ts"; | |
| 23 | import { accountSections } from "./origins.ts"; | |
| 24 | import { badge, btn, icon, row, time } from "../ui.ts"; | |
| 25 | ||
| 26 | const mayRead = (ctx: Ctx): boolean => ctx.auth.user?.isAdmin === true && ctx.auth.user.kind !== "agent"; | |
| 27 | ||
| 28 | const gb = (n: number): string => (n >= 1e12 ? `${+(n / 1e12).toPrecision(3)} TB` : `${+(n / 1e9).toPrecision(3)} GB`); | |
| 29 | const pct = (part: number, whole: number): string => `${+((part / whole) * 100).toPrecision(2)}%`; | |
| 30 | ||
| 31 | export function hardwareJson(ctx: Ctx): Response { | |
| 32 | if (!mayRead(ctx)) return notFoundPage(ctx); | |
| 33 | return json(hardwareView(ctx.app.db, readInventory(), Date.now())); | |
| 34 | } | |
| 35 | ||
| 36 | export function hardwarePage(ctx: Ctx): Response { | |
| 37 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); | |
| 38 | if (!mayRead(ctx)) return notFoundPage(ctx); | |
| 39 | const rendering = performance.now(); | |
| 40 | const now = Date.now(); | |
| 41 | const view = hardwareView(ctx.app.db, readInventory(), now); | |
| 42 | const inv = view.inventory; | |
| 43 | const body = html`${pageHead("Hardware", { sub: "Read from the machine itself; ratings are the vendor's. Only what a drive reports as bad is an alert." })} | |
| 44 | <p class="muted">${inv.cpu === "" ? "CPU unknown" : `${inv.cpu} · ${inv.threads} threads`} · ${inv.memoryBytes === null ? "memory unknown" : `${gb(inv.memoryBytes)} memory`}</p> | |
| 45 | <section class="settings-block"> | |
| 46 | <h2>Disks</h2> | |
| 47 | ${view.disks.length === 0 ? html`<p class="muted">No block devices are visible to the forge.</p>` : html`<ul class="list" id="disks">${view.disks.map((d) => diskRow(d, now))}</ul>`} | |
| 48 | </section> | |
| 49 | <p class="muted small"><a href="/admin/hardware.json">This page as JSON</a></p> | |
| 50 | <section class="settings-block"> | |
| 51 | <h2>Add a reading</h2> | |
| 52 | <p class="muted small">Without smartctl on this host, paste the output of <span class="mono">smartctl -j -a /dev/nvme0</span>. It is stored as a reading at the time you paste it.</p> | |
| 53 | <form method="post" action="/admin/hardware/import"> | |
| 54 | <label class="muted small" for="smart">Device</label> | |
| 55 | <input class="input mono" id="device" name="device" value="nvme0" pattern="nvme[0-9]{1,3}" required /> | |
| 56 | <textarea class="input mono" id="smart" name="smart" rows="4" required aria-label="smartctl JSON"></textarea> | |
| 57 | ${btn("Store reading", { variant: "primary", size: "sm" })} | |
| 58 | </form> | |
| 59 | </section>`; | |
| 60 | const response = page({ ctx, title: "Hardware · admin", body: settingsLayout(accountSections("hardware", { admin: true }), body, "Account settings") }); | |
| 61 | notePhase(ctx.request, "render", performance.now() - rendering); | |
| 62 | return response; | |
| 63 | } | |
| 64 | ||
| 65 | export function diskRow(d: DiskView, now: number): Html { | |
| 66 | const { disk } = d; | |
| 67 | const size = gb(disk.capacityBytes); | |
| 68 | if (disk.kind === "controller") | |
| 69 | return row({ lead: icon("terminal"), title: disk.name, meta: html`${disk.model} · ${size} · behind a controller; SMART not visible` }); | |
| 70 | if (d.reading === undefined) { | |
| 71 | const why = disk.kind === "nvme" ? "no reading yet (needs smartctl on the host, or paste one below)" : "SMART is read for NVMe drives only"; | |
| 72 | return row({ lead: icon("terminal"), title: disk.name, meta: html`${disk.model === "" ? "unknown model" : disk.model} · ${size} · ${why}` }); | |
| 73 | } | |
| 74 | const l = d.reading.log; | |
| 75 | const bad = d.alerts.length > 0; | |
| 76 | const state = bad ? badge(`${d.alerts.length} to look at`, { tone: "danger" }) : badge("healthy", { tone: "success" }); | |
| 77 | const spec = d.spec; | |
| 78 | const written = d.writtenBytes ?? 0; | |
| 79 | const line = (label: string, value: string, rating?: string) => | |
| 80 | html`<tr><th scope="row">${label}</th><td>${value}</td><td class="muted">${rating ?? ""}</td></tr>`; | |
| 81 | const wl = warningBits(l.criticalWarning); | |
| 82 | const projection = d.projection; | |
| 83 | const proj = | |
| 84 | projection === undefined ? "" : projection.kind === "none" ? `no projection: ${projection.why}` : projection.kind === "idle" ? "no writes in the last 30 days" : `${new Date(projection.at).toISOString().slice(0, 10)} at ${gb(projection.bytesPerDay)}/day`; | |
| 85 | return row({ | |
| 86 | lead: icon("chevronRight"), | |
| 87 | title: html`${disk.name} · ${l.model}`, | |
| 88 | meta: html`${size} · firmware ${l.firmware} · ${spec ? html`${gb(written)} of ${gb(spec.ratedBytes)} (${pct(written, spec.ratedBytes)})` : html`${gb(written)} written, no rating for this model`}`, | |
| 89 | trail: state, | |
| 90 | fold: { | |
| 91 | id: `disk-${disk.name}`, | |
| 92 | open: bad, | |
| 93 | body: html`<div class="card-body"> | |
| 94 | ${bad ? html`<ul class="alerts">${d.alerts.map((a) => html`<li>${a}</li>`)}</ul>` : null} | |
| 95 | <table class="table setting-table" data-dense> | |
| 96 | <tbody> | |
| 97 | ${line("Written", gb(written), spec ? `of ${gb(spec.ratedBytes)} rated (${pct(written, spec.ratedBytes)})` : "no rating on file")} | |
| 98 | ${line("Percentage used", `${l.percentUsed}%`, "alert from 80%")} | |
| 99 | ${line("Available spare", `${l.spare}%`, `drive threshold ${l.spareThreshold}%`)} | |
| 100 | ${line("Media errors", String(l.mediaErrors), "alert when rising")} | |
| 101 | ${line("Critical warning", wl.length === 0 ? "none" : wl.join(", "), "alert when any bit is set")} | |
| 102 | ${line("Temperature", `${l.tempC} °C`, spec ? `warning ${spec.tempWarnC} °C, critical ${spec.tempCritC} °C` : "no rating on file")} | |
| 103 | ${line("Time above warning", `${l.minutesWarn} min`)} | |
| 104 | ${line("Time above critical", `${l.minutesCrit} min`, "alert when rising")} | |
| 105 | ${line("Power-on hours", l.powerOnHours.toLocaleString("en-US"))} | |
| 106 | ${line("Unsafe shutdowns", String(l.unsafeShutdowns))} | |
| 107 | ${line("Rated warranty", spec ? `${spec.warrantyYears} years` : "unknown", spec?.source)} | |
| 108 | ${line("Rated TBW reached", proj === "" ? "unknown" : proj, "at the last 30 days' write rate")} | |
| 109 | </tbody> | |
| 110 | </table> | |
| 111 | <p class="muted small">Read ${time(new Date(d.reading.at).toISOString())}; a data unit is 512,000 bytes (NVMe spec), TBW is 10^12 bytes.${now - d.reading.at > 6 * 3_600_000 ? " This reading is more than six hours old." : ""}</p> | |
| 112 | </div>`, | |
| 113 | }, | |
| 114 | }); | |
| 115 | } | |
| 116 | ||
| 117 | // POST /admin/hardware/import | |
| 118 | export async function hardwareImport(ctx: Ctx): Promise<Response> { | |
| 119 | if (!mayRead(ctx)) return notFoundPage(ctx); | |
| 120 | const form = await readForm(ctx, "admin"); | |
| 121 | if (form instanceof Response) return form; | |
| 122 | const device = field(form, "device").trim(); | |
| 123 | const log = parseSmart(field(form, "smart")); | |
| 124 | if (!/^nvme\d{1,3}$/.test(device) || log === null) | |
| 125 | return flashTo(ctx, "/admin/hardware", { tone: "danger", message: "That is not a device name like nvme0 with the JSON from smartctl -j -a." }); | |
| 126 | record(ctx.app.db, device, log, Date.now(), true); | |
| 127 | return flashTo(ctx, "/admin/hardware", { message: "Reading stored." }); | |
| 128 | } |
src/web/pages/origins.ts +2
| @@ -88,6 +88,7 @@ import { | ||
| 88 | 88 | |
| 89 | 89 | export type AccountSection = |
| 90 | 90 | | "adminflows" |
| 91 | | "hardware" | |
| 91 | 92 | | "security" |
| 92 | 93 | | "profile" |
| 93 | 94 | | "confirms" |
| @@ -159,6 +160,7 @@ const SECTION_ROWS: readonly (readonly [ | ||
| 159 | 160 | ["This forge", "Limits", "/admin/limits", "activity", (c) => c === "limits", true], |
| 160 | 161 | ["This forge", "Economics", "/settings/economics", "activity", (c) => c === "economics", true], |
| 161 | 162 | ["This forge", "Flows", "/admin/flows", "activity", (c) => c === "adminflows", true], |
| 163 | ["This forge", "Hardware", "/admin/hardware", "activity", (c) => c === "hardware", true], | |
| 162 | 164 | ["This forge", "Flags", "/settings/flags", "activity", (c) => c === "flags", true], |
| 163 | 165 | ["This forge", "Features", "/settings/features", "rocket", (c) => c === "features", true], |
| 164 | 166 | ["This forge", "Backups", "/settings/backups", "download", (c) => c === "backups", true], |
test/hardware.test.ts added +188
| @@ -0,0 +1,188 @@ | ||
| 1 | /** Hardware health: recorded smartctl logs against the alert rules, the inventory, the projection, and the page. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdirSync, mkdtemp as _m, writeFileSync } from "node:fs"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { createSession } from "../src/core/sessions.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { readSmart } from "../src/hardware/collect.ts"; | |
| 11 | import { readInventory } from "../src/hardware/inventory.ts"; | |
| 12 | import { alerts, project } from "../src/hardware/rules.ts"; | |
| 13 | import { specFor } from "../src/hardware/catalog.ts"; | |
| 14 | import { parseSmart, type NvmeLog } from "../src/hardware/smart.ts"; | |
| 15 | import { hardwareView } from "../src/hardware/view.ts"; | |
| 16 | import { diskRow } from "../src/web/pages/hardware.ts"; | |
| 17 | import { record } from "../src/hardware/store.ts"; | |
| 18 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 19 | import { quietLogs } from "./helpers/globals.ts"; | |
| 20 | ||
| 21 | void _m; | |
| 22 | quietLogs(); | |
| 23 | ||
| 24 | /** A recorded `smartctl -j -a` for a 980 PRO; fields override the healthy reading of 11-09 20:45Z. */ | |
| 25 | function recorded(over: Record<string, number> = {}): string { | |
| 26 | const h = { | |
| 27 | critical_warning: 0, temperature: 39, available_spare: 100, available_spare_threshold: 10, percentage_used: 0, | |
| 28 | data_units_written: 1_074_219, data_units_read: 2_000_000, power_on_hours: 2575, unsafe_shutdowns: 41, | |
| 29 | media_errors: 0, warning_temp_time: 0, critical_comp_time: 0, ...over, | |
| 30 | }; | |
| 31 | return JSON.stringify({ model_name: "Samsung SSD 980 PRO 1TB", firmware_version: "5B2QGXA7", serial_number: "S5GXNX0T000000", user_capacity: { bytes: 1_000_204_886_016 }, nvme_smart_health_information_log: h }); | |
| 32 | } | |
| 33 | const log = (over: Record<string, number> = {}): NvmeLog => parseSmart(recorded(over))!; | |
| 34 | ||
| 35 | describe("alert rules against recorded logs", () => { | |
| 36 | test("healthy: today's reading is quiet, whatever the unsafe shutdowns", () => { | |
| 37 | const l = log(); | |
| 38 | expect(l.firmware).toBe("5B2QGXA7"); | |
| 39 | expect(alerts(l, undefined)).toEqual([]); | |
| 40 | expect(alerts(l, { mediaErrors: 0, minutesCrit: 0 })).toEqual([]); | |
| 41 | }); | |
| 42 | test("warm and busy but not bad stays quiet: 70 C, 60 minutes above warning, 79% used", () => { | |
| 43 | expect(alerts(log({ temperature: 70, warning_temp_time: 60, percentage_used: 79 }), { mediaErrors: 0, minutesCrit: 0 })).toEqual([]); | |
| 44 | }); | |
| 45 | test("worn: percentage used at 80", () => { | |
| 46 | expect(alerts(log({ percentage_used: 80 }), undefined)).toHaveLength(1); | |
| 47 | expect(alerts(log({ percentage_used: 80 }), undefined)[0]).toContain("80%"); | |
| 48 | }); | |
| 49 | test("hot: time above critical rising, but not merely nonzero", () => { | |
| 50 | expect(alerts(log({ critical_comp_time: 12 }), undefined)).toEqual([]); | |
| 51 | expect(alerts(log({ critical_comp_time: 12 }), { mediaErrors: 0, minutesCrit: 12 })).toEqual([]); | |
| 52 | expect(alerts(log({ critical_comp_time: 15, critical_warning: 2 }), { mediaErrors: 0, minutesCrit: 12 })).toHaveLength(2); | |
| 53 | }); | |
| 54 | test("failing: critical bit, spare below threshold, media errors rising", () => { | |
| 55 | const failing = log({ critical_warning: 1, available_spare: 5, media_errors: 7 }); | |
| 56 | const out = alerts(failing, { mediaErrors: 3, minutesCrit: 0 }); | |
| 57 | expect(out).toHaveLength(3); | |
| 58 | expect(out.join(" ")).toContain("available spare below threshold"); | |
| 59 | expect(out.join(" ")).toContain("from 3 to 7"); | |
| 60 | }); | |
| 61 | test("static media errors are a number, not an alert", () => { | |
| 62 | expect(alerts(log({ media_errors: 7 }), { mediaErrors: 7, minutesCrit: 0 })).toEqual([]); | |
| 63 | }); | |
| 64 | test("a log missing a field is refused, never read as zero", () => { | |
| 65 | const o = JSON.parse(recorded()); | |
| 66 | delete o.nvme_smart_health_information_log.media_errors; | |
| 67 | expect(parseSmart(JSON.stringify(o))).toBeNull(); | |
| 68 | expect(parseSmart("not json")).toBeNull(); | |
| 69 | }); | |
| 70 | }); | |
| 71 | ||
| 72 | describe("spec sheet and projection", () => { | |
| 73 | test("980 PRO 1TB is rated; an unknown model is not guessed", () => { | |
| 74 | const s = specFor("Samsung SSD 980 PRO 1TB")!; | |
| 75 | expect([s.ratedBytes, s.warrantyYears, s.tempWarnC, s.tempCritC]).toEqual([600e12, 5, 82, 85]); | |
| 76 | expect(specFor("Samsung SSD 980 PRO 2TB")).toBeUndefined(); | |
| 77 | }); | |
| 78 | test("a hand calculation: 1 TB per day from 550 GB reaches 600 TB in 599.45 days", () => { | |
| 79 | const day = 86_400_000; | |
| 80 | const now = 100 * day; | |
| 81 | const unitsPerDay = 1e12 / 512_000; | |
| 82 | const points = [{ at: now - 10 * day, unitsWritten: 0 }, { at: now, unitsWritten: 10 * unitsPerDay }]; | |
| 83 | const p = project(points, now, specFor("Samsung SSD 980 PRO 1TB"), 550e9); | |
| 84 | if (p.kind !== "date") throw new Error("expected a date"); | |
| 85 | expect(p.bytesPerDay).toBeCloseTo(1e12, 0); | |
| 86 | expect((p.at - now) / day).toBeCloseTo(599.45, 6); | |
| 87 | }); | |
| 88 | test("no history, no rating and no writes say so", () => { | |
| 89 | const spec = specFor("Samsung SSD 980 PRO 1TB"); | |
| 90 | expect(project([{ at: 1, unitsWritten: 5 }], 2, spec, 0).kind).toBe("none"); | |
| 91 | expect(project([], 9e9, undefined, 0).kind).toBe("none"); | |
| 92 | const day = 86_400_000; | |
| 93 | expect(project([{ at: 0, unitsWritten: 5 }, { at: 3 * day, unitsWritten: 5 }], 3 * day, spec, 0).kind).toBe("idle"); | |
| 94 | }); | |
| 95 | }); | |
| 96 | ||
| 97 | describe("inventory", () => { | |
| 98 | test("a PERC virtual disk is behind a controller; the NVMe is read from its controller", async () => { | |
| 99 | const dir = await mkdtemp(join(tmpdir(), "forge-hw-")); | |
| 100 | const put = (path: string, text: string) => { | |
| 101 | mkdirSync(join(dir, path, ".."), { recursive: true }); | |
| 102 | writeFileSync(join(dir, path), text); | |
| 103 | }; | |
| 104 | put("sys/block/sda/size", "1874329600\n"); | |
| 105 | put("sys/block/sda/queue/rotational", "0\n"); | |
| 106 | put("sys/block/sda/device/vendor", "DELL \n"); | |
| 107 | put("sys/block/sda/device/model", "PERC H730P Mini\n"); | |
| 108 | put("sys/block/nvme0n1/size", "1953525168\n"); | |
| 109 | put("sys/block/nvme0n1/queue/rotational", "0\n"); | |
| 110 | put("sys/class/nvme/nvme0/model", "Samsung SSD 980 PRO 1TB\n"); | |
| 111 | put("sys/class/nvme/nvme0/firmware_rev", "5B2QGXA7\n"); | |
| 112 | put("sys/block/loop0/size", "100\n"); | |
| 113 | put("proc/cpuinfo", "processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Gold 6130\nprocessor\t: 1\nmodel name\t: Intel(R) Xeon(R) Gold 6130\n"); | |
| 114 | put("proc/meminfo", "MemTotal: 131072 kB\n"); | |
| 115 | const inv = readInventory({ proc: join(dir, "proc"), sys: join(dir, "sys") }); | |
| 116 | await rm(dir, { recursive: true }); | |
| 117 | expect(inv.threads).toBe(2); | |
| 118 | expect(inv.cpu).toBe("Intel(R) Xeon(R) Gold 6130"); | |
| 119 | expect(inv.memoryBytes).toBe(131072 * 1024); | |
| 120 | expect(inv.disks.map((d) => [d.name, d.kind])).toEqual([["nvme0n1", "nvme"], ["sda", "controller"]]); | |
| 121 | const nvme = inv.disks[0]!; | |
| 122 | expect([nvme.model, nvme.firmware, nvme.controller, nvme.capacityBytes]).toEqual(["Samsung SSD 980 PRO 1TB", "5B2QGXA7", "nvme0", 1953525168 * 512]); | |
| 123 | }); | |
| 124 | test("smartctl is called for a kernel controller name only, and its absence is an answer", async () => { | |
| 125 | const calls: string[][] = []; | |
| 126 | const ok = await readSmart("nvme0", async (argv) => (calls.push(argv), recorded())); | |
| 127 | expect(ok.ok).toBe(true); | |
| 128 | expect(calls[0]).toEqual(["smartctl", "-j", "-a", "/dev/nvme0"]); | |
| 129 | expect((await readSmart("nvme0; rm -rf /", async () => recorded())).ok).toBe(false); | |
| 130 | const none = await readSmart("nvme0", async () => null); | |
| 131 | expect(none.ok ? "" : none.why).toContain("not available"); | |
| 132 | }); | |
| 133 | }); | |
| 134 | ||
| 135 | describe("the hardware page", () => { | |
| 136 | let dir: string; | |
| 137 | let app: App; | |
| 138 | let servers: Servers; | |
| 139 | let admin = ""; | |
| 140 | let other = ""; | |
| 141 | beforeAll(async () => { | |
| 142 | dir = await mkdtemp(join(tmpdir(), "forge-hw-page-")); | |
| 143 | app = createApp({ dataDir: dir, port: 0, host: "forge.test" }); | |
| 144 | servers = await serve(app); | |
| 145 | for (const name of ["boss", "pal"]) { | |
| 146 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "boss" }); | |
| 147 | if (!made.ok) throw new Error(made.error); | |
| 148 | const c = `forge_session=${createSession(app.db, made.value.id).token}`; | |
| 149 | if (name === "boss") admin = c; | |
| 150 | else other = c; | |
| 151 | } | |
| 152 | }); | |
| 153 | afterAll(async () => { | |
| 154 | await servers.stop?.(); | |
| 155 | await rm(dir, { recursive: true, force: true }); | |
| 156 | }); | |
| 157 | const get = (path: string, cookie: string) => fetch(new URL(path, servers.url), { headers: { cookie } }); | |
| 158 | ||
| 159 | test("only a site admin sees it", async () => { | |
| 160 | expect((await get("/admin/hardware", other)).status).toBe(403); | |
| 161 | expect((await get("/admin/hardware.json", other)).status).not.toBe(200); | |
| 162 | expect((await get("/admin/hardware", admin)).status).toBe(200); | |
| 163 | }); | |
| 164 | test("one quiet line when healthy, open with detail when a drive reports bad, a controller named as such", () => { | |
| 165 | const inventory = { | |
| 166 | cpu: "x", threads: 1, memoryBytes: null, | |
| 167 | disks: [ | |
| 168 | { name: "nvme0n1", kind: "nvme" as const, model: "Samsung SSD 980 PRO 1TB", firmware: "5B2QGXA7", capacityBytes: 1e12, controller: "nvme0" }, | |
| 169 | { name: "sda", kind: "controller" as const, model: "DELL PERC H730P Mini", firmware: "", capacityBytes: 960e9, controller: null }, | |
| 170 | ], | |
| 171 | }; | |
| 172 | const now = Date.now(); | |
| 173 | record(app.db, "nvme0", log(), now); | |
| 174 | const healthy = hardwareView(app.db, inventory, now); | |
| 175 | expect(healthy.disks[0]!.alerts).toEqual([]); | |
| 176 | const quiet = diskRow(healthy.disks[0]!, now).text; | |
| 177 | expect(quiet).toContain("healthy"); | |
| 178 | expect(quiet).toContain("550 GB of 600 TB (0.092%)"); | |
| 179 | expect(quiet).toContain("Unsafe shutdowns"); | |
| 180 | expect(quiet).not.toContain("to look at"); | |
| 181 | expect(diskRow(healthy.disks[1]!, now).text).toContain("behind a controller; SMART not visible"); | |
| 182 | ||
| 183 | record(app.db, "nvme0", log({ percentage_used: 85 }), now + 4_000_000); | |
| 184 | const worn = diskRow(hardwareView(app.db, inventory, now + 4_000_000).disks[0]!, now + 4_000_000).text; | |
| 185 | expect(worn).toContain("1 to look at"); | |
| 186 | expect(worn).toContain("Percentage used is 85%"); | |
| 187 | }); | |
| 188 | }); |
test/spawn-pipes.test.ts +1
| @@ -29,6 +29,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 29 | 29 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed); stderr ignored" }, |
| 30 | 30 | "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" }, |
| 31 | 31 | "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" }, |
| 32 | "src/hardware/collect.ts": { sites: ["stdout"], reason: "stdout read whole via Response.text() under a 10 s kill timer; stderr ignored" }, | |
| 32 | 33 | "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" }, |
| 33 | 34 | "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" }, |
| 34 | 35 | "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" }, |
new above
offline