Public: anyone can read this; the crew can change it.

Hardware health: know each machine, its spec sheet, and real wear against the ratings #10363

Parked opened this in answer to
Closed without merging

torque/forge:torque/10352-hardware-health-know-each-machine-its-sp → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
18 files changed +776 −0
docs/design/audit.md +2
@@ -105,6 +105,8 @@ Machine routes are not pages and have no entry: `/audit.csv`,
105105 - `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen.
106106 - `/admin/flows` — `pages/flows-all.ts` · every repository's flow lines with state and last moved, by kind and stale · `/:owner/:name/flows` · none seen.
107107 - `/admin/data` — `pages/data.ts` · the forge's own database: tables, rows, read-only SQL, secrets masked · `/data` (the Forge data button) · none seen; shares its body (`pages/data_view.ts`) with `/data` and `/:owner/:name/data`.
108 - `/admin/hardware` — `pages/hardware.ts` · each machine's CPU, memory and disks, the vendor's ratings and NVMe wear against them · `/system`, `/admin/data` · none seen; paste-a-reading form for hosts without smartctl.
109 - `/admin/hardware.json` — `pages/hardware.ts` · the same answer as JSON · `/admin/hardware` (linked from it) · none seen.
108110 - `/settings/economics` — `pages/economics.ts` · the profit slider's position and the margin, with the policy in words · `/admin/limits`, `/settings/capacity` · none seen.
109111 - `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen.
110112 - `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name.
docs/features/hardware.md added +14
@@ -0,0 +1,14 @@
1 # Hardware health
2
3 `/admin/hardware` (site admins; `/admin/hardware.json` is the same answer) shows
4 the machine the forge runs on: CPU, memory, and one line per disk. A healthy
5 NVMe drive is one quiet line (written against rated TBW, percentage used);
6 opening it shows every reading beside the vendor's rating.
7
8 - **Inventory** (`src/hardware/inventory.ts`): read from `/sys` and `/proc`. A RAID controller's virtual disk (Dell PERC, MegaRAID) is shown as "behind a controller; SMART not visible", never as an SSD.
9 - **Spec sheet** (`src/hardware/catalog.ts`): checked in, keyed by model (980 PRO 1TB: 600 TBW, 5 years, 82/85 °C). A model not in it has no rating and the page says so.
10 - **Readings** (`src/hardware/service.ts`, `collect.ts`): once an hour, `smartctl -j -a /dev/nvmeN` for each NVMe controller the kernel lists, stored in `hardware_reading` (90 days). Without smartctl on the host, paste its output under "Add a reading".
11 - **Alerts** (`src/hardware/rules.ts`) fire only on what the drive itself reports as bad: a critical-warning bit, available spare below its threshold, percentage used ≥ 80, media errors rising, time above the critical temperature rising. Everything else is a number with its rating beside it. "Rising" compares with the reading before, so one reading alone never alerts on a count.
12 - **Projection**: the earliest and latest readings of the last 30 days give bytes per day; remaining rated TBW over that is the date. It needs a day of readings.
13
14 A data unit is 512,000 bytes (NVMe spec); TBW is 10^12 bytes. No dependency: smartctl is run by fixed argv on a name matching `nvme\d+`.
src/core/migration-pins/20261109210000-hardware-readings.pin added +1
@@ -0,0 +1,1 @@
1 85b95a06833f869e07ec288401fbdfc16461075a6c7d9794f95c1d0d5ffed392
src/core/migrations/20261109210000-hardware-readings.ts added +13
@@ -0,0 +1,13 @@
1 // One NVMe health reading per row, for the hardware page: wear over time and
2 // the "rising" alert rules need the reading before. Additive.
3 export default `
4 CREATE TABLE hardware_reading (
5 id INTEGER PRIMARY KEY,
6 at INTEGER NOT NULL,
7 device TEXT NOT NULL,
8 model TEXT NOT NULL,
9 serial TEXT NOT NULL,
10 log TEXT NOT NULL
11 );
12 CREATE INDEX hardware_reading_device_at ON hardware_reading (device, at);
13 `;
src/hardware/catalog.ts added +31
@@ -0,0 +1,31 @@
1 /**
2 * The vendor's rated figures, keyed by the model string the drive reports.
3 * Checked in and small on purpose: a model not listed has no ratings, and the
4 * page says so rather than guess.
5 *
6 * Samsung 980 PRO 1TB (MZ-V8P1T0BW) data sheet: 600 TBW, 5-year limited
7 * warranty, composite temperature warning 82 °C and critical 85 °C.
8 * TBW is terabytes (10^12 bytes) written.
9 */
10 export interface Spec {
11 /** Rated endurance in bytes written (10^12 per TBW). */
12 ratedBytes: number;
13 warrantyYears: number;
14 /** Composite temperature limits, °C. */
15 tempWarnC: number;
16 tempCritC: number;
17 source: string;
18 }
19
20 const TB = 1e12;
21
22 const CATALOG: Readonly<Record<string, Spec>> = {
23 "Samsung SSD 980 PRO 1TB": { ratedBytes: 600 * TB, warrantyYears: 5, tempWarnC: 82, tempCritC: 85, source: "Samsung 980 PRO data sheet" },
24 };
25
26 /** Whitespace-insensitive, case-insensitive match; undefined when the model is not catalogued. */
27 export function specFor(model: string): Spec | undefined {
28 const key = model.trim().replace(/\s+/g, " ").toLowerCase();
29 for (const [name, spec] of Object.entries(CATALOG)) if (name.toLowerCase() === key) return spec;
30 return undefined;
31 }
src/hardware/collect.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * Reads a controller's health log by running `smartctl -j -a /dev/<name>`.
3 * The name comes from the kernel's own list (`nvme0`), checked against a
4 * pattern, never from a request. No smartctl on the host is an answer, not an
5 * error: the page says so.
6 */
7 import { type NvmeLog, parseSmart } from "./smart.ts";
8
9 export type Read = { ok: true; log: NvmeLog } | { ok: false; why: string };
10
11 export async function readSmart(controller: string, run: (argv: string[]) => Promise<string | null> = spawnText): Promise<Read> {
12 if (!/^nvme\d{1,3}$/.test(controller)) return { ok: false, why: "not an NVMe controller" };
13 const out = await run(["smartctl", "-j", "-a", `/dev/${controller}`]);
14 if (out === null) return { ok: false, why: "smartctl is not available on this host" };
15 const log = parseSmart(out);
16 return log === null ? { ok: false, why: "smartctl gave no NVMe health log (needs permission to open the device)" } : { ok: true, log };
17 }
18
19 async function spawnText(argv: string[]): Promise<string | null> {
20 if (Bun.which(argv[0]!) === null) return null;
21 try {
22 const p = Bun.spawn(argv, { stdout: "pipe", stderr: "ignore", stdin: "ignore", env: { PATH: process.env.PATH ?? "" } });
23 const timer = setTimeout(() => p.kill(), 10_000);
24 const text = await new Response(p.stdout).text();
25 clearTimeout(timer);
26 return text;
27 } catch {
28 return null;
29 }
30 }
src/hardware/inventory.ts added +95
@@ -0,0 +1,95 @@
1 /**
2 * What the machine is made of, read from /sys and /proc and nothing else: its
3 * CPU, memory and block devices. A RAID controller's virtual disk (Dell PERC,
4 * LSI MegaRAID) is not an SSD whatever it sits on, so it is named as behind a
5 * controller. Paths are a parameter so a test can hand it a tree it wrote.
6 */
7 import { existsSync, readdirSync, readFileSync } from "node:fs";
8 import { join } from "node:path";
9
10 export type DiskKind = "nvme" | "controller" | "ssd" | "hdd" | "unknown";
11
12 export interface DiskInfo {
13 /** Kernel name: `nvme0n1`, `sda`. */
14 name: string;
15 kind: DiskKind;
16 model: string;
17 firmware: string;
18 capacityBytes: number;
19 /** The controller device (`nvme0`) whose health log holds this namespace's wear. */
20 controller: string | null;
21 }
22
23 export interface Inventory {
24 cpu: string;
25 threads: number;
26 memoryBytes: number | null;
27 disks: DiskInfo[];
28 }
29
30 export const HOST = { proc: "/proc", sys: "/sys" };
31
32 const read = (path: string): string => {
33 try {
34 return readFileSync(path, "utf8").trim().replace(/[\u0000-\u001f]/g, " ").slice(0, 120);
35 } catch {
36 return "";
37 }
38 };
39
40 /** Models that are a RAID controller's logical volume, not a drive. */
41 const RAID = /PERC|MegaRAID|Virtual Disk|LOGICAL VOLUME|\bRAID\b/i;
42
43 export function classify(name: string, vendor: string, model: string, rotational: boolean | null): DiskKind {
44 if (name.startsWith("nvme")) return "nvme";
45 if (RAID.test(`${vendor} ${model}`)) return "controller";
46 if (rotational === true) return "hdd";
47 if (rotational === false) return "ssd";
48 return "unknown";
49 }
50
51 const SKIP = /^(loop|ram|zram|dm-|md|sr|fd|nbd|rbd)/;
52
53 export function readInventory(paths = HOST): Inventory {
54 let cpu = "";
55 let threads = 0;
56 try {
57 const text = readFileSync(join(paths.proc, "cpuinfo"), "utf8");
58 for (const line of text.split("\n")) {
59 if (line.startsWith("processor")) threads++;
60 else if (cpu === "" && line.startsWith("model name")) cpu = line.slice(line.indexOf(":") + 1).trim().slice(0, 120);
61 }
62 } catch {
63 /* left empty: unknown, not guessed */
64 }
65 const mem = /MemTotal:\s+(\d+) kB/.exec(read(join(paths.proc, "meminfo")));
66 const disks: DiskInfo[] = [];
67 let names: string[] = [];
68 try {
69 names = readdirSync(join(paths.sys, "block")).sort();
70 } catch {
71 /* no block devices visible */
72 }
73 for (const name of names) {
74 if (SKIP.test(name) || !/^[a-z0-9]+$/.test(name)) continue;
75 const dir = join(paths.sys, "block", name);
76 const sectors = Number(read(join(dir, "size")));
77 if (!(sectors > 0)) continue;
78 const rot = read(join(dir, "queue", "rotational"));
79 const nvme = /^(nvme\d+)n\d+$/.exec(name);
80 // An NVMe namespace's model and firmware live on its controller under /sys/class/nvme.
81 const ctl = nvme ? join(paths.sys, "class", "nvme", nvme[1]!) : undefined;
82 const vendor = read(join(dir, "device", "vendor"));
83 const model = ctl !== undefined && existsSync(ctl) ? read(join(ctl, "model")) : read(join(dir, "device", "model"));
84 const firmware = ctl !== undefined && existsSync(ctl) ? read(join(ctl, "firmware_rev")) : read(join(dir, "device", "rev"));
85 disks.push({
86 name,
87 kind: classify(name, vendor, model, rot === "" ? null : rot === "1"),
88 model: vendor !== "" && !nvme ? `${vendor} ${model}` : model,
89 firmware,
90 capacityBytes: sectors * 512,
91 controller: nvme ? nvme[1]! : null,
92 });
93 }
94 return { cpu, threads, memoryBytes: mem ? Number(mem[1]) * 1024 : null, disks };
95 }
src/hardware/rules.ts added +63
@@ -0,0 +1,63 @@
1 /**
2 * When a drive is in trouble, and when it is just a number. Only what the
3 * drive itself reports as bad raises an alert; everything else stays a quiet
4 * figure beside its rating. Pure: logs in, words out.
5 */
6 import { type Spec } from "./catalog.ts";
7 import { type NvmeLog, UNIT_BYTES, warningBits } from "./smart.ts";
8
9 /** Percentage used at or above this is worn (the owner's line; the drive's own estimate). */
10 export const WORN_PERCENT = 80;
11
12 /** What the previous stored reading says, for the "rising" rules. */
13 export interface Before {
14 mediaErrors: number;
15 minutesCrit: number;
16 }
17
18 export function alerts(log: NvmeLog, before: Before | undefined): string[] {
19 const out: string[] = [];
20 const bits = warningBits(log.criticalWarning);
21 if (log.criticalWarning !== 0) out.push(`The drive sets critical warning: ${bits.length > 0 ? bits.join(", ") : `0x${log.criticalWarning.toString(16)}`}.`);
22 if (log.spare < log.spareThreshold) out.push(`Available spare ${log.spare}% is below the drive's threshold ${log.spareThreshold}%.`);
23 if (log.percentUsed >= WORN_PERCENT) out.push(`Percentage used is ${log.percentUsed}% (alert from ${WORN_PERCENT}%).`);
24 if (before !== undefined && log.mediaErrors > before.mediaErrors) out.push(`Media errors rose from ${before.mediaErrors} to ${log.mediaErrors}.`);
25 if (before !== undefined && log.minutesCrit > before.minutesCrit) out.push(`Time above the critical temperature rose from ${before.minutesCrit} to ${log.minutesCrit} minutes.`);
26 return out;
27 }
28
29 export function bytesWritten(log: NvmeLog): number {
30 return log.unitsWritten * UNIT_BYTES;
31 }
32
33 export interface Point {
34 at: number;
35 unitsWritten: number;
36 }
37
38 export const WINDOW_DAYS = 30;
39 /** Under this span two readings say nothing about a rate. */
40 const MIN_SPAN_MS = 86_400_000;
41
42 export type Projection =
43 | { kind: "none"; why: string }
44 | { kind: "idle"; bytesPerDay: 0 }
45 | { kind: "date"; bytesPerDay: number; at: number };
46
47 /**
48 * When rated TBW is reached at the last 30 days' write rate: the earliest and
49 * latest readings inside the window give bytes per day; the remaining rated
50 * bytes divided by it give the days left. Needs a day of history.
51 */
52 export function project(points: readonly Point[], now: number, spec: Spec | undefined, written: number): Projection {
53 if (spec === undefined) return { kind: "none", why: "no rating for this model" };
54 const from = now - WINDOW_DAYS * 86_400_000;
55 const inside = points.filter((p) => p.at >= from && p.at <= now).sort((a, b) => a.at - b.at);
56 const first = inside[0];
57 const last = inside[inside.length - 1];
58 if (first === undefined || last === undefined || last.at - first.at < MIN_SPAN_MS) return { kind: "none", why: "needs a day of readings" };
59 const perMs = ((last.unitsWritten - first.unitsWritten) * UNIT_BYTES) / (last.at - first.at);
60 if (!(perMs > 0)) return { kind: "idle", bytesPerDay: 0 };
61 const left = Math.max(0, spec.ratedBytes - written);
62 return { kind: "date", bytesPerDay: perMs * 86_400_000, at: now + left / perMs };
63 }
src/hardware/service.ts added +36
@@ -0,0 +1,36 @@
1 /**
2 * Reads every NVMe controller's health log once an hour and stores it, so the
3 * hardware page is a read of the database and never waits on smartctl.
4 */
5 import type { App } from "../app.ts";
6 import { createLog } from "../kernel/log.ts";
7 import { readSmart } from "./collect.ts";
8 import { HOST, readInventory } from "./inventory.ts";
9 import { record } from "./store.ts";
10
11 const log = createLog("hardware");
12
13 const EVERY = 3_600_000;
14
15 export async function sweep(app: App, now = Date.now()): Promise<void> {
16 const seen = new Set<string>();
17 for (const d of readInventory(HOST).disks) {
18 if (d.controller === null || seen.has(d.controller)) continue;
19 seen.add(d.controller);
20 const read = await readSmart(d.controller);
21 if (read.ok) record(app.db, d.controller, read.log, now);
22 else log.info("hardware: no reading", { controller: d.controller, why: read.why });
23 }
24 }
25
26 export function startHardware(app: App): () => void {
27 const run = () => void sweep(app).catch((error) => log.warn("hardware sweep failed", { error: String(error) }));
28 const first = setTimeout(run, 30_000);
29 const every = setInterval(run, EVERY);
30 first.unref?.();
31 every.unref?.();
32 return () => {
33 clearTimeout(first);
34 clearInterval(every);
35 };
36 }
src/hardware/smart.ts added +74
@@ -0,0 +1,74 @@
1 /**
2 * An NVMe health log read out of `smartctl -j -a` output (smartmontools'
3 * JSON), as plain numbers. Anything the drive did not report is a failure to
4 * parse, not a zero: a missing field must never read as "no errors".
5 * Spec: NVM Express base spec, "SMART / Health Information" log page 02h.
6 */
7 export interface NvmeLog {
8 model: string;
9 firmware: string;
10 serial: string;
11 capacityBytes: number | null;
12 /** Critical warning bit field: bit0 spare below threshold, 1 temperature, 2 reliability, 3 read-only, 4 volatile backup, 5 persistent memory. */
13 criticalWarning: number;
14 tempC: number;
15 spare: number;
16 spareThreshold: number;
17 percentUsed: number;
18 /** Data units written; one unit is 1000 x 512 bytes (NVMe spec). */
19 unitsWritten: number;
20 mediaErrors: number;
21 /** Minutes the composite temperature spent at or over the warning / critical limit. */
22 minutesWarn: number;
23 minutesCrit: number;
24 powerOnHours: number;
25 unsafeShutdowns: number;
26 }
27
28 export const UNIT_BYTES = 512_000;
29
30 const BITS = ["available spare below threshold", "temperature outside range", "reliability degraded", "media read-only", "volatile memory backup failed", "persistent memory region unreliable"];
31
32 export function warningBits(field: number): string[] {
33 return BITS.filter((_, i) => (field & (1 << i)) !== 0);
34 }
35
36 const rec = (v: unknown): Record<string, unknown> | undefined => (typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : undefined);
37 const count = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined);
38 const text = (v: unknown): string => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, "").slice(0, 80).trim() : "");
39
40 export function parseSmart(input: string): NvmeLog | null {
41 if (input.length > 200_000) return null;
42 let root: Record<string, unknown> | undefined;
43 try {
44 root = rec(JSON.parse(input));
45 } catch {
46 return null;
47 }
48 const h = rec(root?.nvme_smart_health_information_log);
49 if (root === undefined || h === undefined) return null;
50 const temp = count(h.temperature) ?? (typeof h.temperature === "number" && Number.isFinite(h.temperature) ? h.temperature : undefined);
51 const n = {
52 criticalWarning: count(h.critical_warning),
53 spare: count(h.available_spare),
54 spareThreshold: count(h.available_spare_threshold),
55 percentUsed: count(h.percentage_used),
56 unitsWritten: count(h.data_units_written),
57 mediaErrors: count(h.media_errors),
58 powerOnHours: count(h.power_on_hours),
59 unsafeShutdowns: count(h.unsafe_shutdowns),
60 };
61 for (const v of Object.values(n)) if (v === undefined) return null;
62 const model = text(root.model_name);
63 if (temp === undefined || model === "") return null;
64 return {
65 model,
66 firmware: text(root.firmware_version),
67 serial: text(root.serial_number),
68 capacityBytes: count(rec(root.user_capacity)?.bytes) ?? count(root.nvme_total_capacity) ?? null,
69 tempC: temp,
70 minutesWarn: count(h.warning_temp_time) ?? 0,
71 minutesCrit: count(h.critical_comp_time) ?? 0,
72 ...(n as Record<keyof typeof n, number>),
73 };
74 }
src/hardware/store.ts added +45
@@ -0,0 +1,45 @@
1 /**
2 * Stored NVMe readings (`hardware_reading`): kept at most hourly per device,
3 * 90 days. The log is stored as the parsed numbers, and read back through the
4 * same shape check as an import.
5 */
6 import type { Database } from "bun:sqlite";
7 import { type NvmeLog } from "./smart.ts";
8
9 export const KEEP_MS = 90 * 86_400_000;
10 export const EVERY_MS = 3_600_000;
11
12 export interface Stored {
13 at: number;
14 device: string;
15 log: NvmeLog;
16 }
17
18 export function latest(db: Database, device: string): Stored | undefined {
19 const r = db.query("SELECT at, device, log FROM hardware_reading WHERE device = ? ORDER BY at DESC LIMIT 1").get(device) as { at: number; device: string; log: string } | null;
20 return r ? { at: r.at, device: r.device, log: JSON.parse(r.log) as NvmeLog } : undefined;
21 }
22
23 /** The reading before the latest one, for "rising" rules. */
24 export function previous(db: Database, device: string): Stored | undefined {
25 const r = db.query("SELECT at, device, log FROM hardware_reading WHERE device = ? ORDER BY at DESC LIMIT 1 OFFSET 1").get(device) as { at: number; device: string; log: string } | null;
26 return r ? { at: r.at, device: r.device, log: JSON.parse(r.log) as NvmeLog } : undefined;
27 }
28
29 export function since(db: Database, device: string, from: number): { at: number; unitsWritten: number }[] {
30 const rows = db.query("SELECT at, log FROM hardware_reading WHERE device = ? AND at >= ? ORDER BY at").all(device, from) as { at: number; log: string }[];
31 return rows.map((r) => ({ at: r.at, unitsWritten: (JSON.parse(r.log) as NvmeLog).unitsWritten }));
32 }
33
34 /** Records a reading unless one under an hour old exists (`force` for an import). Returns whether it did. */
35 export function record(db: Database, device: string, log: NvmeLog, at: number, force = false): boolean {
36 const last = latest(db, device);
37 if (!force && last !== undefined && at - last.at < EVERY_MS) return false;
38 db.query("INSERT INTO hardware_reading (at, device, model, serial, log) VALUES (?, ?, ?, ?, ?)").run(at, device, log.model, log.serial, JSON.stringify(log));
39 db.query("DELETE FROM hardware_reading WHERE at < ?").run(at - KEEP_MS);
40 return true;
41 }
42
43 export function devices(db: Database): string[] {
44 return (db.query("SELECT DISTINCT device FROM hardware_reading ORDER BY device").all() as { device: string }[]).map((r) => r.device);
45 }
src/hardware/view.ts added +45
@@ -0,0 +1,45 @@
1 /**
2 * The hardware answer, as plain data for the page and the JSON: the machine's
3 * inventory, and for each NVMe drive its stored reading set beside its
4 * rating, its alerts and its projection. A disk behind a RAID controller is
5 * named as such and given no wear.
6 */
7 import type { Database } from "bun:sqlite";
8 import { specFor, type Spec } from "./catalog.ts";
9 import { type DiskInfo, type Inventory } from "./inventory.ts";
10 import { alerts, bytesWritten, project, type Projection, WINDOW_DAYS } from "./rules.ts";
11 import { latest, previous, since, type Stored } from "./store.ts";
12
13 export interface DiskView {
14 disk: DiskInfo;
15 /** Present for an NVMe drive with a stored reading. */
16 reading: Stored | undefined;
17 spec: Spec | undefined;
18 alerts: string[];
19 writtenBytes: number | null;
20 projection: Projection | undefined;
21 }
22
23 export interface HardwareView {
24 inventory: Inventory;
25 disks: DiskView[];
26 }
27
28 export function hardwareView(db: Database, inventory: Inventory, now: number): HardwareView {
29 const disks = inventory.disks.map((disk): DiskView => {
30 const reading = disk.kind === "nvme" && disk.controller !== null ? latest(db, disk.controller) : undefined;
31 if (reading === undefined || disk.controller === null) return { disk, reading: undefined, spec: undefined, alerts: [], writtenBytes: null, projection: undefined };
32 const spec = specFor(reading.log.model);
33 const before = previous(db, disk.controller);
34 const written = bytesWritten(reading.log);
35 return {
36 disk,
37 reading,
38 spec,
39 alerts: alerts(reading.log, before ? before.log : undefined),
40 writtenBytes: written,
41 projection: project(since(db, disk.controller, now - WINDOW_DAYS * 86_400_000), now, spec, written),
42 };
43 });
44 return { inventory, disks };
45 }
src/http/server.ts +4
@@ -27,6 +27,7 @@ import { prerenderComments } from "../web/prerender.ts";
2727 import { systemPage, systemStream } from "../web/pages/system.ts";
2828 import { contributePage } from "../web/pages/contribute.ts";
2929 import { pipelinePage } from "../web/pages/pipeline.ts";
30 import { hardwareImport, hardwareJson, hardwarePage } from "../web/pages/hardware.ts";
3031 import { adminDataPage, appDataPage, dataPage } from "../web/pages/data.ts";
3132 import { fleetAct, fleetReceipt, fleetUndo } from "../web/pages/fleet-ops.ts";
3233 import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
@@ -998,6 +999,9 @@ export function buildRouter(): Router<Handler> {
998999 router.add("GET", "/boards/:slug/live", boardLive);
9991000 router.add("POST", "/boards/:slug/:op", boardPost);
10001001 router.add("GET", "/admin/data", adminDataPage);
1002 router.add("GET", "/admin/hardware", hardwarePage);
1003 router.add("GET", "/admin/hardware.json", hardwareJson);
1004 router.add("POST", "/admin/hardware/import", hardwareImport);
10011005 router.add("GET", "/admin/limits", limitsPage);
10021006 router.add("GET", "/admin/flows", allFlowsPage);
10031007 router.add("POST", "/admin/limits", limitsSet);
src/main.ts +4
@@ -53,6 +53,7 @@ import { VERIFY_EVERY_MS } from "./checks/verify.ts";
5353 import { RED_HOLD_MS } from "./checks/red_main.ts";
5454 import { stopQueue } from "./checks/queue.ts";
5555 import { startLooks } from "./looks/service.ts";
56 import { startHardware } from "./hardware/service.ts";
5657 import { startMirrorOut } from "./origins/mirror-out.ts";
5758 import { startMirrorPulls } from "./origins/mirror-pulls.ts";
5859 import { startProbes } from "./contract/probes.ts";
@@ -478,6 +479,8 @@ const mirrorPulls = startMirrorPulls(app);
478479 // The look: a picture of every environment that comes up green, taken
479480 // through this forge's own listener.
480481 const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined;
482 // The machine's NVMe health logs, once an hour, for /admin/hardware.
483 const hardware = startHardware(app);
481484 // Is it so: the forge knocks on every door of an app with a contract after
482485 // each production deploy and once a night, and compares with the table.
483486 const probes = apps === undefined ? undefined : startProbes(app, apps);
@@ -904,6 +907,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
904907 mirrorOut(),
905908 mirrorPulls(),
906909 looks?.stop(),
910 hardware(),
907911 probes?.stop(),
908912 nightly?.stop(),
909913 snapshots.stop(),
src/web/pages/hardware.ts added +128
@@ -0,0 +1,128 @@
1 /**
2 * The machine's hardware, for a forge admin:
3 * - GET /admin/hardware: CPU and memory, then one line per disk. A healthy
4 * NVMe drive is one quiet line; opening it shows every reading beside the
5 * vendor's rating. Only what the drive itself reports as bad is an alert.
6 * A RAID controller's virtual disk is "behind a controller; SMART not visible".
7 * - GET /admin/hardware.json: the same answer as JSON.
8 * - POST /admin/hardware/import: a pasted `smartctl -j -a` output stored as a
9 * reading, for a host with no smartctl of its own.
10 * Rules and ratings: `src/hardware/`.
11 */
12 import type { Ctx } from "../../http/context.ts";
13 import { json } from "../../http/respond.ts";
14 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
15 import { readInventory } from "../../hardware/inventory.ts";
16 import { parseSmart, warningBits } from "../../hardware/smart.ts";
17 import { record } from "../../hardware/store.ts";
18 import { hardwareView, type DiskView } from "../../hardware/view.ts";
19 import { notFoundPage, pageHead, settingsLayout } from "../chrome.ts";
20 import { field, readForm } from "../form.ts";
21 import { html, type Html } from "../html.ts";
22 import { flashTo, page } from "../layout.ts";
23 import { accountSections } from "./origins.ts";
24 import { badge, btn, icon, row, time } from "../ui.ts";
25
26 const mayRead = (ctx: Ctx): boolean => ctx.auth.user?.isAdmin === true && ctx.auth.user.kind !== "agent";
27
28 const gb = (n: number): string => (n >= 1e12 ? `${+(n / 1e12).toPrecision(3)} TB` : `${+(n / 1e9).toPrecision(3)} GB`);
29 const pct = (part: number, whole: number): string => `${+((part / whole) * 100).toPrecision(2)}%`;
30
31 export function hardwareJson(ctx: Ctx): Response {
32 if (!mayRead(ctx)) return notFoundPage(ctx);
33 return json(hardwareView(ctx.app.db, readInventory(), Date.now()));
34 }
35
36 export function hardwarePage(ctx: Ctx): Response {
37 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
38 if (!mayRead(ctx)) return notFoundPage(ctx);
39 const rendering = performance.now();
40 const now = Date.now();
41 const view = hardwareView(ctx.app.db, readInventory(), now);
42 const inv = view.inventory;
43 const body = html`${pageHead("Hardware", { sub: "Read from the machine itself; ratings are the vendor's. Only what a drive reports as bad is an alert." })}
44 <p class="muted">${inv.cpu === "" ? "CPU unknown" : `${inv.cpu} · ${inv.threads} threads`} · ${inv.memoryBytes === null ? "memory unknown" : `${gb(inv.memoryBytes)} memory`}</p>
45 <section class="settings-block">
46 <h2>Disks</h2>
47 ${view.disks.length === 0 ? html`<p class="muted">No block devices are visible to the forge.</p>` : html`<ul class="list" id="disks">${view.disks.map((d) => diskRow(d, now))}</ul>`}
48 </section>
49 <p class="muted small"><a href="/admin/hardware.json">This page as JSON</a></p>
50 <section class="settings-block">
51 <h2>Add a reading</h2>
52 <p class="muted small">Without smartctl on this host, paste the output of <span class="mono">smartctl -j -a /dev/nvme0</span>. It is stored as a reading at the time you paste it.</p>
53 <form method="post" action="/admin/hardware/import">
54 <label class="muted small" for="smart">Device</label>
55 <input class="input mono" id="device" name="device" value="nvme0" pattern="nvme[0-9]{1,3}" required />
56 <textarea class="input mono" id="smart" name="smart" rows="4" required aria-label="smartctl JSON"></textarea>
57 ${btn("Store reading", { variant: "primary", size: "sm" })}
58 </form>
59 </section>`;
60 const response = page({ ctx, title: "Hardware · admin", body: settingsLayout(accountSections("hardware", { admin: true }), body, "Account settings") });
61 notePhase(ctx.request, "render", performance.now() - rendering);
62 return response;
63 }
64
65 export function diskRow(d: DiskView, now: number): Html {
66 const { disk } = d;
67 const size = gb(disk.capacityBytes);
68 if (disk.kind === "controller")
69 return row({ lead: icon("terminal"), title: disk.name, meta: html`${disk.model} · ${size} · behind a controller; SMART not visible` });
70 if (d.reading === undefined) {
71 const why = disk.kind === "nvme" ? "no reading yet (needs smartctl on the host, or paste one below)" : "SMART is read for NVMe drives only";
72 return row({ lead: icon("terminal"), title: disk.name, meta: html`${disk.model === "" ? "unknown model" : disk.model} · ${size} · ${why}` });
73 }
74 const l = d.reading.log;
75 const bad = d.alerts.length > 0;
76 const state = bad ? badge(`${d.alerts.length} to look at`, { tone: "danger" }) : badge("healthy", { tone: "success" });
77 const spec = d.spec;
78 const written = d.writtenBytes ?? 0;
79 const line = (label: string, value: string, rating?: string) =>
80 html`<tr><th scope="row">${label}</th><td>${value}</td><td class="muted">${rating ?? ""}</td></tr>`;
81 const wl = warningBits(l.criticalWarning);
82 const projection = d.projection;
83 const proj =
84 projection === undefined ? "" : projection.kind === "none" ? `no projection: ${projection.why}` : projection.kind === "idle" ? "no writes in the last 30 days" : `${new Date(projection.at).toISOString().slice(0, 10)} at ${gb(projection.bytesPerDay)}/day`;
85 return row({
86 lead: icon("chevronRight"),
87 title: html`${disk.name} · ${l.model}`,
88 meta: html`${size} · firmware ${l.firmware} · ${spec ? html`${gb(written)} of ${gb(spec.ratedBytes)} (${pct(written, spec.ratedBytes)})` : html`${gb(written)} written, no rating for this model`}`,
89 trail: state,
90 fold: {
91 id: `disk-${disk.name}`,
92 open: bad,
93 body: html`<div class="card-body">
94 ${bad ? html`<ul class="alerts">${d.alerts.map((a) => html`<li>${a}</li>`)}</ul>` : null}
95 <table class="table setting-table" data-dense>
96 <tbody>
97 ${line("Written", gb(written), spec ? `of ${gb(spec.ratedBytes)} rated (${pct(written, spec.ratedBytes)})` : "no rating on file")}
98 ${line("Percentage used", `${l.percentUsed}%`, "alert from 80%")}
99 ${line("Available spare", `${l.spare}%`, `drive threshold ${l.spareThreshold}%`)}
100 ${line("Media errors", String(l.mediaErrors), "alert when rising")}
101 ${line("Critical warning", wl.length === 0 ? "none" : wl.join(", "), "alert when any bit is set")}
102 ${line("Temperature", `${l.tempC} °C`, spec ? `warning ${spec.tempWarnC} °C, critical ${spec.tempCritC} °C` : "no rating on file")}
103 ${line("Time above warning", `${l.minutesWarn} min`)}
104 ${line("Time above critical", `${l.minutesCrit} min`, "alert when rising")}
105 ${line("Power-on hours", l.powerOnHours.toLocaleString("en-US"))}
106 ${line("Unsafe shutdowns", String(l.unsafeShutdowns))}
107 ${line("Rated warranty", spec ? `${spec.warrantyYears} years` : "unknown", spec?.source)}
108 ${line("Rated TBW reached", proj === "" ? "unknown" : proj, "at the last 30 days' write rate")}
109 </tbody>
110 </table>
111 <p class="muted small">Read ${time(new Date(d.reading.at).toISOString())}; a data unit is 512,000 bytes (NVMe spec), TBW is 10^12 bytes.${now - d.reading.at > 6 * 3_600_000 ? " This reading is more than six hours old." : ""}</p>
112 </div>`,
113 },
114 });
115 }
116
117 // POST /admin/hardware/import
118 export async function hardwareImport(ctx: Ctx): Promise<Response> {
119 if (!mayRead(ctx)) return notFoundPage(ctx);
120 const form = await readForm(ctx, "admin");
121 if (form instanceof Response) return form;
122 const device = field(form, "device").trim();
123 const log = parseSmart(field(form, "smart"));
124 if (!/^nvme\d{1,3}$/.test(device) || log === null)
125 return flashTo(ctx, "/admin/hardware", { tone: "danger", message: "That is not a device name like nvme0 with the JSON from smartctl -j -a." });
126 record(ctx.app.db, device, log, Date.now(), true);
127 return flashTo(ctx, "/admin/hardware", { message: "Reading stored." });
128 }
src/web/pages/origins.ts +2
@@ -88,6 +88,7 @@ import {
8888
8989 export type AccountSection =
9090 | "adminflows"
91 | "hardware"
9192 | "security"
9293 | "profile"
9394 | "confirms"
@@ -159,6 +160,7 @@ const SECTION_ROWS: readonly (readonly [
159160 ["This forge", "Limits", "/admin/limits", "activity", (c) => c === "limits", true],
160161 ["This forge", "Economics", "/settings/economics", "activity", (c) => c === "economics", true],
161162 ["This forge", "Flows", "/admin/flows", "activity", (c) => c === "adminflows", true],
163 ["This forge", "Hardware", "/admin/hardware", "activity", (c) => c === "hardware", true],
162164 ["This forge", "Flags", "/settings/flags", "activity", (c) => c === "flags", true],
163165 ["This forge", "Features", "/settings/features", "rocket", (c) => c === "features", true],
164166 ["This forge", "Backups", "/settings/backups", "download", (c) => c === "backups", true],
test/hardware.test.ts added +188
@@ -0,0 +1,188 @@
1 /** Hardware health: recorded smartctl logs against the alert rules, the inventory, the projection, and the page. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { mkdirSync, mkdtemp as _m, writeFileSync } from "node:fs";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { createSession } from "../src/core/sessions.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { readSmart } from "../src/hardware/collect.ts";
11 import { readInventory } from "../src/hardware/inventory.ts";
12 import { alerts, project } from "../src/hardware/rules.ts";
13 import { specFor } from "../src/hardware/catalog.ts";
14 import { parseSmart, type NvmeLog } from "../src/hardware/smart.ts";
15 import { hardwareView } from "../src/hardware/view.ts";
16 import { diskRow } from "../src/web/pages/hardware.ts";
17 import { record } from "../src/hardware/store.ts";
18 import { serve, type Servers } from "../src/http/server.ts";
19 import { quietLogs } from "./helpers/globals.ts";
20
21 void _m;
22 quietLogs();
23
24 /** A recorded `smartctl -j -a` for a 980 PRO; fields override the healthy reading of 11-09 20:45Z. */
25 function recorded(over: Record<string, number> = {}): string {
26 const h = {
27 critical_warning: 0, temperature: 39, available_spare: 100, available_spare_threshold: 10, percentage_used: 0,
28 data_units_written: 1_074_219, data_units_read: 2_000_000, power_on_hours: 2575, unsafe_shutdowns: 41,
29 media_errors: 0, warning_temp_time: 0, critical_comp_time: 0, ...over,
30 };
31 return JSON.stringify({ model_name: "Samsung SSD 980 PRO 1TB", firmware_version: "5B2QGXA7", serial_number: "S5GXNX0T000000", user_capacity: { bytes: 1_000_204_886_016 }, nvme_smart_health_information_log: h });
32 }
33 const log = (over: Record<string, number> = {}): NvmeLog => parseSmart(recorded(over))!;
34
35 describe("alert rules against recorded logs", () => {
36 test("healthy: today's reading is quiet, whatever the unsafe shutdowns", () => {
37 const l = log();
38 expect(l.firmware).toBe("5B2QGXA7");
39 expect(alerts(l, undefined)).toEqual([]);
40 expect(alerts(l, { mediaErrors: 0, minutesCrit: 0 })).toEqual([]);
41 });
42 test("warm and busy but not bad stays quiet: 70 C, 60 minutes above warning, 79% used", () => {
43 expect(alerts(log({ temperature: 70, warning_temp_time: 60, percentage_used: 79 }), { mediaErrors: 0, minutesCrit: 0 })).toEqual([]);
44 });
45 test("worn: percentage used at 80", () => {
46 expect(alerts(log({ percentage_used: 80 }), undefined)).toHaveLength(1);
47 expect(alerts(log({ percentage_used: 80 }), undefined)[0]).toContain("80%");
48 });
49 test("hot: time above critical rising, but not merely nonzero", () => {
50 expect(alerts(log({ critical_comp_time: 12 }), undefined)).toEqual([]);
51 expect(alerts(log({ critical_comp_time: 12 }), { mediaErrors: 0, minutesCrit: 12 })).toEqual([]);
52 expect(alerts(log({ critical_comp_time: 15, critical_warning: 2 }), { mediaErrors: 0, minutesCrit: 12 })).toHaveLength(2);
53 });
54 test("failing: critical bit, spare below threshold, media errors rising", () => {
55 const failing = log({ critical_warning: 1, available_spare: 5, media_errors: 7 });
56 const out = alerts(failing, { mediaErrors: 3, minutesCrit: 0 });
57 expect(out).toHaveLength(3);
58 expect(out.join(" ")).toContain("available spare below threshold");
59 expect(out.join(" ")).toContain("from 3 to 7");
60 });
61 test("static media errors are a number, not an alert", () => {
62 expect(alerts(log({ media_errors: 7 }), { mediaErrors: 7, minutesCrit: 0 })).toEqual([]);
63 });
64 test("a log missing a field is refused, never read as zero", () => {
65 const o = JSON.parse(recorded());
66 delete o.nvme_smart_health_information_log.media_errors;
67 expect(parseSmart(JSON.stringify(o))).toBeNull();
68 expect(parseSmart("not json")).toBeNull();
69 });
70 });
71
72 describe("spec sheet and projection", () => {
73 test("980 PRO 1TB is rated; an unknown model is not guessed", () => {
74 const s = specFor("Samsung SSD 980 PRO 1TB")!;
75 expect([s.ratedBytes, s.warrantyYears, s.tempWarnC, s.tempCritC]).toEqual([600e12, 5, 82, 85]);
76 expect(specFor("Samsung SSD 980 PRO 2TB")).toBeUndefined();
77 });
78 test("a hand calculation: 1 TB per day from 550 GB reaches 600 TB in 599.45 days", () => {
79 const day = 86_400_000;
80 const now = 100 * day;
81 const unitsPerDay = 1e12 / 512_000;
82 const points = [{ at: now - 10 * day, unitsWritten: 0 }, { at: now, unitsWritten: 10 * unitsPerDay }];
83 const p = project(points, now, specFor("Samsung SSD 980 PRO 1TB"), 550e9);
84 if (p.kind !== "date") throw new Error("expected a date");
85 expect(p.bytesPerDay).toBeCloseTo(1e12, 0);
86 expect((p.at - now) / day).toBeCloseTo(599.45, 6);
87 });
88 test("no history, no rating and no writes say so", () => {
89 const spec = specFor("Samsung SSD 980 PRO 1TB");
90 expect(project([{ at: 1, unitsWritten: 5 }], 2, spec, 0).kind).toBe("none");
91 expect(project([], 9e9, undefined, 0).kind).toBe("none");
92 const day = 86_400_000;
93 expect(project([{ at: 0, unitsWritten: 5 }, { at: 3 * day, unitsWritten: 5 }], 3 * day, spec, 0).kind).toBe("idle");
94 });
95 });
96
97 describe("inventory", () => {
98 test("a PERC virtual disk is behind a controller; the NVMe is read from its controller", async () => {
99 const dir = await mkdtemp(join(tmpdir(), "forge-hw-"));
100 const put = (path: string, text: string) => {
101 mkdirSync(join(dir, path, ".."), { recursive: true });
102 writeFileSync(join(dir, path), text);
103 };
104 put("sys/block/sda/size", "1874329600\n");
105 put("sys/block/sda/queue/rotational", "0\n");
106 put("sys/block/sda/device/vendor", "DELL \n");
107 put("sys/block/sda/device/model", "PERC H730P Mini\n");
108 put("sys/block/nvme0n1/size", "1953525168\n");
109 put("sys/block/nvme0n1/queue/rotational", "0\n");
110 put("sys/class/nvme/nvme0/model", "Samsung SSD 980 PRO 1TB\n");
111 put("sys/class/nvme/nvme0/firmware_rev", "5B2QGXA7\n");
112 put("sys/block/loop0/size", "100\n");
113 put("proc/cpuinfo", "processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Gold 6130\nprocessor\t: 1\nmodel name\t: Intel(R) Xeon(R) Gold 6130\n");
114 put("proc/meminfo", "MemTotal: 131072 kB\n");
115 const inv = readInventory({ proc: join(dir, "proc"), sys: join(dir, "sys") });
116 await rm(dir, { recursive: true });
117 expect(inv.threads).toBe(2);
118 expect(inv.cpu).toBe("Intel(R) Xeon(R) Gold 6130");
119 expect(inv.memoryBytes).toBe(131072 * 1024);
120 expect(inv.disks.map((d) => [d.name, d.kind])).toEqual([["nvme0n1", "nvme"], ["sda", "controller"]]);
121 const nvme = inv.disks[0]!;
122 expect([nvme.model, nvme.firmware, nvme.controller, nvme.capacityBytes]).toEqual(["Samsung SSD 980 PRO 1TB", "5B2QGXA7", "nvme0", 1953525168 * 512]);
123 });
124 test("smartctl is called for a kernel controller name only, and its absence is an answer", async () => {
125 const calls: string[][] = [];
126 const ok = await readSmart("nvme0", async (argv) => (calls.push(argv), recorded()));
127 expect(ok.ok).toBe(true);
128 expect(calls[0]).toEqual(["smartctl", "-j", "-a", "/dev/nvme0"]);
129 expect((await readSmart("nvme0; rm -rf /", async () => recorded())).ok).toBe(false);
130 const none = await readSmart("nvme0", async () => null);
131 expect(none.ok ? "" : none.why).toContain("not available");
132 });
133 });
134
135 describe("the hardware page", () => {
136 let dir: string;
137 let app: App;
138 let servers: Servers;
139 let admin = "";
140 let other = "";
141 beforeAll(async () => {
142 dir = await mkdtemp(join(tmpdir(), "forge-hw-page-"));
143 app = createApp({ dataDir: dir, port: 0, host: "forge.test" });
144 servers = await serve(app);
145 for (const name of ["boss", "pal"]) {
146 const made = await createUser(app.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: name === "boss" });
147 if (!made.ok) throw new Error(made.error);
148 const c = `forge_session=${createSession(app.db, made.value.id).token}`;
149 if (name === "boss") admin = c;
150 else other = c;
151 }
152 });
153 afterAll(async () => {
154 await servers.stop?.();
155 await rm(dir, { recursive: true, force: true });
156 });
157 const get = (path: string, cookie: string) => fetch(new URL(path, servers.url), { headers: { cookie } });
158
159 test("only a site admin sees it", async () => {
160 expect((await get("/admin/hardware", other)).status).toBe(403);
161 expect((await get("/admin/hardware.json", other)).status).not.toBe(200);
162 expect((await get("/admin/hardware", admin)).status).toBe(200);
163 });
164 test("one quiet line when healthy, open with detail when a drive reports bad, a controller named as such", () => {
165 const inventory = {
166 cpu: "x", threads: 1, memoryBytes: null,
167 disks: [
168 { name: "nvme0n1", kind: "nvme" as const, model: "Samsung SSD 980 PRO 1TB", firmware: "5B2QGXA7", capacityBytes: 1e12, controller: "nvme0" },
169 { name: "sda", kind: "controller" as const, model: "DELL PERC H730P Mini", firmware: "", capacityBytes: 960e9, controller: null },
170 ],
171 };
172 const now = Date.now();
173 record(app.db, "nvme0", log(), now);
174 const healthy = hardwareView(app.db, inventory, now);
175 expect(healthy.disks[0]!.alerts).toEqual([]);
176 const quiet = diskRow(healthy.disks[0]!, now).text;
177 expect(quiet).toContain("healthy");
178 expect(quiet).toContain("550 GB of 600 TB (0.092%)");
179 expect(quiet).toContain("Unsafe shutdowns");
180 expect(quiet).not.toContain("to look at");
181 expect(diskRow(healthy.disks[1]!, now).text).toContain("behind a controller; SMART not visible");
182
183 record(app.db, "nvme0", log({ percentage_used: 85 }), now + 4_000_000);
184 const worn = diskRow(hardwareView(app.db, inventory, now + 4_000_000).disks[0]!, now + 4_000_000).text;
185 expect(worn).toContain("1 to look at");
186 expect(worn).toContain("Percentage used is 85%");
187 });
188 });
test/spawn-pipes.test.ts +1
@@ -29,6 +29,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
2929 "src/git/read.ts": { sites: ["stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed); stderr ignored" },
3030 "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" },
3131 "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" },
32 "src/hardware/collect.ts": { sites: ["stdout"], reason: "stdout read whole via Response.text() under a 10 s kill timer; stderr ignored" },
3233 "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
3334 "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
3435 "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs