trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns11g rEventsFlows

Telemetry history: per-core CPU and temperature series on the machine board (#1789 follow-up) #2141

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/2081-telemetry-history-per-core-cpu-and-tempe → main 1 commit fork

  • Merged. torque/2081-telemetry-history-per-core-cpu-and-tempe was merged into main as 5ba342b.

Review

Sign in to review this change.
11 files changed +570 −37
docs/APPS.md +13 −2
@@ -1149,7 +1149,8 @@ a column holding a secret or free text is never listed).
11491149 - `source`: a name from the list; `aggregate`: `count`, `sum`, `avg` or `max`
11501150 (the last three need a numeric `column`).
11511151 - `from` (inclusive) and `to` (exclusive): dates, applied to the source's time column.
1152 - `filter`: up to 16 of `{column, op, value}`, `op` one of `eq ne lt lte gt gte`.
1152 - `filter`: up to 16 of `{column, op, value}`, `op` one of `eq ne lt lte gt gte`,
1153 or `starts` (a prefix of a text or time column, matched literally).
11531154 - `groupBy`: a column, or `{"bucket": "minute" | "hour" | "day"}`.
11541155
11551156 It answers `{"rows": [{"group", "value"}]}`, at most 1000 rows. A query the
@@ -1160,7 +1161,17 @@ The `machine` source is what the machine itself used (`boards/sampler.ts`):
11601161 every 15 s the forge reads its pod's cgroup v2 (`cpu.stat` `usage_usec` as a
11611162 rate, `memory.current`), each labelled check and app cell's process tree
11621163 (`check:<run id>`, `app:<slug>/<env>`), and the node (`/proc/stat`,
1163 `/proc/meminfo`, `/proc/loadavg`). `boards/ring.ts` keeps it in
1164 `/proc/meminfo`, `/proc/loadavg`). Each logical CPU is a target of its own,
1165 `cpu:<nnn>` (zero-padded, so they sort): its `/proc/stat` line as a rate, at
1166 most 1 core, and in `celsius` the temperature of the core it runs on — the
1167 `coretemp` chip's `Core <K>` input under `/sys/class/hwmon`, matched by the
1168 CPU's `topology/physical_package_id` and `core_id`. Every other thermometer
1169 (a `coretemp` package, `k10temp`, `acpitz`, a disk) is `package:<id>` or
1170 `sensor:<chip>/<label>`, °C only. A board picks the cores with the filter
1171 `target starts cpu:`; the machine card on `/boards/data?source=machine`
1172 links a per-core CPU and a per-core temperature query. On an 80-CPU node
1173 that is 80 more targets, about 1.2 M rows at most (~80 MB) in the ring.
1174 `boards/ring.ts` keeps it in
11641175 `machine_sample`: 15-s points for 24 h and 5-min mean and max for 30 days,
11651176 dropping what leaves its window on every write. A file this machine lacks
11661177 leaves its series absent, never zero, and `/boards/data?source=machine`
src/boards/query.ts +6 −2
@@ -6,7 +6,8 @@
66 import type { Database } from "bun:sqlite";
77 import { SOURCES, type ColumnType } from "./sources.ts";
88
9 const OPS = { eq: "=", ne: "<>", lt: "<", lte: "<=", gt: ">", gte: ">=" } as const;
9 /** `starts` is a prefix match on a text or time column (`target` starts `cpu:`); the rest compare. */
10 const OPS = { eq: "=", ne: "<>", lt: "<", lte: "<=", gt: ">", gte: ">=", starts: null } as const;
1011 const BUCKETS = {
1112 minute: "%Y-%m-%dT%H:%M:00Z",
1213 hour: "%Y-%m-%dT%H:00:00Z",
@@ -95,8 +96,11 @@ export function runQuery(db: Database, q: unknown, scope: Scope): Answer {
9596 if (typeof column !== "string" || !has(cols, column)) return refuse(`unknown column: ${JSON.stringify(column)}`);
9697 if (typeof op !== "string" || !has(OPS, op)) return refuse(`unknown op: ${JSON.stringify(op)}`);
9798 if (!valueFits(cols[column]!, value)) return refuse(`bad value for column ${column}`);
99 const sqlOp = OPS[op as Op];
100 if (sqlOp === null && cols[column] === "number") return refuse(`column ${column} is numeric: starts needs text`);
98101 params.push(value as string | number);
99 where.push(`${ident(column)} ${OPS[op as Op]} ?`);
102 // instr(), not LIKE: a % or _ in the value is a character, never a wildcard.
103 where.push(sqlOp === null ? `instr(${ident(column)}, ?) = 1` : `${ident(column)} ${sqlOp} ?`);
100104 }
101105
102106 const agg = query.aggregate;
src/boards/ring.ts +19 −6
@@ -22,6 +22,8 @@ export type Point = {
2222 memoryBytes: number | null;
2323 /** The node's 1-min load average, runnable tasks; null for every other target. */
2424 load: number | null;
25 /** °C, for a CPU or a sensor with a thermometer (`boards/thermal.ts`); absent for every other target. */
26 celsius?: number;
2527 };
2628
2729 /** One target over one 5-min window, `at` its start. */
@@ -33,6 +35,8 @@ export type Rolled = {
3335 memoryBytes: number | null;
3436 memoryBytesMax: number | null;
3537 load: number | null;
38 celsius: number | null;
39 celsiusMax: number | null;
3640 };
3741
3842 const iso = (ms: number) => new Date(ms).toISOString();
@@ -58,27 +62,32 @@ export function rollUp(points: readonly Point[]): Rolled[] {
5862 .map(({ at, target, ps }) => {
5963 const cpu = stats(ps.map((p) => p.cpuCores));
6064 const mem = stats(ps.map((p) => p.memoryBytes));
65 const temp = stats(ps.map((p) => p.celsius ?? null));
6166 return {
6267 at, target,
6368 cpuCores: cpu.mean, cpuCoresMax: cpu.max,
6469 memoryBytes: mem.mean, memoryBytesMax: mem.max,
6570 load: stats(ps.map((p) => p.load)).mean,
71 celsius: temp.mean, celsiusMax: temp.max,
6672 };
6773 });
6874 }
6975
70 type FineRow = { at: string; target: string; cpu_cores: number | null; memory_bytes: number | null; load: number | null };
76 type FineRow = {
77 at: string; target: string; cpu_cores: number | null; memory_bytes: number | null; load: number | null; celsius: number | null;
78 };
7179
7280 /** Keep `points` (taken at `now`), roll up the windows that closed, drop what is past its window. One transaction. */
7381 export function record(db: Database, points: readonly Point[], now: number): void {
7482 const put = db.query(
7583 `INSERT OR REPLACE INTO machine_sample
76 (at, step, target, cpu_cores, cpu_cores_max, memory_bytes, memory_bytes_max, load)
77 VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
84 (at, step, target, cpu_cores, cpu_cores_max, memory_bytes, memory_bytes_max, load, celsius, celsius_max)
85 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
7886 );
7987 db.transaction(() => {
8088 for (const p of points) {
81 put.run(iso(p.at), 15, p.target, p.cpuCores, p.cpuCores, p.memoryBytes, p.memoryBytes, p.load);
89 const c = p.celsius ?? null;
90 put.run(iso(p.at), 15, p.target, p.cpuCores, p.cpuCores, p.memoryBytes, p.memoryBytes, p.load, c, c);
8291 }
8392 // Every window that ended by now and is not rolled yet: after the last rolled one.
8493 const last = db.query<{ at: string | null }, []>(
@@ -87,13 +96,17 @@ export function record(db: Database, points: readonly Point[], now: number): voi
8796 const from = last == null ? "" : iso(Date.parse(last) + COARSE_MS);
8897 const to = iso(Math.floor(now / COARSE_MS) * COARSE_MS);
8998 const fine = db.query<FineRow, [string, string]>(
90 `SELECT at, target, cpu_cores, memory_bytes, load FROM machine_sample
99 `SELECT at, target, cpu_cores, memory_bytes, load, celsius FROM machine_sample
91100 WHERE step = 15 AND at >= ? AND at < ?`,
92101 ).all(from, to);
93102 for (const r of rollUp(fine.map((f) => ({
94103 at: Date.parse(f.at), target: f.target, cpuCores: f.cpu_cores, memoryBytes: f.memory_bytes, load: f.load,
104 ...(f.celsius === null ? {} : { celsius: f.celsius }),
95105 })))) {
96 put.run(iso(r.at), 300, r.target, r.cpuCores, r.cpuCoresMax, r.memoryBytes, r.memoryBytesMax, r.load);
106 put.run(
107 iso(r.at), 300, r.target, r.cpuCores, r.cpuCoresMax, r.memoryBytes, r.memoryBytesMax, r.load,
108 r.celsius, r.celsiusMax,
109 );
97110 }
98111 db.query("DELETE FROM machine_sample WHERE step = 15 AND at < ?").run(iso(now - FINE_KEEP_MS));
99112 db.query("DELETE FROM machine_sample WHERE step = 300 AND at < ?").run(iso(now - COARSE_KEEP_MS));
src/boards/sampler.ts +62 −9
@@ -10,6 +10,11 @@
1010 * from `/proc`: user + system CPU as a rate, and resident memory.
1111 * - `node`: `/proc/stat` busy time as a rate, `MemTotal − MemAvailable`
1212 * from `/proc/meminfo`, and the 1-min load from `/proc/loadavg`.
13 * - `cpu:<n>` (n zero-padded to three digits, so they sort): each logical
14 * CPU's `cpu<n>` line of `/proc/stat` as a rate — at most 1 core — and
15 * its core's temperature in °C (`boards/thermal.ts`).
16 * - `package:<P>` and `sensor:<chip>/<label>`: every other thermometer
17 * under `/sys/class/hwmon`, °C only.
1318 *
1419 * A file that is missing or unreadable (not Linux, no cgroup v2) leaves its
1520 * series null — never a guess — and `gaps` names it for the page.
@@ -19,9 +24,10 @@ import { join } from "node:path";
1924 import type { App } from "../app.ts";
2025 import { cellCounters, type CellCounter } from "../apps/contain.ts";
2126 import { FINE_MS, record, type Point } from "./ring.ts";
27 import { readHwmon, readTopology, temperatures } from "./thermal.ts";
2228
23 export type Paths = { cgroup: string; proc: string };
24 export const SYSTEM: Paths = { cgroup: "/sys/fs/cgroup", proc: "/proc" };
29 export type Paths = { cgroup: string; proc: string; sys: string };
30 export const SYSTEM: Paths = { cgroup: "/sys/fs/cgroup", proc: "/proc", sys: "/sys" };
2531
2632 /** `/proc/stat` counts in USER_HZ, 100 a second on every Linux the forge runs on. */
2733 const USER_HZ = 100;
@@ -33,6 +39,10 @@ export type Counters = {
3339 forge: { cpuUsec: number | null; memoryBytes: number | null };
3440 node: { cpuUsec: number | null; memoryBytes: number | null; load: number | null };
3541 cells: readonly CellCounter[];
42 /** Each logical CPU: busy µs since boot, and its core's °C when a thermometer maps to it. */
43 cpus: readonly { id: number; busyUsec: number; celsius: number | null }[];
44 /** Every thermometer no CPU took, °C. */
45 sensors: readonly { target: string; celsius: number }[];
3646 };
3747
3848 function read(path: string): string | null {
@@ -51,16 +61,33 @@ export function parseCpuStat(text: string | null): number | null {
5161 return m ? finite(Number(m[1])) : null;
5262 }
5363
54 /** Busy CPU time of the whole node from `/proc/stat`'s `cpu` line, µs: user nice system irq softirq steal (guest is inside user). */
55 export function parseProcStat(text: string | null): number | null {
56 const m = text === null ? null : /^cpu +(.*)$/m.exec(text);
57 if (!m) return null;
58 const f = m[1]!.trim().split(/\s+/).map(Number);
64 /** Busy µs from one `/proc/stat` line's tick fields: user nice system irq softirq steal (guest is inside user). */
65 function busyUsec(fields: string): number | null {
66 const f = fields.trim().split(/\s+/).map(Number);
5967 if (f.length < 4 || f.some((n) => !Number.isFinite(n))) return null;
6068 const busy = f[0]! + f[1]! + f[2]! + (f[5] ?? 0) + (f[6] ?? 0) + (f[7] ?? 0);
6169 return (busy * 1_000_000) / USER_HZ;
6270 }
6371
72 /** Busy CPU time of the whole node from `/proc/stat`'s `cpu` line, µs. */
73 export function parseProcStat(text: string | null): number | null {
74 const m = text === null ? null : /^cpu +(.*)$/m.exec(text);
75 return m ? busyUsec(m[1]!) : null;
76 }
77
78 /** Busy time of each logical CPU from `/proc/stat`'s `cpu<n>` lines, µs, in CPU order. */
79 export function parseProcStatCpus(text: string | null): { id: number; busyUsec: number }[] {
80 const out: { id: number; busyUsec: number }[] = [];
81 for (const m of text?.matchAll(/^cpu(\d+) +(.*)$/gm) ?? []) {
82 const busy = busyUsec(m[2]!);
83 if (busy !== null) out.push({ id: Number(m[1]), busyUsec: busy });
84 }
85 return out.sort((a, b) => a.id - b.id);
86 }
87
88 /** A logical CPU's target: `cpu:` and its number, zero-padded to three digits so a board lists them in order. */
89 export const cpuTarget = (id: number) => `cpu:${String(id).padStart(3, "0")}`;
90
6491 /** Memory in use on the node, bytes: `MemTotal − MemAvailable` (kB × 1024). */
6592 export function parseMeminfo(text: string | null): number | null {
6693 if (text === null) return null;
@@ -78,6 +105,9 @@ export function parseLoadavg(text: string | null): number | null {
78105 /** Read every counter now. */
79106 export function readCounters(paths: Paths, cells: readonly CellCounter[], at: number): Counters {
80107 const mem = read(join(paths.cgroup, "memory.current"));
108 const stat = read(join(paths.proc, "stat"));
109 const perCpu = parseProcStatCpus(stat);
110 const temps = temperatures(readHwmon(paths.sys), readTopology(paths.sys, perCpu.map((c) => c.id)));
81111 return {
82112 at,
83113 forge: {
@@ -85,11 +115,13 @@ export function readCounters(paths: Paths, cells: readonly CellCounter[], at: nu
85115 memoryBytes: mem === null || !/^\d+\s*$/.test(mem) ? null : Number(mem),
86116 },
87117 node: {
88 cpuUsec: parseProcStat(read(join(paths.proc, "stat"))),
118 cpuUsec: parseProcStat(stat),
89119 memoryBytes: parseMeminfo(read(join(paths.proc, "meminfo"))),
90120 load: parseLoadavg(read(join(paths.proc, "loadavg"))),
91121 },
92122 cells,
123 cpus: perCpu.map((c) => ({ ...c, celsius: temps.perCpu.get(c.id) ?? null })),
124 sensors: temps.sensors,
93125 };
94126 }
95127
@@ -109,7 +141,7 @@ export function pointsOf(prev: Counters | undefined, next: Counters): Point[] {
109141 const seconds = prev === undefined ? 0 : (next.at - prev.at) / 1000;
110142 const out: Point[] = [];
111143 const push = (p: Point) => {
112 if (p.cpuCores !== null || p.memoryBytes !== null || p.load !== null) out.push(p);
144 if (p.cpuCores !== null || p.memoryBytes !== null || p.load !== null || p.celsius != null) out.push(p);
113145 };
114146 push({
115147 at: next.at, target: "forge",
@@ -131,6 +163,18 @@ export function pointsOf(prev: Counters | undefined, next: Counters): Point[] {
131163 cpuCores: cores(prev?.node.cpuUsec, next.node.cpuUsec, seconds),
132164 memoryBytes: next.node.memoryBytes, load: next.node.load,
133165 });
166 const busyBefore = new Map((prev?.cpus ?? []).map((c) => [c.id, c.busyUsec]));
167 for (const c of next.cpus) {
168 const p: Point = {
169 at: next.at, target: cpuTarget(c.id),
170 cpuCores: cores(busyBefore.get(c.id), c.busyUsec, seconds), memoryBytes: null, load: null,
171 };
172 if (c.celsius !== null) p.celsius = c.celsius;
173 push(p);
174 }
175 for (const s of next.sensors) {
176 push({ at: next.at, target: s.target, cpuCores: null, memoryBytes: null, load: null, celsius: s.celsius });
177 }
134178 return out;
135179 }
136180
@@ -144,6 +188,15 @@ export function gaps(paths: Paths): string[] {
144188 if (missing(join(paths.proc, "stat"))) out.push(say("node cpu_cores", join(paths.proc, "stat")));
145189 if (missing(join(paths.proc, "meminfo"))) out.push(say("node memory_bytes", join(paths.proc, "meminfo")));
146190 if (missing(join(paths.proc, "loadavg"))) out.push(say("node load", join(paths.proc, "loadavg")));
191 const cpus = parseProcStatCpus(read(join(paths.proc, "stat"))).map((c) => c.id);
192 if (cpus.length === 0) out.push(`cpu:<n> cpu_cores: ${join(paths.proc, "stat")} has no per-CPU lines here`);
193 const hwmon = join(paths.sys, "class", "hwmon");
194 const temps = temperatures(readHwmon(paths.sys), readTopology(paths.sys, cpus));
195 if (temps.perCpu.size === 0 && temps.sensors.length === 0) {
196 out.push(`celsius: ${hwmon} has no readable temperature here`);
197 } else if (cpus.length > 0 && temps.perCpu.size === 0) {
198 out.push(`cpu:<n> celsius: no coretemp chip under ${hwmon} maps a temperature to each core here`);
199 }
147200 return out;
148201 }
149202
src/boards/sources.ts +7 −2
@@ -112,16 +112,21 @@ export const SOURCES: Readonly<Record<string, Source>> = {
112112 target: "text", at: "time", step: "number",
113113 cpu_cores: "number", cpu_cores_max: "number",
114114 memory_bytes: "number", memory_bytes_max: "number", load: "number",
115 celsius: "number", celsius_max: "number",
115116 },
116117 docs: {
117 target: "forge (the pod), check:<run id>, app:<slug>/<env>, or node",
118 target:
119 "forge (the pod), check:<run id>, app:<slug>/<env>, node, cpu:<nnn> (one logical CPU, zero-padded), " +
120 "package:<id> or sensor:<chip>/<label> (a thermometer)",
118121 at: "when the point was read (15 s) or its window began (5 min), UTC",
119122 step: "seconds the point covers: 15, or 300 for a 5-min roll-up",
120 cpu_cores: "cores in use, mean over the step (CPU seconds per second)",
123 cpu_cores: "cores in use, mean over the step (CPU seconds per second); at most 1 for a cpu:<nnn>",
121124 cpu_cores_max: "cores, the highest 15-s point in the step",
122125 memory_bytes: "bytes resident, mean over the step",
123126 memory_bytes_max: "bytes, the highest 15-s point in the step",
124127 load: "node only: the 1-min load average, runnable tasks",
128 celsius: "°C, mean over the step: a cpu:<nnn>'s core, a package or a sensor; null for the rest",
129 celsius_max: "°C, the highest 15-s point in the step",
125130 },
126131 },
127132 };
src/boards/thermal.ts added +139
@@ -0,0 +1,139 @@
1 /**
2 * What the machine's thermometers say, for the sampler (`boards/sampler.ts`):
3 *
4 * - Every `temp<N>_input` under `/sys/class/hwmon/hwmon*` is millidegrees
5 * Celsius (Documentation/hwmon/sysfs-interface.rst), so °C = value / 1000.
6 * - Intel's `coretemp` chip labels one input `Package id <P>` and the rest
7 * `Core <K>`, K being the physical core's `core_id`. A logical CPU n is
8 * on core (`physical_package_id`, `core_id`) from
9 * `/sys/devices/system/cpu/cpu<n>/topology/`, so its temperature is that
10 * core's: two hyper-threads of one core read the same thermometer.
11 * - A coretemp chip with no `Package id` label is not guessed onto a
12 * package; it and every other chip (k10temp, acpitz, nvme…) is a sensor
13 * of its own, `sensor:<chip>/<label>`.
14 *
15 * Nothing here is typed by a person, but chip names and labels are still
16 * cut to a short, plain alphabet before they become a target.
17 */
18 import { readdirSync, readFileSync } from "node:fs";
19 import { join } from "node:path";
20
21 /** One thermometer: the chip it is on (its hwmon directory and name), its label, °C. */
22 export type Reading = { dir: string; chip: string; label: string; celsius: number };
23
24 /** Per logical CPU, °C; and every reading no CPU took, as its own target. */
25 export type Temperatures = {
26 perCpu: ReadonlyMap<number, number>;
27 sensors: readonly { target: string; celsius: number }[];
28 };
29
30 /** At most this many hwmon chips, and inputs per chip, are read in one tick. */
31 const MAX_CHIPS = 64;
32 const MAX_INPUTS = 128;
33
34 function read(path: string): string | null {
35 try {
36 return readFileSync(path, "utf8");
37 } catch {
38 return null;
39 }
40 }
41
42 function list(path: string): string[] {
43 try {
44 return readdirSync(path);
45 } catch {
46 return [];
47 }
48 }
49
50 /**
51 * °C from a `temp<N>_input`: an integer of millidegrees. A value outside
52 * −100…250 °C is a sensor saying it is broken (−128000, 255000), not a
53 * temperature, and is dropped.
54 */
55 export function parseMillidegrees(text: string | null): number | null {
56 const s = text?.trim();
57 if (s === undefined || !/^-?\d+$/.test(s)) return null;
58 const c = Number(s) / 1000;
59 return c > -100 && c < 250 ? c : null;
60 }
61
62 /** A name or label as a target part: printable ASCII from a short set, at most 48 characters. */
63 export const clean = (s: string) => s.trim().replace(/[^A-Za-z0-9 ._+-]/g, "_").slice(0, 48) || "_";
64
65 const byNumber = (prefix: string) => (a: string, b: string) =>
66 Number(a.slice(prefix.length)) - Number(b.slice(prefix.length));
67
68 /** Every temperature under `<sys>/class/hwmon`, chip by chip in hwmon order, input by input. */
69 export function readHwmon(sys: string): Reading[] {
70 const root = join(sys, "class", "hwmon");
71 const out: Reading[] = [];
72 const dirs = list(root).filter((d) => /^hwmon\d+$/.test(d)).sort(byNumber("hwmon")).slice(0, MAX_CHIPS);
73 for (const dir of dirs) {
74 const at = join(root, dir);
75 const chip = read(join(at, "name"))?.trim() || dir;
76 const inputs = list(at)
77 .map((f) => /^temp(\d+)_input$/.exec(f)?.[1])
78 .filter((n): n is string => n !== undefined)
79 .sort((a, b) => Number(a) - Number(b))
80 .slice(0, MAX_INPUTS);
81 for (const n of inputs) {
82 const celsius = parseMillidegrees(read(join(at, `temp${n}_input`)));
83 if (celsius === null) continue;
84 const label = read(join(at, `temp${n}_label`))?.trim() || `temp${n}`;
85 out.push({ dir, chip, label, celsius });
86 }
87 }
88 return out;
89 }
90
91 /** Where each logical CPU sits: (package, core) from its topology, or absent when unreadable. */
92 export function readTopology(sys: string, cpus: readonly number[]): Map<number, { pkg: number; core: number }> {
93 const out = new Map<number, { pkg: number; core: number }>();
94 for (const n of cpus) {
95 const at = join(sys, "devices", "system", "cpu", `cpu${n}`, "topology");
96 const pkg = read(join(at, "physical_package_id"))?.trim();
97 const core = read(join(at, "core_id"))?.trim();
98 if (pkg !== undefined && core !== undefined && /^\d+$/.test(pkg) && /^\d+$/.test(core)) {
99 out.set(n, { pkg: Number(pkg), core: Number(core) });
100 }
101 }
102 return out;
103 }
104
105 /** Give each CPU its core's temperature; every other reading becomes a `package:<P>` or `sensor:` target. */
106 export function temperatures(
107 readings: readonly Reading[],
108 topology: ReadonlyMap<number, { pkg: number; core: number }>,
109 ): Temperatures {
110 const chips = new Map<string, Reading[]>();
111 for (const r of readings) chips.set(r.dir, [...(chips.get(r.dir) ?? []), r]);
112
113 const cores = new Map<string, number>();
114 const sensors: { target: string; celsius: number }[] = [];
115 const seen = new Map<string, number>();
116 const sensor = (target: string, celsius: number) => {
117 const n = (seen.get(target) ?? 0) + 1;
118 seen.set(target, n);
119 sensors.push({ target: n === 1 ? target : `${target} #${n}`, celsius });
120 };
121 for (const rs of chips.values()) {
122 const pkg = rs[0]!.chip === "coretemp"
123 ? rs.map((r) => /^Package id (\d+)$/.exec(r.label)?.[1]).find((p) => p !== undefined)
124 : undefined;
125 for (const r of rs) {
126 const core = pkg === undefined ? undefined : /^Core (\d+)$/.exec(r.label)?.[1];
127 if (pkg !== undefined && r.label === `Package id ${pkg}`) sensor(`package:${pkg}`, r.celsius);
128 else if (pkg !== undefined && core !== undefined) cores.set(`${pkg}/${Number(core)}`, r.celsius);
129 else sensor(`sensor:${clean(r.chip)}/${clean(r.label)}`, r.celsius);
130 }
131 }
132
133 const perCpu = new Map<number, number>();
134 for (const [n, { pkg, core }] of topology) {
135 const c = cores.get(`${pkg}/${core}`);
136 if (c !== undefined) perCpu.set(n, c);
137 }
138 return { perCpu, sensors };
139 }
src/core/migrations/20261016140000-machine-temperature.ts added +8
@@ -0,0 +1,8 @@
1 // What the machine's thermometers said (#2081, boards/thermal.ts): °C for
2 // each logical CPU (`cpu:<n>`) and every other hwmon sensor, beside the
3 // CPU and memory columns of 0088 — the 15-s reading, or the 5-min mean
4 // and max. Telemetry, not state: a lost row is a gap in a chart.
5 export default `
6 ALTER TABLE machine_sample ADD COLUMN celsius REAL;
7 ALTER TABLE machine_sample ADD COLUMN celsius_max REAL;
8 `;
src/web/pages/boards.ts +22 −2
@@ -21,7 +21,7 @@ import { btn, card, empty } from "../ui.ts";
2121
2222 const BUCKET_MS = { minute: 60_000, hour: 3_600_000, day: 86_400_000 } as const;
2323 const AGGREGATES = ["count", "sum", "avg", "max"] as const;
24 const OPS = ["eq", "ne", "lt", "lte", "gt", "gte"] as const;
24 const OPS = ["eq", "ne", "lt", "lte", "gt", "gte", "starts"] as const;
2525 const MAX_PARAM = 256;
2626 /** The fields a panel is made from: its chart and its query. */
2727 export const PANEL_FIELDS = ["chart", "source", "aggregate", "column", "by", "series", "from", "to", "fcolumn", "fop", "fvalue"] as const;
@@ -128,6 +128,25 @@ function behind(p: Params, row: Row): string | null {
128128 });
129129 }
130130
131 /**
132 * Ready-made queries over the machine's cores and thermometers, from `now`:
133 * the last hour per CPU, and twelve hours of hourly heat — 12 × the CPUs
134 * stays under the 1000-row cap up to 83 CPUs.
135 */
136 export function machineLinks(now: number): { label: string; href: string }[] {
137 const ago = (h: number) => new Date(Math.floor(now / 60_000) * 60_000 - h * 3_600_000).toISOString();
138 const base = { ...readParams(new URLSearchParams()), source: "machine", by: "target", fcolumn: "target", fop: "starts", fvalue: "cpu:" };
139 return [
140 { label: "CPU per core, last hour", href: link({ ...base, chart: "bar", aggregate: "avg", column: "cpu_cores", from: ago(1) }) },
141 { label: "Temperature per core, last hour (max °C)", href: link({ ...base, chart: "bar", aggregate: "max", column: "celsius", from: ago(1) }) },
142 { label: "CPU per core, last 12 h", href: link({ ...base, chart: "heat", aggregate: "avg", column: "cpu_cores", by: "hour", series: "target", from: ago(12) }) },
143 {
144 label: "Every thermometer, last hour (max °C)",
145 href: link({ ...base, chart: "bar", aggregate: "max", column: "celsius", fcolumn: "celsius", fop: "gt", fvalue: "-273.15", from: ago(1) }),
146 },
147 ];
148 }
149
131150 const option = (value: string, label: string, selected: boolean) =>
132151 html`<option value="${value}" ${selected ? raw("selected") : null}>${label}</option>`;
133152
@@ -276,7 +295,8 @@ export async function boardsPage(ctx: Ctx): Promise<Response> {
276295 p.source !== "machine"
277296 ? null
278297 : card(
279 html`<p class="muted small">Sampled every 15 s: the forge's pod from its cgroup, each check and app cell from its process tree, the node from /proc.</p>
298 html`<p class="muted small">Sampled every 15 s: the forge's pod from its cgroup, each check and app cell from its process tree, the node and each of its CPUs (<code>cpu:&lt;nnn&gt;</code>) from /proc, and temperatures in °C from /sys/class/hwmon.</p>
299 <ul class="small">${machineLinks(Date.now()).map((l) => html`<li><a href="${l.href}">${l.label}</a></li>`)}</ul>
280300 ${missing.length === 0
281301 ? html`<p class="small">Every series is readable on this machine.</p>`
282302 : html`<ul class="small">${missing.map((g) => html`<li>Absent — ${g}</li>`)}</ul>`}`,
test/boards-page.test.ts +23
@@ -137,6 +137,29 @@ describe("/boards/data", () => {
137137 expect(body).toContain(">0.25</td>");
138138 expect(await (await get("/boards/data?source=runs", "admin")).text()).not.toContain("What this machine measures");
139139 });
140
141 test("the machine card links per-core CPU and temperature, and the link draws the cores alone", async () => {
142 const at = new Date(Date.now() - 60_000).toISOString();
143 const put = app.db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores, celsius) VALUES (?, 15, ?, ?, ?)");
144 put.run(at, "cpu:000", 0.75, 71);
145 put.run(at, "cpu:001", 0.5, 82);
146 put.run(at, "node", 1.25, null);
147 const body = await (await get("/boards/data?source=machine&aggregate=count&by=target", "admin")).text();
148 const href = (label: string) => {
149 const m = new RegExp(`<a href="([^"]+)">${label}</a>`).exec(body);
150 expect(m).not.toBeNull();
151 return m![1]!.replaceAll("&amp;", "&");
152 };
153 const cpu = await (await get(href("CPU per core, last hour"), "admin")).text();
154 expect(cpu).toContain(">cpu:000</a></td>");
155 expect(cpu).toContain(">0.75</td>");
156 expect(cpu).toContain(">cpu:001</a></td>");
157 // The filter is `target starts cpu:`: the node is not a core.
158 expect(cpu).not.toContain(">node</a></td>");
159 const temp = await (await get(href("Temperature per core, last hour \\(max °C\\)"), "admin")).text();
160 expect(temp).toContain(">71</td>");
161 expect(temp).toContain(">82</td>");
162 });
140163 });
141164
142165 describe("every chart type from one seeded query", () => {
test/boards-sampler.test.ts +31 −14
@@ -23,10 +23,10 @@ let paths: Paths;
2323 const T0 = Date.parse("2026-10-15T12:00:00.000Z");
2424
2525 /** Write one moment of the fake machine. */
26 function machine(m: { usage: number; memory: number; stat: string; avail: number; load: string }) {
26 function machine(m: { usage: number; memory: number; stat: string; avail: number; load: string; cpu0?: string }) {
2727 writeFileSync(join(paths.cgroup, "cpu.stat"), `usage_usec ${m.usage}\nuser_usec 1\nsystem_usec 2\n`);
2828 writeFileSync(join(paths.cgroup, "memory.current"), `${m.memory}\n`);
29 writeFileSync(join(paths.proc, "stat"), `${m.stat}\ncpu0 1 2 3 4 5 6 7 8 0 0\nintr 9\n`);
29 writeFileSync(join(paths.proc, "stat"), `${m.stat}\ncpu0 ${m.cpu0 ?? "1 2 3 4 5 6 7 8 0 0"}\nintr 9\n`);
3030 writeFileSync(join(paths.proc, "meminfo"), `MemTotal: 8000000 kB\nMemFree: 1 kB\nMemAvailable: ${m.avail} kB\n`);
3131 writeFileSync(join(paths.proc, "loadavg"), `${m.load} 0.80 0.40 2/300 1234\n`);
3232 }
@@ -35,9 +35,21 @@ const fresh = () => openDb(":memory:", migrations);
3535
3636 beforeAll(() => {
3737 root = mkdtempSync(join(tmpdir(), "forge-sampler-"));
38 paths = { cgroup: join(root, "sys/fs/cgroup"), proc: join(root, "proc") };
38 paths = { cgroup: join(root, "sys/fs/cgroup"), proc: join(root, "proc"), sys: join(root, "sys") };
3939 mkdirSync(paths.cgroup, { recursive: true });
4040 mkdirSync(paths.proc, { recursive: true });
41 // cpu0 is core 0 of package 0, and coretemp says that core is at 61 °C.
42 const topo = join(paths.sys, "devices/system/cpu/cpu0/topology");
43 mkdirSync(topo, { recursive: true });
44 writeFileSync(join(topo, "physical_package_id"), "0\n");
45 writeFileSync(join(topo, "core_id"), "0\n");
46 const chip = join(paths.sys, "class/hwmon/hwmon0");
47 mkdirSync(chip, { recursive: true });
48 writeFileSync(join(chip, "name"), "coretemp\n");
49 writeFileSync(join(chip, "temp1_label"), "Package id 0\n");
50 writeFileSync(join(chip, "temp1_input"), "64000\n");
51 writeFileSync(join(chip, "temp2_label"), "Core 0\n");
52 writeFileSync(join(chip, "temp2_input"), "61000\n");
4153 });
4254
4355 afterAll(() => rmSync(root, { recursive: true, force: true }));
@@ -88,12 +100,14 @@ describe("rates from a fake cgroup and /proc", () => {
88100 });
89101
90102 test("no cgroup and no /proc: those series are absent and named, never zero", () => {
91 const none: Paths = { cgroup: join(root, "nowhere/cgroup"), proc: join(root, "nowhere/proc") };
103 const none: Paths = { cgroup: join(root, "nowhere/cgroup"), proc: join(root, "nowhere/proc"), sys: join(root, "nowhere/sys") };
92104 const c = readCounters(none, [], T0);
93105 expect(pointsOf(c, { ...c, at: T0 + 15_000 })).toEqual([]);
94106 const missing = gaps(none);
95 expect(missing).toHaveLength(5);
107 expect(missing).toHaveLength(7);
96108 expect(missing[0]).toBe(`forge cpu_cores: ${join(none.cgroup, "cpu.stat")} is not readable here`);
109 expect(missing[5]).toBe(`cpu:<n> cpu_cores: ${join(none.proc, "stat")} has no per-CPU lines here`);
110 expect(missing[6]).toBe(`celsius: ${join(none.sys, "class/hwmon")} has no readable temperature here`);
97111 expect(gaps(paths)).toEqual([]);
98112 });
99113 });
@@ -109,8 +123,8 @@ describe("the 5-min roll-up", () => {
109123 pts.push({ at: T0 + 315_000, target: "forge", cpuCores: null, memoryBytes: 42, load: null });
110124 expect(rollUp(pts)).toEqual([
111125 // (0 + 1 + … + 19) / 4 / 20 = 190 / 80 = 2.375; max 19 / 4 = 4.75
112 { at: T0, target: "forge", cpuCores: 2.375, cpuCoresMax: 4.75, memoryBytes: 9500, memoryBytesMax: 19000, load: null },
113 { at: T0 + 300_000, target: "forge", cpuCores: 7, cpuCoresMax: 7, memoryBytes: 42, memoryBytesMax: 42, load: null },
126 { at: T0, target: "forge", cpuCores: 2.375, cpuCoresMax: 4.75, memoryBytes: 9500, memoryBytesMax: 19000, load: null, celsius: null, celsiusMax: null },
127 { at: T0 + 300_000, target: "forge", cpuCores: 7, cpuCoresMax: 7, memoryBytes: 42, memoryBytesMax: 42, load: null, celsius: null, celsiusMax: null },
114128 ]);
115129 });
116130
@@ -206,20 +220,23 @@ describe("startSampler", () => {
206220 let usec = 0;
207221 const s = startSampler(app, { paths, everyMs: 3_600_000, cells: () => [{ key: 9, label: "app:x/production", cpuUsec: (usec += 3_000_000), memoryBytes: 5 }] });
208222 try {
209 machine({ usage: 0, memory: 7, stat: "cpu 0 0 0 0 0 0 0 0", avail: 7_000_000, load: "0.10" });
223 machine({ usage: 0, memory: 7, stat: "cpu 0 0 0 0 0 0 0 0", avail: 7_000_000, load: "0.10", cpu0: "0 0 0 0 0 0 0 0" });
210224 s.tick(T0);
211 machine({ usage: 15_000_000, memory: 7, stat: "cpu 1500 0 0 0 0 0 0 0", avail: 7_000_000, load: "0.10" });
225 machine({ usage: 15_000_000, memory: 7, stat: "cpu 1500 0 0 0 0 0 0 0", avail: 7_000_000, load: "0.10", cpu0: "500 0 250 0 0 0 0 0" });
212226 s.tick(T0 + 15_000);
213227 } finally {
214228 s.stop();
215229 }
216 const rows = db.query("SELECT target, cpu_cores FROM machine_sample WHERE at = ? ORDER BY target")
230 const rows = db.query("SELECT target, cpu_cores, celsius FROM machine_sample WHERE at = ? ORDER BY target")
217231 .all(new Date(T0 + 15_000).toISOString());
218 // forge: 15 s of CPU in 15 s = 1 core; node: 1500 ticks / 100 / 15 = 1 core; cell: 3 s / 15 s = 0.2
232 // forge: 15 s of CPU in 15 s = 1 core; node: 1500 ticks / 100 / 15 = 1 core; cell: 3 s / 15 s = 0.2;
233 // cpu0: (500 + 250) ticks / 100 / 15 s = 0.5 core, on core 0 at 61 °C; package 0 at 64 °C
219234 expect(rows).toEqual([
220 { target: "app:x/production", cpu_cores: 0.2 },
221 { target: "forge", cpu_cores: 1 },
222 { target: "node", cpu_cores: 1 },
235 { target: "app:x/production", cpu_cores: 0.2, celsius: null },
236 { target: "cpu:000", cpu_cores: 0.5, celsius: 61 },
237 { target: "forge", cpu_cores: 1, celsius: null },
238 { target: "node", cpu_cores: 1, celsius: null },
239 { target: "package:0", cpu_cores: null, celsius: 64 },
223240 ]);
224241 expect(warnings).toEqual([]);
225242 });
test/boards-thermal.test.ts added +240
@@ -0,0 +1,240 @@
1 /**
2 * Per-core CPU and temperature on the machine board (#2081): a fake
3 * /sys with two coretemp packages gives each logical CPU its core's °C
4 * (hyper-threads of one core share it), every other thermometer is a
5 * sensor of its own, a broken reading is dropped, per-CPU rates match a
6 * hand calculation, °C rolls up as a mean and a max, and a board selects
7 * the cores with `target starts cpu:`.
8 */
9 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
10 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { runQuery } from "../src/boards/query.ts";
14 import { record, rollUp, type Point } from "../src/boards/ring.ts";
15 import { cpuTarget, gaps, parseProcStatCpus, pointsOf, readCounters, type Paths } from "../src/boards/sampler.ts";
16 import { clean, parseMillidegrees, readHwmon, readTopology, temperatures } from "../src/boards/thermal.ts";
17 import { migrations } from "../src/core/schema.ts";
18 import { openDb } from "../src/kernel/db.ts";
19
20 let root: string;
21 const T0 = Date.parse("2026-10-15T12:00:00.000Z");
22
23 function write(path: string, text: string) {
24 mkdirSync(join(path, ".."), { recursive: true });
25 writeFileSync(path, text);
26 }
27
28 /** A machine at `at`: cgroup, /proc and /sys under one directory. */
29 function fakeMachine(name: string): Paths {
30 const base = join(root, name);
31 const paths = { cgroup: join(base, "cgroup"), proc: join(base, "proc"), sys: join(base, "sys") };
32 mkdirSync(paths.cgroup, { recursive: true });
33 mkdirSync(paths.proc, { recursive: true });
34 return paths;
35 }
36
37 function chip(paths: Paths, hwmon: string, name: string, temps: Record<number, [label: string | null, milli: string]>) {
38 const at = join(paths.sys, "class/hwmon", hwmon);
39 write(join(at, "name"), `${name}\n`);
40 for (const [n, [label, milli]] of Object.entries(temps)) {
41 if (label !== null) write(join(at, `temp${n}_label`), `${label}\n`);
42 write(join(at, `temp${n}_input`), `${milli}\n`);
43 }
44 }
45
46 function cpu(paths: Paths, n: number, pkg: number, core: number) {
47 const at = join(paths.sys, "devices/system/cpu", `cpu${n}`, "topology");
48 write(join(at, "physical_package_id"), `${pkg}\n`);
49 write(join(at, "core_id"), `${core}\n`);
50 }
51
52 let two: Paths;
53
54 beforeAll(() => {
55 root = mkdtempSync(join(tmpdir(), "forge-thermal-"));
56 // Two packages of two cores, each core two hyper-threads: cpu0/cpu2 on (0,0), cpu1/cpu3 on (0,1),
57 // cpu4/cpu6 on (1,0), cpu5/cpu7 on (1,4) — core ids need not be dense.
58 two = fakeMachine("two");
59 const place: [number, number, number][] = [[0, 0, 0], [1, 0, 1], [2, 0, 0], [3, 0, 1], [4, 1, 0], [5, 1, 4], [6, 1, 0], [7, 1, 4]];
60 for (const [n, p, c] of place) cpu(two, n, p, c);
61 chip(two, "hwmon2", "coretemp", { 1: ["Package id 1", "80000"], 2: ["Core 0", "79000"], 6: ["Core 4", "82000"] });
62 chip(two, "hwmon1", "coretemp", { 1: ["Package id 0", "66000"], 2: ["Core 0", "75000"], 3: ["Core 1", "77500"] });
63 chip(two, "hwmon10", "nvme", { 1: ["Composite", "41850"], 2: [null, "-128000"] });
64 chip(two, "hwmon0", "acpitz", { 1: [null, "27800"], 2: [null, "27800"] });
65 write(join(two.sys, "class/hwmon/hwmon3/name"), "no thermometer here\n");
66 write(
67 join(two.proc, "stat"),
68 "cpu 800 0 0 0 0 0 0 0\n" + [0, 1, 2, 3, 4, 5, 6, 7].map((n) => `cpu${n} 100 0 0 900 0 0 0 0`).join("\n") + "\n",
69 );
70 });
71
72 afterAll(() => rmSync(root, { recursive: true, force: true }));
73
74 describe("thermometers", () => {
75 test("millidegrees are °C × 1000, and a reading no sensor could make is dropped", () => {
76 expect(parseMillidegrees("75000\n")).toBe(75);
77 expect(parseMillidegrees("41850")).toBe(41.85);
78 expect(parseMillidegrees("-5000")).toBe(-5);
79 // −128 °C and 255 °C are what a broken or absent sensor reports.
80 expect(parseMillidegrees("-128000")).toBeNull();
81 expect(parseMillidegrees("255000")).toBeNull();
82 expect(parseMillidegrees("hot")).toBeNull();
83 expect(parseMillidegrees(null)).toBeNull();
84 });
85
86 test("hwmon is read chip by chip in number order, an input without a label is temp<N>", () => {
87 const rs = readHwmon(two.sys);
88 expect(rs.map((r) => `${r.dir} ${r.chip} ${r.label} ${r.celsius}`)).toEqual([
89 "hwmon0 acpitz temp1 27.8",
90 "hwmon0 acpitz temp2 27.8",
91 "hwmon1 coretemp Package id 0 66",
92 "hwmon1 coretemp Core 0 75",
93 "hwmon1 coretemp Core 1 77.5",
94 "hwmon2 coretemp Package id 1 80",
95 "hwmon2 coretemp Core 0 79",
96 "hwmon2 coretemp Core 4 82",
97 // nvme temp2 read −128 °C: dropped, not a point
98 "hwmon10 nvme Composite 41.85",
99 ]);
100 });
101
102 test("each CPU reads its own package's core; hyper-threads of one core share it", () => {
103 const t = temperatures(readHwmon(two.sys), readTopology(two.sys, [0, 1, 2, 3, 4, 5, 6, 7, 8]));
104 expect([...t.perCpu].sort((a, b) => a[0] - b[0])).toEqual([
105 [0, 75], [1, 77.5], [2, 75], [3, 77.5],
106 // core 0 of package 1 is not core 0 of package 0
107 [4, 79], [5, 82], [6, 79], [7, 82],
108 ]);
109 expect(t.sensors).toEqual([
110 { target: "sensor:acpitz/temp1", celsius: 27.8 },
111 { target: "sensor:acpitz/temp2", celsius: 27.8 },
112 { target: "package:0", celsius: 66 },
113 { target: "package:1", celsius: 80 },
114 { target: "sensor:nvme/Composite", celsius: 41.85 },
115 ]);
116 });
117
118 test("a coretemp chip with no package label is not guessed onto a package", () => {
119 const t = temperatures(
120 [{ dir: "hwmon0", chip: "coretemp", label: "Core 0", celsius: 70 }, { dir: "hwmon1", chip: "k10temp", label: "Tctl", celsius: 60 },
121 { dir: "hwmon2", chip: "k10temp", label: "Tctl", celsius: 61 }],
122 new Map([[0, { pkg: 0, core: 0 }]]),
123 );
124 expect(t.perCpu.size).toBe(0);
125 // Two chips of one name and label stay two series.
126 expect(t.sensors.map((s) => s.target)).toEqual(["sensor:coretemp/Core 0", "sensor:k10temp/Tctl", "sensor:k10temp/Tctl #2"]);
127 });
128
129 test("a chip name or label becomes a short, plain target part", () => {
130 expect(clean("Package id 0")).toBe("Package id 0");
131 expect(clean("<b>x</b>\u0000\n/é")).toBe("_b_x__b_____");
132 expect(clean(" ")).toBe("_");
133 expect(clean("x".repeat(100))).toHaveLength(48);
134 });
135 });
136
137 describe("per-core CPU", () => {
138 test("each cpu<n> line is its busy time; the all-CPU line is not one of them", () => {
139 expect(parseProcStatCpus("cpu 9 9 9 9\ncpu10 200 0 100 5 0 0 0 0\ncpu2 1 2 3 4 5 6 7\nintr 1\n")).toEqual([
140 // (1 + 2 + 3 + 6 + 7) ticks; (200 + 100) ticks; × 10⁶ / 100 Hz µs
141 { id: 2, busyUsec: 190_000 },
142 { id: 10, busyUsec: 3_000_000 },
143 ]);
144 expect(parseProcStatCpus(null)).toEqual([]);
145 expect(cpuTarget(7)).toBe("cpu:007");
146 expect(cpuTarget(1234)).toBe("cpu:1234");
147 });
148
149 test("two readings 15 s apart give each CPU its share of a core and its core's °C", () => {
150 const a = readCounters(two, [], T0);
151 write(
152 join(two.proc, "stat"),
153 "cpu 0 0 0 0\n" + [0, 1, 2, 3, 4, 5, 6, 7].map((n) => `cpu${n} ${100 + 150 * n} 0 ${n === 7 ? 600 : 0} 900 0 0 0 0`).join("\n") + "\n",
154 );
155 const points = pointsOf(a, readCounters(two, [], T0 + 15_000));
156 const by = Object.fromEntries(points.map((p) => [p.target, p]));
157 // cpu0: 0 ticks more → 0 cores; cpu3: 450 ticks / 100 Hz / 15 s = 0.3; cpu7: (1050 + 600) / 100 / 15 = 1.1
158 expect(by["cpu:000"]).toEqual({ at: T0 + 15_000, target: "cpu:000", cpuCores: 0, memoryBytes: null, load: null, celsius: 75 });
159 expect(by["cpu:003"]!.cpuCores).toBeCloseTo(0.3, 12);
160 expect(by["cpu:003"]!.celsius).toBe(77.5);
161 expect(by["cpu:007"]!.cpuCores).toBeCloseTo(1.1, 12);
162 expect(by["package:1"]).toEqual({ at: T0 + 15_000, target: "package:1", cpuCores: null, memoryBytes: null, load: null, celsius: 80 });
163 // No meminfo and a node counter that went back: no node point. Cores in order, then thermometers.
164 expect(points.map((p) => p.target)).toEqual([
165 "cpu:000", "cpu:001", "cpu:002", "cpu:003", "cpu:004", "cpu:005", "cpu:006", "cpu:007",
166 "sensor:acpitz/temp1", "sensor:acpitz/temp2", "package:0", "package:1", "sensor:nvme/Composite",
167 ]);
168 });
169
170 test("a machine with thermometers but none per core says so; one with every series says nothing", () => {
171 const amd = fakeMachine("amd");
172 write(join(amd.proc, "stat"), "cpu 1 0 0 0\ncpu0 1 0 0 0\n");
173 cpu(amd, 0, 0, 0);
174 chip(amd, "hwmon0", "k10temp", { 1: ["Tctl", "60000"] });
175 expect(gaps(amd)).toContain(`cpu:<n> celsius: no coretemp chip under ${join(amd.sys, "class/hwmon")} maps a temperature to each core here`);
176 expect(gaps(amd).some((g) => g.startsWith("celsius:"))).toBe(false);
177 expect(gaps(two).filter((g) => g.includes("celsius") || g.startsWith("cpu:"))).toEqual([]);
178 });
179 });
180
181 describe("°C in the ring", () => {
182 test("the 5-min roll-up keeps the mean and the max, and a target with no thermometer stays null", () => {
183 const pts: Point[] = [
184 { at: T0, target: "cpu:000", cpuCores: 0.5, memoryBytes: null, load: null, celsius: 70 },
185 { at: T0 + 15_000, target: "cpu:000", cpuCores: 1, memoryBytes: null, load: null, celsius: 76 },
186 { at: T0 + 30_000, target: "cpu:000", cpuCores: 0, memoryBytes: null, load: null },
187 { at: T0, target: "node", cpuCores: 2, memoryBytes: 1, load: 1 },
188 ];
189 expect(rollUp(pts).map((r) => [r.target, r.celsius, r.celsiusMax])).toEqual([
190 // (70 + 76) / 2 = 73; the point without one is skipped, not a 0
191 ["cpu:000", 73, 76],
192 ["node", null, null],
193 ]);
194 });
195
196 test("record writes °C at 15 s and rolls it up once the window closes", () => {
197 const db = openDb(":memory:", migrations);
198 for (let i = 0; i < 20; i++) {
199 record(db, [{ at: T0 + i * 15_000, target: "cpu:000", cpuCores: 0.5, memoryBytes: null, load: null, celsius: 60 + i }], T0 + i * 15_000);
200 }
201 record(db, [], T0 + 300_000);
202 expect(db.query("SELECT celsius, celsius_max FROM machine_sample WHERE step = 15 AND at = ?").get(new Date(T0).toISOString()))
203 .toEqual({ celsius: 60, celsius_max: 60 });
204 // (60 + … + 79) / 20 = 69.5; max 79
205 expect(db.query("SELECT target, celsius, celsius_max FROM machine_sample WHERE step = 300").all())
206 .toEqual([{ target: "cpu:000", celsius: 69.5, celsius_max: 79 }]);
207 });
208 });
209
210 describe("a board picks the cores", () => {
211 const db = openDb(":memory:", migrations);
212 const put = db.prepare("INSERT INTO machine_sample (at, step, target, cpu_cores, celsius) VALUES (?, 15, ?, ?, ?)");
213 put.run("2026-10-15T12:00:00.000Z", "cpu:000", 0.25, 70);
214 put.run("2026-10-15T12:00:00.000Z", "cpu:001", 0.75, 80);
215 put.run("2026-10-15T12:00:00.000Z", "node", 1, null);
216 put.run("2026-10-15T12:00:00.000Z", "check:cpu:x", 3, null);
217 put.run("2026-10-15T12:00:00.000Z", "c%u:9", 9, null);
218 const all = { repoIds: "all" } as const;
219
220 test("starts is a prefix of the column, and the per-core max °C reads back", () => {
221 expect(runQuery(db, {
222 source: "machine", aggregate: "max", column: "celsius", groupBy: "target",
223 filter: [{ column: "target", op: "starts", value: "cpu:" }],
224 }, all)).toEqual({ ok: true, rows: [{ group: "cpu:000", value: 70 }, { group: "cpu:001", value: 80 }] });
225 });
226
227 test("a % or _ in the prefix is a character, not a wildcard", () => {
228 const rows = (value: string) => runQuery(db, {
229 source: "machine", aggregate: "count", groupBy: "target", filter: [{ column: "target", op: "starts", value }],
230 }, all);
231 expect(rows("c%u")).toEqual({ ok: true, rows: [{ group: "c%u:9", value: 1 }] });
232 expect(rows("c_u")).toEqual({ ok: true, rows: [] });
233 });
234
235 test("starts on a number column is refused by name", () => {
236 expect(runQuery(db, {
237 source: "machine", aggregate: "count", filter: [{ column: "cpu_cores", op: "starts", value: 1 }],
238 }, all)).toEqual({ ok: false, error: "column cpu_cores is numeric: starts needs text" });
239 });
240 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs