Public: anyone can read this; the crew can change it.
/api/v1/health and /system report the cgroup's memory and CPU limit, not the host's #12539
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12518-api-v1-health-and-system-report-the-cgro → main 11 commits fork 2 threads · 2 unresolved
10 files
changed +231
−24
docs/features/system.md added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | # The machine's size on /system and in health (#12518) | |
| 2 | ||
| 3 | A forge in a pod may use only what its cgroup allows, and that can be much | |
| 4 | less than the host it runs on. `/system`, the terminal's System screen and | |
| 5 | `GET /api/v1/health` report what this forge may use. They do not report the | |
| 6 | host's whole size. | |
| 7 | ||
| 8 | ## Where the limits come from | |
| 9 | ||
| 10 | Each value is read when it is asked for: | |
| 11 | ||
| 12 | - **CPUs**: cgroup v2 `/sys/fs/cgroup/cpu.max`, `quota / period`, rounded | |
| 13 | down. The result is at least 1 and at most the host's count. A `max` quota, | |
| 14 | a missing file or a malformed one means the host's count | |
| 15 | (`availableParallelism()`). | |
| 16 | - **Memory**: cgroup v2 `/sys/fs/cgroup/memory.max`. If that file is missing, | |
| 17 | cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes` is read instead. A | |
| 18 | value counts only when it is a whole number below the host's memory | |
| 19 | (`totalmem()`). Otherwise the host's memory is used: `max`, v1's huge | |
| 20 | "unset" number, or a missing or malformed file. | |
| 21 | ||
| 22 | **cgroup v1 caveat:** on cgroup v1, only the memory limit is read. The CPU | |
| 23 | limit (`cpu.cfs_quota_us` / `cpu.cfs_period_us`) is not, so a v1 pod reports | |
| 24 | the host's CPU count even when its quota is lower. | |
| 25 | ||
| 26 | ## What each door shows | |
| 27 | ||
| 28 | - **`GET /api/v1/health`**, read by an admin: the full answer carries | |
| 29 | `machine: { cpus, memoryBytes, limitedBy }`. `memory.totalBytes` is the | |
| 30 | same memory limit. A signed-in reader who is not an admin, and a stranger, | |
| 31 | get the short answer, which has no `machine` key. | |
| 32 | - **`machine.limitedBy`** is `"cgroup"` when the cgroup holds the CPUs or the | |
| 33 | memory below the host's, and `"host"` when both are the host's own. | |
| 34 | - **`/system`**: the summary has a `limited by cgroup` pill next to the core | |
| 35 | count. The footprint's Machine fact ends with `, limited by cgroup`. | |
| 36 | - **The terminal's System screen**: the first line reads | |
| 37 | `CPU … over N cores · limited by cgroup · load …`. | |
| 38 | ||
| 39 | When no limit is set below the host, none of these say "cgroup", and | |
| 40 | `limitedBy` is `"host"`. |
src/core/capacity.ts +70 −10
| @@ -14,7 +14,7 @@ | ||
| 14 | 14 | import type { Database } from "bun:sqlite"; |
| 15 | 15 | import { readFileSync } from "node:fs"; |
| 16 | 16 | import { isIP } from "node:net"; |
| 17 | import { availableParallelism } from "node:os"; | |
| 17 | import { availableParallelism, totalmem } from "node:os"; | |
| 18 | 18 | import { PAYMENT_HOSTS } from "./egress_grants.ts"; |
| 19 | 19 | import { memoryLimitBytes } from "../system/memory_limit.ts"; |
| 20 | 20 | |
| @@ -253,15 +253,76 @@ export function provideCpusForTest(cpus: number | undefined): void { | ||
| 253 | 253 | */ |
| 254 | 254 | export function cpuLimit(file = "/sys/fs/cgroup/cpu.max"): number { |
| 255 | 255 | if (cpusForTest !== undefined) return cpusForTest; |
| 256 | const machine = availableParallelism(); | |
| 257 | try { | |
| 258 | const [quota, period] = readFileSync(file, "utf8").trim().split(/\s+/).map(Number); | |
| 259 | if (quota !== undefined && period !== undefined && quota > 0 && period > 0) | |
| 260 | return Math.max(1, Math.min(machine, Math.floor(quota / period))); | |
| 261 | } catch { | |
| 262 | // No cgroup v2 CPU controller here: the machine's count. | |
| 256 | return cpusOf(readOrNull(file), availableParallelism()); | |
| 257 | } | |
| 258 | ||
| 259 | /** The CPUs a cgroup v2 `cpu.max` allows, rounded down, never below one nor above the host's. */ | |
| 260 | export function cpusOf(cpuMax: string | null | undefined, hostCpus: number): number { | |
| 261 | const [quota, period] = (cpuMax ?? "").trim().split(/\s+/).map(Number); | |
| 262 | return quota !== undefined && period !== undefined && quota > 0 && period > 0 | |
| 263 | ? Math.max(1, Math.min(hostCpus, Math.floor(quota / period))) | |
| 264 | : hostCpus; | |
| 265 | } | |
| 266 | ||
| 267 | /** The cgroup files `cgroupLimits` reads, as their contents; null where a file is not there. */ | |
| 268 | export interface CgroupFiles { | |
| 269 | /** cgroup v2 `cpu.max`: "quota period" in µs, or "max period". */ | |
| 270 | readonly cpuMax?: string | null; | |
| 271 | /** cgroup v2 `memory.max`: bytes, or "max". */ | |
| 272 | readonly memoryMax?: string | null; | |
| 273 | /** cgroup v1 `memory.limit_in_bytes`: bytes; a huge number when unset. */ | |
| 274 | readonly memoryV1?: string | null; | |
| 275 | } | |
| 276 | ||
| 277 | /** What this forge may use, and whether a cgroup or the host's own size says so. */ | |
| 278 | export interface MachineLimits { | |
| 279 | readonly cpus: number; | |
| 280 | readonly memoryBytes: number; | |
| 281 | readonly cpuLimited: boolean; | |
| 282 | readonly memoryLimited: boolean; | |
| 283 | /** "cgroup" when either limit is the cgroup's, else "host". */ | |
| 284 | readonly limitedBy: "cgroup" | "host"; | |
| 285 | } | |
| 286 | ||
| 287 | /** | |
| 288 | * The limits a cgroup sets, from its files' contents (#12518): `cpu.max` rounded down, | |
| 289 | * never below one; `memory.max`, then v1's `memory.limit_in_bytes`. Unset ("max"), missing, | |
| 290 | * malformed or not below the host: the host's. Pure, so a test hands in fixture contents. | |
| 291 | */ | |
| 292 | export function cgroupLimits(files: CgroupFiles, host: { readonly cpus: number; readonly memoryBytes: number }): MachineLimits { | |
| 293 | const cpus = cpusOf(files.cpuMax, host.cpus); | |
| 294 | let memoryBytes = host.memoryBytes; | |
| 295 | for (const text of [files.memoryMax, files.memoryV1]) { | |
| 296 | const t = (text ?? "").trim(); | |
| 297 | const n = /^\d+$/.test(t) ? Number(t) : NaN; | |
| 298 | if (Number.isFinite(n) && n > 0 && n < host.memoryBytes) { | |
| 299 | memoryBytes = n; | |
| 300 | break; | |
| 301 | } | |
| 263 | 302 | } |
| 264 | return machine; | |
| 303 | const cpuLimited = cpus < host.cpus; | |
| 304 | const memoryLimited = memoryBytes !== host.memoryBytes; | |
| 305 | return { cpus, memoryBytes, cpuLimited, memoryLimited, limitedBy: cpuLimited || memoryLimited ? "cgroup" : "host" }; | |
| 306 | } | |
| 307 | ||
| 308 | const readOrNull = (file: string): string | null => { | |
| 309 | try { | |
| 310 | return readFileSync(file, "utf8"); | |
| 311 | } catch { | |
| 312 | return null; | |
| 313 | } | |
| 314 | }; | |
| 315 | ||
| 316 | /** | |
| 317 | * This pod's limits, read from its cgroup (v2, with v1's memory as the | |
| 318 | * fallback), else the host's. | |
| 319 | */ | |
| 320 | export function machineLimits(dir = "/sys/fs/cgroup"): MachineLimits { | |
| 321 | const limits = cgroupLimits( | |
| 322 | { cpuMax: readOrNull(`${dir}/cpu.max`), memoryMax: readOrNull(`${dir}/memory.max`), memoryV1: readOrNull(`${dir}/memory/memory.limit_in_bytes`) }, | |
| 323 | { cpus: availableParallelism(), memoryBytes: totalmem() }, | |
| 324 | ); | |
| 325 | return cpusForTest === undefined ? limits : { ...limits, cpus: cpusForTest }; | |
| 265 | 326 | } |
| 266 | 327 | |
| 267 | 328 | /** |
| @@ -320,7 +381,6 @@ export function floorMiBForMemory(bytes: number): number { | ||
| 320 | 381 | return Number.isFinite(n) && n > 0 ? n : 0; |
| 321 | 382 | } |
| 322 | 383 | |
| 323 | ||
| 324 | 384 | /** |
| 325 | 385 | * What this pod's cgroup holds in memory now: `memory.current` less the inactive file cache |
| 326 | 386 | * (the kernel gives that back under pressure, as `MemAvailable` counts it). Null when |
src/kernel/vitals.ts +11 −2
| @@ -41,7 +41,7 @@ import { running as runningVersion } from "./version.ts"; | ||
| 41 | 41 | import { rollbackNeed } from "../release/launch.ts"; |
| 42 | 42 | import { behindStanding } from "../release/behind.ts"; |
| 43 | 43 | import { newestDrills } from "../core/domain_drills.ts"; |
| 44 | import { capacityValue, podMemoryUsedBytes } from "../core/capacity.ts"; | |
| 44 | import { capacityValue, machineLimits, podMemoryUsedBytes, type MachineLimits } from "../core/capacity.ts"; | |
| 45 | 45 | import { memoryLimitBytes } from "../system/memory_limit.ts"; |
| 46 | 46 | import { limitValue } from "../settings/limits.ts"; |
| 47 | 47 | import { unwrittenFor, SILENT_AFTER_S } from "../http/access_log.ts"; |
| @@ -143,6 +143,11 @@ export interface Vitals { | ||
| 143 | 143 | /** The pod's memory limit where one is set (cgroup), else the host's memory. */ |
| 144 | 144 | totalBytes: number; |
| 145 | 145 | }; |
| 146 | /** | |
| 147 | * The CPUs and memory this forge may use, and whether its cgroup or the | |
| 148 | * host says so (#12518). | |
| 149 | */ | |
| 150 | readonly machine: { cpus: number; memoryBytes: number; limitedBy: "cgroup" | "host" }; | |
| 146 | 151 | readonly queue: Record<string, number>; |
| 147 | 152 | readonly lane: Lane; |
| 148 | 153 | readonly stuck: { |
| @@ -177,6 +182,8 @@ export interface VitalsInput { | ||
| 177 | 182 | readonly diskFreeMin?: number; |
| 178 | 183 | /** The pod's memory limit and usage; read from the cgroup unless a test gives them. */ |
| 179 | 184 | readonly podMemory?: PodMemory; |
| 185 | /** The pod's CPU and memory limits; `machineLimits()` unless a test gives them. */ | |
| 186 | readonly limits?: MachineLimits; | |
| 180 | 187 | /** |
| 181 | 188 | * The last walk of the database file, asked for only while a db-size need |
| 182 | 189 | * fires; `dbSize.last` unless a test gives one. |
| @@ -398,6 +405,7 @@ export function memoryReading( | ||
| 398 | 405 | |
| 399 | 406 | export function vitals(input: VitalsInput): Vitals { |
| 400 | 407 | const now = input.now ?? Date.now(); |
| 408 | const limits = input.limits ?? machineLimits(); | |
| 401 | 409 | const at = new Date(now).toISOString(); |
| 402 | 410 | const needs: string[] = []; |
| 403 | 411 | const degraded: Degraded[] = []; |
| @@ -730,7 +738,8 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 730 | 738 | todo, |
| 731 | 739 | degraded, |
| 732 | 740 | disk, |
| 733 | memory: memoryReading(input.podMemory, undefined, hostFreeBytes), | |
| 741 | memory: memoryReading(input.podMemory ?? { limitBytes: limits.memoryBytes, usedBytes: podMemoryUsedBytes() }, undefined, hostFreeBytes), | |
| 742 | machine: { cpus: limits.cpus, memoryBytes: limits.memoryBytes, limitedBy: limits.limitedBy }, | |
| 734 | 743 | queue, |
| 735 | 744 | lane, |
| 736 | 745 | stuck, |
src/system/footprint.ts +10 −8
| @@ -5,8 +5,8 @@ | ||
| 5 | 5 | * it shows. The release proof records a reading (`release/prove.ts`) and |
| 6 | 6 | * /system shows the last one. |
| 7 | 7 | */ |
| 8 | import { arch, availableParallelism } from "node:os"; | |
| 9 | import { memoryLimitBytes } from "./memory_limit.ts"; | |
| 8 | import { arch } from "node:os"; | |
| 9 | import { machineLimits, type MachineLimits } from "../core/capacity.ts"; | |
| 10 | 10 | import { findChromium } from "../looks/capture.ts"; |
| 11 | 11 | import { treeBytes } from "./largest_dirs.ts"; |
| 12 | 12 | import { largePageCache } from "./cache_share.ts"; |
| @@ -33,8 +33,11 @@ export interface Optional { | ||
| 33 | 33 | export interface Footprint { |
| 34 | 34 | readonly at: string; |
| 35 | 35 | readonly arch: string; |
| 36 | /** The CPUs and memory this forge may use: its cgroup's limits where set, else the host's. */ | |
| 36 | 37 | readonly cpus: number; |
| 37 | 38 | readonly machineBytes: number; |
| 39 | /** Who set `cpus` and `machineBytes`; absent on a reading kept before #12518. */ | |
| 40 | readonly limitedBy?: "cgroup" | "host"; | |
| 38 | 41 | /** Resident memory of this process, the one that serves pages and git. */ |
| 39 | 42 | readonly rssBytes: number; |
| 40 | 43 | readonly heapBytes: number; |
| @@ -90,16 +93,15 @@ export function optionals( | ||
| 90 | 93 | } |
| 91 | 94 | |
| 92 | 95 | /** Reads the footprint now; the data directory's walk is asynchronous, off the loop. */ |
| 93 | export async function readFootprint(dataDir: string, now: () => number = Date.now): Promise<Footprint> { | |
| 96 | export async function readFootprint(dataDir: string, now: () => number = Date.now, limits: MachineLimits = machineLimits()): Promise<Footprint> { | |
| 94 | 97 | const mem = process.memoryUsage(); |
| 95 | const cpus = availableParallelism(); | |
| 96 | const machineBytes = memoryLimitBytes(); | |
| 97 | 98 | const diskBytes = await treeBytes(dataDir).catch(() => null); |
| 98 | 99 | return { |
| 99 | 100 | at: new Date(now()).toISOString(), |
| 100 | 101 | arch: arch(), |
| 101 | cpus, | |
| 102 | machineBytes, | |
| 102 | cpus: limits.cpus, | |
| 103 | machineBytes: limits.memoryBytes, | |
| 104 | limitedBy: limits.limitedBy, | |
| 103 | 105 | rssBytes: mem.rss, |
| 104 | 106 | heapBytes: mem.heapUsed, |
| 105 | 107 | ageSeconds: Math.round(process.uptime()), |
| @@ -144,6 +146,6 @@ export function footprintFacts(f: Footprint): Fact[] { | ||
| 144 | 146 | { label: "Memory", value: `${mib(f.rssBytes)} after ${f.ageSeconds} s running; budget ${mib(BUDGET.restBytes)}` }, |
| 145 | 147 | { label: "First page", value: f.readyMs === null ? "not up yet" : `${seconds(f.readyMs)} after start; budget ${seconds(BUDGET.readyMs)}` }, |
| 146 | 148 | { label: "Data on disk", value: f.diskBytes === null ? "not read" : `${mib(f.diskBytes)}; an empty forge's budget ${mib(BUDGET.emptyDiskBytes)}` }, |
| 147 | { label: "Machine", value: `${f.arch}, ${f.cpus} CPU, ${mib(f.machineBytes)}` }, | |
| 149 | { label: "Machine", value: `${f.arch}, ${f.cpus} CPU, ${mib(f.machineBytes)}${f.limitedBy === "cgroup" ? ", limited by cgroup" : ""}` }, | |
| 148 | 150 | ]; |
| 149 | 151 | } |
src/system/view.ts +5
| @@ -50,6 +50,8 @@ export type SystemView = { | ||
| 50 | 50 | processes: number; |
| 51 | 51 | groups: Group[]; |
| 52 | 52 | duplicates: Duplicate[]; |
| 53 | /** Present when the pod's cgroup holds its cores or memory under the host's (#12518). */ | |
Carry this to src/tui/ system screen too (both doors), with a test. |
||
| 54 | limitedBy?: "cgroup"; | |
| 53 | 55 | }; |
| 54 | 56 | |
| 55 | 57 | /** Processes kept per group. */ |
| @@ -162,7 +164,10 @@ export function buildSystem(prev: Raw | undefined, next: Raw, roots: Roots, self | ||
| 162 | 164 | if (g.cpuCores !== null) g.cpuCores = Math.round(g.cpuCores * 100) / 100; |
| 163 | 165 | } |
| 164 | 166 | const mem = next.memory; |
| 167 | const limited = | |
| 168 | limits !== undefined && ((cores.length > 0 && limits.cpus < cores.length) || (mem !== undefined && mem !== null && limits.memoryBytes < mem.total)); | |
| 165 | 169 | return { |
| 170 | ...(limited ? { limitedBy: "cgroup" as const } : {}), | |
| 166 | 171 | at: new Date(next.at).toISOString(), |
| 167 | 172 | seconds, |
| 168 | 173 | cpu: { |
src/tui/screens/system.ts +1 −1
| @@ -116,7 +116,7 @@ export function systemLines(v: SystemView): string[] { | ||
| 116 | 116 | (c, i) => `${pad(`cpu${i}`, 6)} ${pad(c === null ? "—" : `${Math.round(c)}%`, 4, "right")} ${spark(v.cpu.history[i] ?? [])}`, |
| 117 | 117 | ); |
| 118 | 118 | return [ |
| 119 | `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.limitCores ?? v.cpu.cores.length} cores · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`, | |
| 119 | `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.limitCores ?? v.cpu.cores.length} cores${v.limitedBy === "cgroup" ? " · limited by cgroup" : ""} · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`, | |
| 120 | 120 | ...cores, |
| 121 | 121 | ...(v.cpu.cores.length > CORES ? [`… and ${v.cpu.cores.length - CORES} more cores`] : []), |
| 122 | 122 | `hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`} · memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"} · ${v.processes} processes`, |
src/web/pages/system.ts +1
| @@ -517,6 +517,7 @@ export function regions(v: SystemView): Record<string, string> { | ||
| 517 | 517 | "sys-dup": dup, |
| 518 | 518 | summary: html`${pill(`CPU ${pct(v.cpu.busyPct)}`, v.cpu.busyPct !== null && v.cpu.busyPct >= 90 ? "danger" : undefined)} |
| 519 | 519 | ${pill(`${String(v.cpu.limitCores ?? v.cpu.cores.length)} cores`)} |
| 520 | ${v.limitedBy === "cgroup" ? pill("limited by cgroup") : null} | |
| 520 | 521 | ${pill(`load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`)} |
| 521 | 522 | ${pill(`hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`}`)} |
| 522 | 523 | ${pill(`memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"}`)} |
test/footprint.test.ts +75 −1
| @@ -3,7 +3,10 @@ import { mkdtempSync, writeFileSync } from "node:fs"; | ||
| 3 | 3 | import { join } from "node:path"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { BUDGET, clearReadyMark, footprintFacts, footprintVerdict, markReady, optionals, readFootprint, type Footprint } from "../src/system/footprint.ts"; |
| 6 | import { footprintSection } from "../src/web/pages/system.ts"; | |
| 6 | import { footprintSection, regions } from "../src/web/pages/system.ts"; | |
| 7 | import { cgroupLimits, machineLimits } from "../src/core/capacity.ts"; | |
| 8 | import { buildSystem } from "../src/system/view.ts"; | |
| 9 | import type { Raw } from "../src/system/proc.ts"; | |
| 7 | 10 | import { proveRelease, lastProof } from "../src/release/prove.ts"; |
| 8 | 11 | |
| 9 | 12 | const MIB = 1024 * 1024; |
| @@ -93,3 +96,74 @@ test("the release proof records the footprint it is handed once the version answ | ||
| 93 | 96 | await proveRelease({ dataDir: dir, version: "1.0.0", url: "http://x.test", fetch: fetcher, giveUpMs: 1, sleep: async () => {}, footprint: async () => fp }); |
| 94 | 97 | expect(lastProof(dir)?.footprint?.rssBytes).toBe(123 * MIB); |
| 95 | 98 | }); |
| 99 | ||
| 100 | // ---- the cgroup's limits, not the host's (#12518) ---- | |
| 101 | ||
| 102 | const GIB = 1024 * MIB; | |
| 103 | const host = { cpus: 80, memoryBytes: 64 * GIB }; | |
| 104 | ||
| 105 | test("cgroup v2 limits are read from the files' contents: 512 MiB and one CPU on an 80-core host", () => { | |
| 106 | const l = cgroupLimits({ cpuMax: "100000 100000\n", memoryMax: "536870912\n" }, host); | |
| 107 | expect(l).toEqual({ cpus: 1, memoryBytes: 512 * MIB, cpuLimited: true, memoryLimited: true, limitedBy: "cgroup" }); | |
| 108 | expect(cgroupLimits({ cpuMax: "50000 100000" }, host).cpus).toBe(1); // half a core is still one | |
| 109 | expect(cgroupLimits({ cpuMax: "250000 100000" }, host).cpus).toBe(2); | |
| 110 | }); | |
| 111 | ||
| 112 | test("unset, missing or malformed cgroup files fall back to the host", () => { | |
| 113 | for (const files of [{}, { cpuMax: "max 100000", memoryMax: "max" }, { cpuMax: null, memoryMax: null }, { cpuMax: "junk", memoryMax: "12abc" }]) { | |
| 114 | expect(cgroupLimits(files, host)).toEqual({ ...host, cpuLimited: false, memoryLimited: false, limitedBy: "host" }); | |
| 115 | } | |
| 116 | // A limit at or over the host's own size is no limit. | |
| 117 | expect(cgroupLimits({ memoryMax: String(128 * GIB) }, host).limitedBy).toBe("host"); | |
| 118 | }); | |
| 119 | ||
| 120 | test("a cgroup v1 host keeps its memory limit; v1's unset sentinel is the host", () => { | |
| 121 | expect(cgroupLimits({ memoryMax: null, memoryV1: "1073741824" }, host).memoryBytes).toBe(GIB); | |
| 122 | expect(cgroupLimits({ memoryMax: null, memoryV1: "9223372036854771712" }, host).limitedBy).toBe("host"); | |
| 123 | }); | |
| 124 | ||
| 125 | test("machineLimits reads a cgroup directory, and a missing one is the host", () => { | |
| 126 | const dir = dataDir(); | |
| 127 | writeFileSync(join(dir, "cpu.max"), "200000 100000\n"); | |
| 128 | writeFileSync(join(dir, "memory.max"), "536870912\n"); | |
| 129 | const l = machineLimits(dir); | |
| 130 | expect(l.memoryBytes).toBe(512 * MIB); | |
| 131 | expect(l.limitedBy).toBe("cgroup"); | |
| 132 | expect(machineLimits(join(dir, "none")).limitedBy).toBe("host"); | |
| 133 | }); | |
| 134 | ||
| 135 | test("the footprint and its Machine fact carry the cgroup's limit and say so", async () => { | |
| 136 | const f = await readFootprint(dataDir(), Date.now, cgroupLimits({ cpuMax: "100000 100000", memoryMax: "536870912" }, host)); | |
| 137 | expect([f.cpus, f.machineBytes, f.limitedBy]).toEqual([1, 512 * MIB, "cgroup"]); | |
| 138 | expect(footprintFacts(f).find((x) => x.label === "Machine")!.value).toBe(`${f.arch}, 1 CPU, 512 MiB, limited by cgroup`); | |
| 139 | expect(footprintFacts(reading()).find((x) => x.label === "Machine")!.value).not.toContain("cgroup"); | |
| 140 | }); | |
| 141 | ||
| 142 | test("health's vitals report the cgroup's memory, never the host's, when a limit is given", async () => { | |
| 143 | const { createApp } = await import("../src/app.ts"); | |
| 144 | const { vitals } = await import("../src/kernel/vitals.ts"); | |
| 145 | const dir = dataDir(); | |
| 146 | const app = await createApp({ dataDir: dir }); | |
| 147 | try { | |
| 148 | const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, disk: () => ({ freeBytes: 95, totalBytes: 100 }), | |
| 149 | limits: cgroupLimits({ cpuMax: "100000 100000", memoryMax: "536870912" }, host) }); | |
| 150 | expect(v.machine).toEqual({ cpus: 1, memoryBytes: 512 * MIB, limitedBy: "cgroup" }); | |
| 151 | expect(v.memory.totalBytes).toBe(512 * MIB); | |
| 152 | expect(v.memory.freeBytes).toBeLessThanOrEqual(512 * MIB); | |
| 153 | } finally { | |
| 154 | app.close(); | |
| 155 | } | |
| 156 | }); | |
| 157 | ||
| 158 | test("/system's summary says 'limited by cgroup' only under a limit", () => { | |
| 159 | const raw: Raw = { | |
| 160 | at: 1_000, cores: Array.from({ length: 80 }, () => ({ busy: 0, total: 0 })), temps: [], load: null, | |
| 161 | memory: { total: 64 * GIB, available: 4 * GIB, swapTotal: 0, swapFree: 0 }, disks: [], net: [], io: [], procs: [], | |
| 162 | }; | |
| 163 | const limited = buildSystem(undefined, raw, [], 1, { cpus: 1, memoryBytes: 512 * MIB, usedBytes: 100 * MIB }); | |
| 164 | expect(limited.limitedBy).toBe("cgroup"); | |
| 165 | expect(String(regions(limited)["summary"])).toContain("limited by cgroup"); | |
| 166 | const free = buildSystem(undefined, raw, [], 1, { cpus: 80, memoryBytes: 64 * GIB }); | |
| 167 | expect(free.limitedBy).toBeUndefined(); | |
| 168 | expect(String(regions(free)["summary"])).not.toContain("cgroup"); | |
| 169 | }); |
test/tui-system.test.ts +16 −1
| @@ -16,8 +16,9 @@ import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts"; | ||
| 16 | 16 | import { setTelemetryGrant } from "../src/actions/telemetry.ts"; |
| 17 | 17 | import { hasGrant } from "../src/core/telemetry.ts"; |
| 18 | 18 | import { countRead, resetTelemetryReads } from "../src/system/telemetry_reads.ts"; |
| 19 | import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemScreen } from "../src/tui/screens/system.ts"; | |
| 19 | import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts"; | |
| 20 | 20 | import type { DbSize } from "../src/system/db_size.ts"; |
| 21 | import type { SystemView } from "../src/system/view.ts"; | |
| 21 | 22 | import { changeCapacity } from "../src/actions/capacity.ts"; |
| 22 | 23 | import { EGRESS_REPORTED } from "../src/system/egress_line.ts"; |
| 23 | 24 | import { keepFootprint } from "../src/system/footprint.ts"; |
| @@ -468,3 +469,17 @@ describe("tui system footprint (#12092)", () => { | ||
| 468 | 469 | tui.stop?.(); |
| 469 | 470 | }); |
| 470 | 471 | }); |
| 472 | ||
| 473 | describe("the machine line says when a cgroup holds the forge under the host (#12518)", () => { | |
| 474 | const view: SystemView = { | |
| 475 | at: "", seconds: 0, cpu: { busyPct: null, cores: [10, 20, 30, 40], limitCores: 2, history: [] }, | |
| 476 | temps: [], hottestC: null, load: null, memory: null, disks: [], net: [], io: [], | |
| 477 | processes: 0, groups: [], duplicates: [], | |
| 478 | }; | |
| 479 | test("limited: the first line says so, as the page's pill does", () => { | |
| 480 | expect(systemLines({ ...view, limitedBy: "cgroup" })[0]).toContain("over 2 cores · limited by cgroup"); | |
| 481 | }); | |
| 482 | test("the host's own size: no such words", () => { | |
| 483 | expect(systemLines(view).join("\n")).not.toContain("cgroup"); | |
| 484 | }); | |
| 485 | }); |
test/vitals.test.ts +2 −1
| @@ -95,7 +95,7 @@ test("a site admin with a read-only token gets the shape; others get the short o | ||
| 95 | 95 | [ |
| 96 | 96 | "accessLog", |
| 97 | 97 | "apps", "at", "broken", "checkedAt", "commit", "degraded", "disk", "draining", "errors", "flakes", |
| 98 | "incidents", "lane", "memory", "needs", "ok", "optional", "queue", "rollout", "schedules", "stale", | |
| 98 | "incidents", "lane", "machine", "memory", "needs", "ok", "optional", "queue", "rollout", "schedules", "stale", | |
| 99 | 99 | "stuck", |
| 100 | 100 | "todo", "version", |
| 101 | 101 | ].sort(), |
| @@ -111,6 +111,7 @@ test("a site admin with a read-only token gets the shape; others get the short o | ||
| 111 | 111 | const short = (await res2.json()) as Record<string, unknown>; |
| 112 | 112 | expect(short["disk"]).toBeUndefined(); |
| 113 | 113 | expect(short["memory"]).toBeUndefined(); |
| 114 | expect(short["machine"]).toBeUndefined(); | |
| 114 | 115 | expect(short["rollout"]).toBeDefined(); |
| 115 | 116 | // A stranger is told whether it is well, and no more (#2917). |
| 116 | 117 | const stranger = (await (await health()).json()) as Record<string, unknown>; |
new above
offline
Sentinel host memoryBytes: 0 just to reuse the parser. Read the cpu limit without a fake host memory.