Public: anyone can read this; the crew can change it.
/system and /api/v1/health read the pod's memory limit, not the host's #12353
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12347-system-and-api-v1-health-read-the-pod-s → main 5 commits fork 3 threads · 3 unresolved
8 files
changed +134
−19
src/core/capacity.ts +16
| @@ -322,6 +322,22 @@ export function memoryLimitBytes(): number { | ||
| 322 | 322 | return totalmem(); |
| 323 | 323 | } |
| 324 | 324 | |
| 325 | /** | |
| 326 | * What this pod's cgroup holds in memory now: `memory.current` less the inactive file cache | |
| 327 | * (the kernel gives that back under pressure, as `MemAvailable` counts it). Null when | |
| 328 | * there is no cgroup v2 memory controller here or the files do not read. | |
| 329 | */ | |
| 330 | export function podMemoryUsedBytes(dir = "/sys/fs/cgroup"): number | null { | |
| 331 | try { | |
| 332 | const current = Number(readFileSync(`${dir}/memory.current`, "utf8").trim()); | |
| 333 | if (!Number.isFinite(current) || current < 0) return null; | |
| 334 | const inactive = /^inactive_file (\d+)$/m.exec(readFileSync(`${dir}/memory.stat`, "utf8")); | |
| 335 | return Math.max(0, current - (inactive ? Number(inactive[1]) : 0)); | |
| 336 | } catch { | |
| 337 | return null; | |
| 338 | } | |
| 339 | } | |
| 340 | ||
| 325 | 341 | /** The value stored by an admin, if any. */ |
| 326 | 342 | export function storedCapacity( |
| 327 | 343 | db: Database, |
src/kernel/vitals.ts +29 −9
| @@ -14,7 +14,7 @@ import { fullSentence, fullSince } from "./full_disk.ts"; | ||
| 14 | 14 | import { readDisk, type DiskReading } from "./disk_reading.ts"; |
| 15 | 15 | import { DISK_FLOOR, roomy } from "../spawn/disk_floor.ts"; |
| 16 | 16 | import { dirname } from "node:path"; |
| 17 | import { freemem, totalmem } from "node:os"; | |
| 17 | import { freemem } from "node:os"; | |
| 18 | 18 | import { refusedInternal, type RefusedLedger } from "./internal_refused.ts"; |
| 19 | 19 | import { recentErrors, redactSecrets, type ErrorLedger, type ErrorLine } from "./log.ts"; |
| 20 | 20 | import { |
| @@ -41,7 +41,7 @@ import { running as runningVersion } from "./version.ts"; | ||
| 41 | 41 | import { rollbackNeed } from "../release/launch.ts"; |
| 42 | 42 | import { behindStanding } from "../release/behind.ts"; |
| 43 | 43 | import { newestDrills } from "../core/domain_drills.ts"; |
| 44 | import { capacityValue } from "../core/capacity.ts"; | |
| 44 | import { capacityValue, memoryLimitBytes, podMemoryUsedBytes } from "../core/capacity.ts"; | |
| 45 | 45 | import { unwrittenFor, SILENT_AFTER_S } from "../http/access_log.ts"; |
| 46 | 46 | import { DOMAINS } from "./domains.ts"; |
| 47 | 47 | import { domainDrillNeeds } from "./domain_drill.ts"; |
| @@ -108,8 +108,13 @@ export interface Vitals { | ||
| 108 | 108 | readonly disk: { freeBytes: number; totalBytes: number }; |
| 109 | 109 | readonly memory: { |
| 110 | 110 | rssBytes: number; |
| 111 | hostFreeBytes: number; | |
| 112 | hostTotalBytes: number; | |
| 111 | /** | |
| 112 | * Memory left under `totalBytes`: the pod's limit less its own usage where a cgroup | |
| 113 | * limit is set, else the host's free memory. | |
| 114 | */ | |
| 115 | freeBytes: number; | |
| 116 | /** The pod's memory limit where one is set (cgroup), else the host's memory. */ | |
| 117 | totalBytes: number; | |
| 113 | 118 | }; |
| 114 | 119 | readonly queue: Record<string, number>; |
| 115 | 120 | readonly lane: Lane; |
| @@ -143,6 +148,8 @@ export interface VitalsInput { | ||
| 143 | 148 | readonly disk?: (dir: string) => DiskReading; |
| 144 | 149 | /** The free-disk share below which the disk is a need; `DISK_FREE_MIN` unless a test says. */ |
| 145 | 150 | readonly diskFreeMin?: number; |
| 151 | /** The pod's memory limit and usage; read from the cgroup unless a test gives them. */ | |
| 152 | readonly podMemory?: PodMemory; | |
| 146 | 153 | /** |
| 147 | 154 | * The last walk of the database file, asked for only while a db-size need |
| 148 | 155 | * fires; `dbSize.last` unless a test gives one. |
| @@ -321,6 +328,23 @@ export function laneSentence( | ||
| 321 | 328 | return `${head}; the longest running is ${name(longest)} at ${minutes(longest)} min (limit ${BUDGETS.gate / 60_000})${more}.`; |
| 322 | 329 | } |
| 323 | 330 | |
| 331 | /** The pod's memory as `memoryReading` takes it; a test gives its own. */ | |
| 332 | export type PodMemory = { readonly limitBytes: number; readonly usedBytes: number | null }; | |
| 333 | ||
| 334 | /** | |
| 335 | * This process's memory beside the pod's own limit where one is set: a forge in 512 MiB | |
| 336 | * is not on a 64 GiB host (#12347). Free is the limit less what the pod's cgroup holds | |
| 337 | * (this process's resident memory when the cgroup cannot be read), never more than the host | |
| 338 | * has free. | |
| 339 | */ | |
| 340 | export function memoryReading( | |
| 341 | pod: PodMemory = { limitBytes: memoryLimitBytes(), usedBytes: podMemoryUsedBytes() }, | |
| 342 | rssBytes = process.memoryUsage().rss, | |
| 343 | ): Vitals["memory"] { | |
| 344 | const used = pod.usedBytes ?? rssBytes; | |
| 345 | return { rssBytes, freeBytes: Math.min(freemem(), Math.max(0, pod.limitBytes - used)), totalBytes: pod.limitBytes }; | |
| 346 | } | |
| 347 | ||
| 324 | 348 | export function vitals(input: VitalsInput): Vitals { |
| 325 | 349 | const now = input.now ?? Date.now(); |
| 326 | 350 | const at = new Date(now).toISOString(); |
| @@ -616,11 +640,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 616 | 640 | needs, |
| 617 | 641 | degraded, |
| 618 | 642 | disk, |
| 619 | memory: { | |
| 620 | rssBytes: process.memoryUsage().rss, | |
| 621 | hostFreeBytes: freemem(), | |
| 622 | hostTotalBytes: totalmem(), | |
| 623 | }, | |
| 643 | memory: memoryReading(input.podMemory), | |
| 624 | 644 | queue, |
| 625 | 645 | lane, |
| 626 | 646 | stuck, |
src/system/live.ts +5 −3
| @@ -8,7 +8,8 @@ | ||
| 8 | 8 | import type { App } from "../app.ts"; |
| 9 | 9 | import { cellRoots } from "../apps/contain.ts"; |
| 10 | 10 | import { HOST, readRaw, type Paths, type Raw } from "./proc.ts"; |
| 11 | import { buildSystem, forAgent, type Roots, type SystemView } from "./view.ts"; | |
| 11 | import { buildSystem, forAgent, type PodLimits, type Roots, type SystemView } from "./view.ts"; | |
| 12 | import { cpuLimit, memoryLimitBytes, podMemoryUsedBytes } from "../core/capacity.ts"; | |
| 12 | 13 | import { during } from "../perf/jobs.ts"; |
| 13 | 14 | |
| 14 | 15 | const FRESH_MS = 1_000; |
| @@ -38,10 +39,11 @@ const realSchedule: Schedule = (tick, ms) => { | ||
| 38 | 39 | return () => clearInterval(t); |
| 39 | 40 | }; |
| 40 | 41 | |
| 41 | export function createLive(options: { paths?: Paths; roots?: () => Roots; self?: number; pushMs?: number; schedule?: Schedule } = {}): Live { | |
| 42 | export function createLive(options: { paths?: Paths; roots?: () => Roots; self?: number; pushMs?: number; schedule?: Schedule; limits?: () => PodLimits } = {}): Live { | |
| 42 | 43 | const paths = options.paths ?? HOST; |
| 43 | 44 | const roots = options.roots ?? (() => cellRoots()); |
| 44 | 45 | const self = options.self ?? process.pid; |
| 46 | const limits = options.limits ?? (() => ({ cpus: cpuLimit(), memoryBytes: memoryLimitBytes(), usedBytes: podMemoryUsedBytes() })); | |
| 45 | 47 | let prev: Raw | undefined; |
| 46 | 48 | let last: Raw | undefined; |
| 47 | 49 | let view: SystemView | undefined; |
| @@ -79,7 +81,7 @@ export function createLive(options: { paths?: Paths; roots?: () => Roots; self?: | ||
| 79 | 81 | // A first reading has no rate: its rates are null until a second reading is taken. |
| 80 | 82 | prev = last; |
| 81 | 83 | last = raw; |
| 82 | const built = buildSystem(prev, last, roots(), self); | |
| 84 | const built = buildSystem(prev, last, roots(), self, limits()); | |
| 83 | 85 | history = built.cpu.cores.map((c, i) => [...(history[i] ?? []), c ?? 0].slice(-HISTORY)); |
| 84 | 86 | view = { ...built, cpu: { ...built.cpu, history } }; |
| 85 | 87 | return view; |
src/system/view.ts +24 −3
| @@ -32,6 +32,11 @@ export type SystemView = { | ||
| 32 | 32 | cpu: { |
| 33 | 33 | busyPct: number | null; |
| 34 | 34 | cores: (number | null)[]; |
| 35 | /** | |
| 36 | * The cores this pod may use (cgroup `cpu.max` where set), else the host's; | |
| 37 | * absent when not given. | |
| 38 | */ | |
| 39 | limitCores?: number; | |
| 35 | 40 | /** Each core's last readings, percent, oldest first (kept by `live.ts`; empty from `buildSystem`). */ |
| 36 | 41 | history: number[][]; |
| 37 | 42 | }; |
| @@ -97,7 +102,18 @@ function groupOf(procs: readonly Proc[], roots: Roots, self: number): Map<number | ||
| 97 | 102 | const rate = (a: number | undefined, b: number | undefined, s: number): number | null => |
| 98 | 103 | a === undefined || b === undefined || s <= 0 || b < a ? null : (b - a) / s; |
| 99 | 104 | |
| 100 | export function buildSystem(prev: Raw | undefined, next: Raw, roots: Roots, self: number): SystemView { | |
| 105 | /** The pod's own limits, which the host's numbers are held under (#12347). */ | |
| 106 | export type PodLimits = { | |
| 107 | cpus: number; | |
| 108 | memoryBytes: number; | |
| 109 | /** | |
| 110 | * What the pod's cgroup holds now (`podMemoryUsedBytes`); null when unreadable, then the | |
| 111 | * host's is shown. | |
| 112 | */ | |
| 113 | usedBytes?: number | null; | |
| 114 | }; | |
| 115 | ||
| 116 | export function buildSystem(prev: Raw | undefined, next: Raw, roots: Roots, self: number, limits?: PodLimits): SystemView { | |
| 101 | 117 | const seconds = prev === undefined ? 0 : Math.max(0, (next.at - prev.at) / 1000); |
| 102 | 118 | const cores = next.cores.map((c, i) => { |
| 103 | 119 | const p = prev?.cores[i]; |
| @@ -152,14 +168,19 @@ export function buildSystem(prev: Raw | undefined, next: Raw, roots: Roots, self | ||
| 152 | 168 | cpu: { |
| 153 | 169 | busyPct: known.length === 0 ? null : Math.round((known.reduce((a, b) => a + b, 0) / known.length) * 10) / 10, |
| 154 | 170 | cores, |
| 171 | ...(limits ? { limitCores: Math.min(limits.cpus, cores.length || limits.cpus) } : {}), | |
| 155 | 172 | history: [], |
| 156 | 173 | }, |
| 157 | 174 | temps: next.temps, |
| 158 | 175 | hottestC: next.temps.length === 0 ? null : Math.max(...next.temps.map((t) => t.celsius)), |
| 159 | 176 | load: next.load, |
| 160 | 177 | memory: mem && { |
| 161 | totalBytes: mem.total, | |
| 162 | usedBytes: mem.total - mem.available, | |
| 178 | totalBytes: limits ? Math.min(mem.total, limits.memoryBytes) : mem.total, | |
| 179 | // Under a pod limit, the pod's own usage: the host's used memory is not the pod's. | |
| 180 | usedBytes: | |
| 181 | limits && limits.memoryBytes < mem.total | |
| 182 | ? Math.min(limits.memoryBytes, limits.usedBytes ?? 0) | |
| 183 | : mem.total - mem.available, | |
| 163 | 184 | swapUsedBytes: mem.swapTotal - mem.swapFree, |
| 164 | 185 | swapTotalBytes: mem.swapTotal, |
| 165 | 186 | }, |
src/tui/screens/system.ts +1 −1
| @@ -107,7 +107,7 @@ export function systemLines(v: SystemView): string[] { | ||
| 107 | 107 | (c, i) => `${pad(`cpu${i}`, 6)} ${pad(c === null ? "—" : `${Math.round(c)}%`, 4, "right")} ${spark(v.cpu.history[i] ?? [])}`, |
| 108 | 108 | ); |
| 109 | 109 | return [ |
| 110 | `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.cores.length} cores · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`, | |
| 110 | `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.limitCores ?? v.cpu.cores.length} cores · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`, | |
| 111 | 111 | ...cores, |
| 112 | 112 | ...(v.cpu.cores.length > CORES ? [`… and ${v.cpu.cores.length - CORES} more cores`] : []), |
| 113 | 113 | `hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`} · memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"} · ${v.processes} processes`, |
src/web/pages/system.ts +2 −2
| @@ -446,12 +446,12 @@ export function regions(v: SystemView): Record<string, string> { | ||
| 446 | 446 | const out: Record<string, Html> = { |
| 447 | 447 | "sys-dup": dup, |
| 448 | 448 | summary: html`${pill(`CPU ${pct(v.cpu.busyPct)}`, v.cpu.busyPct !== null && v.cpu.busyPct >= 90 ? "danger" : undefined)} |
| 449 | ${pill(`${String(v.cpu.cores.length)} cores`)} | |
| 449 | ${pill(`${String(v.cpu.limitCores ?? v.cpu.cores.length)} cores`)} | |
| 450 | 450 | ${pill(`load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`)} |
| 451 | 451 | ${pill(`hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`}`)} |
| 452 | 452 | ${pill(`memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"}`)} |
| 453 | 453 | ${pill(`${String(v.processes)} processes`)}`, |
| 454 | cores: html`${v.cpu.cores.map((c, i) => meterRow(`cpu${String(i)}`, c, 100, c === null ? "—" : `${c.toFixed(0)}%`, spark(v.cpu.history[i] ?? [])))}`, | |
| 454 | cores: html`${v.cpu.limitCores !== undefined && v.cpu.limitCores < v.cpu.cores.length ? html`<li class="small">This pod may use ${String(v.cpu.limitCores)} of the host's ${String(v.cpu.cores.length)} cores; the rows below are the host's.</li>` : null}${v.cpu.cores.map((c, i) => meterRow(`cpu${String(i)}`, c, 100, c === null ? "—" : `${c.toFixed(0)}%`, spark(v.cpu.history[i] ?? [])))}`, | |
| 455 | 455 | mem: html`${v.memory |
| 456 | 456 | ? html`${meterRow("mem", v.memory.usedBytes, v.memory.totalBytes, `${bytes(v.memory.usedBytes)} / ${bytes(v.memory.totalBytes)}`)} |
| 457 | 457 | ${meterRow("swap", v.memory.swapUsedBytes, v.memory.swapTotalBytes, v.memory.swapTotalBytes === 0 ? "none" : `${bytes(v.memory.swapUsedBytes)} / ${bytes(v.memory.swapTotalBytes)}`)}` |
test/system-crew.test.ts +31
| @@ -1,6 +1,9 @@ | ||
| 1 | 1 | import { expect, test } from "bun:test"; |
| 2 | 2 | import type { Proc, Raw } from "../src/system/proc.ts"; |
| 3 | 3 | import { buildSystem, forAgent } from "../src/system/view.ts"; |
| 4 | import { createLive } from "../src/system/live.ts"; | |
| 5 | import { regions } from "../src/web/pages/system.ts"; | |
| 6 | import { systemLines } from "../src/tui/screens/system.ts"; | |
| 4 | 7 | |
| 5 | 8 | const proc = (pid: number, ppid: number): Proc => ({ pid, ppid, comm: "bun", ticks: 0, rss: 1 << 20, age: 5, tests: [] }); |
| 6 | 9 | const raw: Raw = { |
| @@ -18,3 +21,31 @@ test("a crew run's tree is a crew:<name> group, hidden by name from agents", () | ||
| 18 | 21 | expect(seen.label).toBe("crew:private-1"); |
| 19 | 22 | expect(seen.top).toEqual([]); |
outdated
Covers buildSystem only; nothing proves live.ts passes the limits, and the page and TUI pills (limitCores ?? cores.length) have no test. |
||
| 20 | 23 | }); |
| 24 | ||
| 25 | const GiB = 1024 ** 3; | |
| 26 | const hostRaw: Raw = { | |
| 27 | ...raw, | |
| 28 | cores: Array.from({ length: 80 }, () => ({ busy: 0, total: 0 })), | |
| 29 | memory: { total: 64 * GiB, available: 4 * GiB, swapTotal: 0, swapFree: 0 }, | |
| 30 | }; | |
| 31 | ||
| 32 | test("a limited pod shows its own memory use, not the host's (#12347)", () => { | |
| 33 | const v = buildSystem(undefined, hostRaw, [], 1, { cpus: 2, memoryBytes: GiB / 2, usedBytes: GiB / 4 }); | |
| 34 | expect(v.cpu.limitCores).toBe(2); | |
| 35 | expect(v.memory!.totalBytes).toBe(GiB / 2); | |
| 36 | expect(v.memory!.usedBytes).toBe(GiB / 4); // host uses 60 GiB; not "512 MiB of 512 MiB" | |
| 37 | expect(buildSystem(undefined, hostRaw, [], 1).memory!.usedBytes).toBe(60 * GiB); | |
| 38 | }); | |
| 39 | ||
| 40 | test("live passes the pod's limits to the view, and the page and TUI say the limit", () => { | |
| 41 | const l = createLive({ paths: { proc: "/nonexistent", sys: "/nonexistent" }, roots: () => [], self: 1, | |
| 42 | limits: () => ({ cpus: 2, memoryBytes: GiB / 2, usedBytes: GiB / 4 }) }); | |
| 43 | const v = l.read(1_000_000); | |
| 44 | expect(v.cpu.limitCores).toBe(2); | |
| 45 | const cores = Array.from({ length: 8 }, () => 0); | |
| 46 | const view = { ...v, cpu: { ...v.cpu, cores }, memory: { totalBytes: GiB / 2, usedBytes: GiB / 4, swapUsedBytes: 0, swapTotalBytes: 0 } }; | |
| 47 | expect(regions(view)["summary"]).toContain("2 cores"); | |
| 48 | expect(regions(view)["summary"]).not.toContain("8 cores"); | |
| 49 | expect(regions(view)["cores"]).toContain("the rows below are the host's"); | |
| 50 | expect(systemLines(view)[0]).toContain("over 2 cores"); | |
| 51 | }); |
test/vitals.test.ts +26 −1
| @@ -4,7 +4,7 @@ import { laneSentence } from "../src/kernel/vitals.ts"; | ||
| 4 | 4 | import { afterAll, beforeAll, expect, test } from "bun:test"; |
| 5 | 5 | import { writeFileSync } from "node:fs"; |
| 6 | 6 | import { accessDir, flushAccess, recordAccess } from "../src/http/access_log.ts"; |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { mkdtemp, rm, writeFile } from "node:fs/promises"; | |
| 8 | 8 | import { tmpdir } from "node:os"; |
| 9 | 9 | import { join } from "node:path"; |
| 10 | 10 | import { createApp, type App } from "../src/app.ts"; |
| @@ -535,3 +535,28 @@ test("an access log that cannot write is a need past five minutes; an idle forge | ||
| 535 | 535 | await rm(dir, { recursive: true, force: true }); |
| 536 | 536 | } |
| 537 | 537 | }); |
| 538 | ||
| 539 | test("vitals memory follows the pod's injected limit and usage (#12347)", async () => { | |
| 540 | const { memoryReading, vitals } = await import("../src/kernel/vitals.ts"); | |
| 541 | const { podMemoryUsedBytes } = await import("../src/core/capacity.ts"); | |
| 542 | const MiB = 1024 * 1024; | |
| 543 | expect(memoryReading({ limitBytes: 512 * MiB, usedBytes: 100 * MiB }, 50 * MiB)).toMatchObject({ | |
outdated
This restates the implementation using the same functions; it passes with the old totalmem() whenever no cgroup limit is set. Inject the limit and assert vitals().memory follows it. |
||
| 544 | totalBytes: 512 * MiB, | |
| 545 | rssBytes: 50 * MiB, | |
| 546 | }); | |
| 547 | expect(memoryReading({ limitBytes: 512 * MiB, usedBytes: 100 * MiB }, 50 * MiB).freeBytes).toBeLessThanOrEqual(412 * MiB); | |
| 548 | // Unreadable cgroup: the process's own resident memory stands in. | |
| 549 | expect(memoryReading({ limitBytes: 512 * MiB, usedBytes: null }, 500 * MiB).freeBytes).toBeLessThanOrEqual(12 * MiB); | |
| 550 | const v = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, podMemory: { limitBytes: 123 * MiB, usedBytes: 23 * MiB } }); | |
| 551 | expect(v.memory.totalBytes).toBe(123 * MiB); | |
| 552 | expect(v.memory.freeBytes).toBeLessThanOrEqual(100 * MiB); | |
| 553 | const dir = await mkdtemp(join(tmpdir(), "cg-")); | |
| 554 | try { | |
| 555 | await writeFile(join(dir, "memory.current"), "1000\n"); | |
| 556 | await writeFile(join(dir, "memory.stat"), "anon 5\ninactive_file 300\n"); | |
| 557 | expect(podMemoryUsedBytes(dir)).toBe(700); | |
| 558 | expect(podMemoryUsedBytes(join(dir, "none"))).toBeNull(); | |
| 559 | } finally { | |
| 560 | await rm(dir, { recursive: true, force: true }); | |
| 561 | } | |
| 562 | }); |
new above
offline
Host used memory capped at the limit: on a 512 MiB pod on a 64 GiB host with 60 GiB used this reads "512 MiB of 512 MiB", always full. Use the pod cgroup's own memory.current.