279e46e · 96ab2fa · 725d1db
@@ -592,6 +592,22 @@
writable cgroup mount owned by the forge's uid. Without that, cells are
watched only. /system says which holds the cap, and why, and shows each
cell's memory beside its cap.
Processes are held the same way. The watch stops a cell whose tree passes
512 processes; where the pids controller is delegated with memory, each
app and check cell also gets `pids.max` 512, so a fork bomb fails inside
its cell before a tick of the watch, and the forge's process table is
never exhausted (#10518). Crew runs get no process cap, as before.
/system says beside the memory line who holds the process count, and why
when it is the watch alone.
On a Kubernetes pod the forge is usually not at a cgroup namespace root
(`/proc/self/cgroup` is not `0::/`) and `/sys/fs/cgroup` is read-only, so
cells are watched only. The operator delegates one: run the container in
its own cgroup namespace with a writable cgroup2 mount owned by the
forge's uid, or set `FORGE_CELL_CGROUP` to a delegated subtree with
`memory` (and `pids`) in its `cgroup.controllers`. /system then says the
kernel holds the cap.
The forge's own process outranks what it starts (#3319, `apps/background.ts`).
Every check step (install, test, check) and every crew run starts in the
background class: `SCHED_BATCH`, nice 10 (a CFS weight of 110 against the
@@ -20,6 +20,7 @@
 *                    cpu.weight and io.weight 1000 against the cells' 100
 *   R/cells/c<n>/    one per cell: memory.max, memory.swap.max = 0,
 *                    memory.oom.group = 1 (the whole cell dies together);
 *                    pids.max the watch's process limit (#10518);
 *                    a step's cpu.weight and io.weight 10, below an app's
 *                    100, and cpu.max its declared cores (`apps/background.ts`)
 *   R/cells/c<n>/    one per crew run, too: no memory cap, weight 10 and
@@ -174,16 +175,18 @@
export function prepare(
    const cells = join(root, "cells");
    mkdirSync(cells, { recursive: true });
    writeFileSync(join(cells, "cgroup.subtree_control"), "+memory");
    // The forge outweighs every cell, and a step weighs less than an app
    // (#3319). Asked for, never required: without the controller, nice and
    // SCHED_BATCH still hold.
    for (const controller of ["cpu", "io"]) {
    // A cell's process count is the kernel's too (`pids.max`, #10518), and
    // the forge outweighs every cell, a step less than an app (#3319).
    // Asked for, never required: without pids the watch still counts every
    // tick; without cpu or io, nice and SCHED_BATCH still hold.
    for (const controller of ["pids", "cpu", "io"]) {
      if (!words(read(join(root, "cgroup.controllers"))).includes(controller)) continue;
      try {
        for (const dir of [root, cells])
          if (!words(read(join(dir, "cgroup.subtree_control"))).includes(controller))
            writeFileSync(join(dir, "cgroup.subtree_control"), `+${controller}`);
        writeFileSync(join(root, "forge", `${controller}.weight`), `${controller === "io" ? "default " : ""}${FORGE_WEIGHT}`);
        if (controller !== "pids")
          writeFileSync(join(root, "forge", `${controller}.weight`), `${controller === "io" ? "default " : ""}${FORGE_WEIGHT}`);
      } catch {
        // not delegated here, or the forge's leaf is elsewhere: the memory cap still holds
      }
@@ -224,16 +227,28 @@
function removeDir(dir: string, tries = 10): void {
  }
}
/**
 * Whether cells get the pids controller here: `R/cells` lists it in its
 * `cgroup.subtree_control`, as cgroupfs reads it back. Then each cell's
 * `pids.max` is the kernel's; otherwise the watch alone counts processes.
 */
export function pidsDelegated(at: CgroupHost): boolean {
  return at.ok && words(read(join(at.root, "cells", "cgroup.subtree_control"))).includes("pids");
}
/**
 * A cgroup of its own for one cell, capped at `memoryMiB` (`undefined`: no
 * cap, for a crew run); `undefined` where the host has none. `share` is its
 * `cpu.weight`, `io.weight` and `cpu.max` where those controllers reach it;
 * unset leaves the defaults, 100 and no quota.
 * unset leaves the defaults, 100 and no quota. `processes` is its
 * `pids.max` where the pids controller reaches it (`pidsDelegated`); unset
 * or not a positive integer, no kernel cap.
 */
export function openCellCgroup(
  memoryMiB: number | undefined,
  at: CgroupHost = cgroupHost(),
  share?: CpuShare,
  processes?: number,
): CellCgroup | undefined {
  if (!at.ok) return undefined;
  const dir = join(at.root, "cells", `c${nextCell++}-${process.pid}`);
@@ -272,6 +287,13 @@
export function openCellCgroup(
      }
    }
  }
  if (processes !== undefined && Number.isInteger(processes) && processes > 0 && pidsDelegated(at)) {
    try {
      writeFileSync(join(dir, "pids.max"), String(processes));
    } catch {
      // no pids controller here: the watch still counts the cell's processes
    }
  }
  return {
    dir,
    maxBytes,
@@ -38,7 +38,7 @@
import { dirname, join } from "node:path";
import { portInsteadOfSocket } from "../git/hook-address.ts";
import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
import { cgroupHost, enterArgv, openCellCgroup } from "./cgroup.ts";
import { cgroupHost, enterArgv, openCellCgroup, pidsDelegated, type CellCgroup, type CgroupHost } from "./cgroup.ts";
/** `ceiling`: a declaration was cut to the machine's maximum; `declared`: forge.json's own; `default`: neither. */
export type CeilingFrom = "env" | "pod" | "default";
@@ -441,6 +441,23 @@
export function cellMounts(
  return { hide, blank, reveal };
}
/**
 * A cell's own cgroup on `at`: its memory cap, its share of the cores, and
 * `pids.max` at the watch's process limit where pids is delegated (#10518).
 */
export function cellCgroup(spec: CellSpec, at: CgroupHost = cgroupHost()): CellCgroup | undefined {
  return openCellCgroup(
    spec.limits.memoryMiB,
    at,
    spec.background === true
      ? { weight: BACKGROUND_CPU_WEIGHT, cpus: spec.limits.cpus }
      : spec.cpuQuota === true
        ? { weight: 100, cpus: spec.limits.cpus }
        : undefined,
    PROCESSES_MAX,
  );
}
/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
export function openCell(
  spec: CellSpec,
@@ -486,15 +503,7 @@
export function openCell(
  // small and bounded, and the ones that only exist to hide are read-only.
  const SMALL = String(1 << 20);
  const cgroup = openCellCgroup(
    spec.limits.memoryMiB,
    cgroupHost(),
    spec.background === true
      ? { weight: BACKGROUND_CPU_WEIGHT, cpus: spec.limits.cpus }
      : spec.cpuQuota === true
        ? { weight: 100, cpus: spec.limits.cpus }
        : undefined,
  );
  const cgroup = cellCgroup(spec);
  const argv = (command: string): string[] => {
    const inner = bwrapArgv(command);
    return cgroup === undefined ? inner : enterArgv(join(cgroup.dir, "cgroup.procs"), inner);
@@ -702,7 +711,8 @@
function readMountinfo(): string {
/** How often the process table is read, and over how long CPU is averaged. */
export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15 };
const PROCESSES_MAX = 512;
/** A cell's process limit: the watch's every tick, and the kernel's `pids.max` where pids is delegated. */
export const PROCESSES_MAX = 512;
const CLK_TCK = 100;
const DISK_WALK_MAX = 200_000;
@@ -737,6 +747,14 @@
export function memoryCapHolder(): { by: "kernel" } | { by: "watch"; reason: str
  return host.ok ? { by: "kernel" } : { by: "watch", reason: host.reason };
}
/** Whether cells' process counts are the kernel's here (`pids.max`), or why they are watched only. */
export function processCapHolder(host: CgroupHost = cgroupHost()): { by: "kernel" } | { by: "watch"; reason: string } {
  if (!host.ok) return { by: "watch", reason: host.reason };
  return pidsDelegated(host)
    ? { by: "kernel" }
    : { by: "watch", reason: "the pids controller is not delegated to the cells" };
}
/** One labelled cell as its watch last read it: cumulative CPU and resident memory. */
export interface CellCounter {
  /** Unique per watched tree, so a rate never spans two processes. */
@@ -19,7 +19,7 @@
import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
import { telemetryReader } from "../../actions/telemetry.ts";
import type { Ctx } from "../../http/context.ts";
import { memoryCapHolder } from "../../apps/contain.ts";
import { memoryCapHolder, PROCESSES_MAX, processCapHolder } from "../../apps/contain.ts";
import { bytes, spark, span } from "../../system/format.ts";
import { eventStream } from "../../http/sse.ts";
import type { SystemView } from "../../system/view.ts";
@@ -239,11 +239,15 @@
export function systemPage(ctx: Ctx): Response {
  const reader = timePhase(ctx.request, "reader", () => telemetryReader(ctx.app, ctx.auth.user!));
  if (reader?.kind !== "owner") return notFoundPage(ctx);
  const v = timePhase(ctx.request, "sample", () => live().read());
  const holder = timePhase(ctx.request, "caps", () => memoryCapHolder());
  const [holder, pids] = timePhase(ctx.request, "caps", () => [memoryCapHolder(), processCapHolder()] as const);
  const caps: Html =
    holder.by === "kernel"
      ? html`<p class="small" id="cell-caps" data-by="kernel">Each cell's memory is capped by the kernel (its own cgroup's <code>memory.max</code>): a cell over it is killed alone.</p>`
      : html`<p class="small" id="cell-caps" data-by="watch">Each cell's memory is watched once a second and the cell stopped over its cap; the kernel does not cap it (${holder.reason}).</p>`;
  const processCaps: Html =
    pids.by === "kernel"
      ? html`<p class="small" id="cell-pids" data-by="kernel">Each cell's processes are capped by the kernel at ${PROCESSES_MAX} (its cgroup's <code>pids.max</code>; not a crew run's): a fork past it fails inside that cell.</p>`
      : html`<p class="small" id="cell-pids" data-by="watch">Each cell's processes are counted once a second and the cell stopped past ${PROCESSES_MAX}; the kernel does not cap them (${pids.reason}).</p>`;
  const memory = memorySection(ctx);
  const stalls = timePhase(ctx.request, "stalls", () => stallHour(loopWatch.tally(performance.now())));
  const rendering = performance.now();
@@ -264,6 +268,7 @@
export function systemPage(ctx: Ctx): Response {
        <ul class="list" id="cores" aria-label="Per-core CPU, percent, with its last readings">${at("cores")}</ul>
        <ul class="list" id="mem">${at("mem")}</ul>
        ${caps}
        ${processCaps}
        <p class="small mono" id="temps">${at("temps")}</p></section>
    <section class="settings-block"><h2>Disks and network</h2>
      <ul class="list" id="sys-disks">${at("sys-disks")}</ul></section>
@@ -22,7 +22,7 @@
import {
  openCellCgroup,
  prepare,
} from "../src/apps/cgroup.ts";
import { cellsAvailable, type CellSpec } from "../src/apps/contain.ts";
import { cellCgroup, cellsAvailable, processCapHolder, PROCESSES_MAX, type CellSpec } from "../src/apps/contain.ts";
import { childEnv, runStep } from "../src/apps/process.ts";
import type { Proc, Raw } from "../src/system/proc.ts";
import { buildSystem } from "../src/system/view.ts";
@@ -88,8 +88,9 @@
describe("the layout, on a directory shaped like cgroupfs", () => {
    expect(host).toEqual({ ok: true, root: dir });
    // One pid a write: the last one written is what the plain file holds.
    expect(readFileSync(join(dir, "forge", "cgroup.procs"), "utf8")).toBe("12");
    expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+memory");
    expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+memory");
    // The fake holds the last write; cgroupfs adds each: memory, then pids.
    expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+pids");
    expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+pids");
    expect(existsSync(join(dir, "cells", `c7-${gone}`))).toBe(false);
    expect(existsSync(join(dir, "cells", `c8-${process.pid}`))).toBe(true);
    // Already laid out (cgroupfs reads back `memory`): nothing moves again.
@@ -121,6 +122,54 @@
describe("the layout, on a directory shaped like cgroupfs", () => {
    expect(app.cpuCapped).toBe(false);
  });
  const spec = (dir: string): CellSpec => ({
    dir,
    writable: [],
    home: join(dir, ".home"),
    hide: [],
    egress: [],
    reach: [],
    limits: { memoryMiB: 64, cpus: 2, diskMiB: 512 },
  });
  test("where pids is delegated, a cell's process count is the kernel's: pids.max at the watch's limit, and /system says so", () => {
    const dir = fake("pids", "memory pids\n");
    const host = prepare({ ok: true, root: dir });
    expect(host.ok).toBe(true);
    expect(existsSync(join(dir, "forge", "pids.weight"))).toBe(false);
    // cgroupfs reads back every controller enabled, without the `+`.
    writeFileSync(join(dir, "cells", "cgroup.subtree_control"), "memory pids\n");
    expect(processCapHolder(host)).toEqual({ by: "kernel" });
    // What openCell opens: its own spec, the watch's limit as pids.max.
    const cell = cellCgroup(spec(join(dir, "app")), host)!;
    expect(readFileSync(join(cell.dir, "memory.max"), "utf8")).toBe(String(64 * 1024 * 1024));
    expect(readFileSync(join(cell.dir, "pids.max"), "utf8")).toBe(String(PROCESSES_MAX));
    expect(PROCESSES_MAX).toBe(512);
    // Not asked for (a crew run), or not a positive integer: no file.
    for (const processes of [undefined, 0, -1, 1.5, Number.NaN]) {
      const run = openCellCgroup(undefined, host, { weight: 10, cpus: 4 }, processes)!;
      expect(existsSync(join(run.dir, "pids.max"))).toBe(false);
    }
  });
  test("no pids controller: the cell still opens with its memory cap, no pids.max, and /system says the watch counts", () => {
    const dir = fake("nopids", "memory\n");
    const host = prepare({ ok: true, root: dir });
    expect(host.ok).toBe(true);
    // Memory is still what the cells are given.
    expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+memory");
    expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+memory");
    const cell = cellCgroup(spec(join(dir, "app")), host)!;
    expect(cell).toBeDefined();
    expect(readFileSync(join(cell.dir, "memory.max"), "utf8")).toBe(String(64 * 1024 * 1024));
    expect(existsSync(join(cell.dir, "pids.max"))).toBe(false);
    const holder = processCapHolder(host);
    expect(holder.by).toBe("watch");
    if (holder.by === "watch") expect(holder.reason).toContain("pids controller");
    // No cgroup at all: the host's own reason.
    expect(processCapHolder({ ok: false, reason: "FORGE_CELL_CGROUP=off" })).toEqual({ by: "watch", reason: "FORGE_CELL_CGROUP=off" });
  });
  test("a crew run's cell has no memory cap, weighs 10 and is held to its cores", () => {
    const dir = fake("crew", "cpu io memory\n");
    const host = prepare({ ok: true, root: dir });
@@ -306,6 +306,8 @@
describe("telemetry:read: the owner grants it, an agent reads with it, nobody el
    // Each check cell's memory beside its cap, and who holds the cap (#2410).
    expect(html).toContain('data-cap="8589934592" data-cap-by="kernel">of 8.0 GiB (kernel cap)');
    expect(html).toMatch(/id="cell-caps" data-by="(kernel|watch)"/);
    // And who holds each cell's process count (#10518).
    expect(html).toMatch(/id="cell-pids" data-by="(kernel|watch)"/);
    // Built from the system's own parts: pills, meters and the process table grouped by kind (#2721), no style attribute (#2691).
    expect(html).toMatch(/id="summary"[^>]*>\s*<span class="pill"/);
    expect(html).toContain('<tbody data-kind="check"');
		@@ -592,6 +592,22 @@ writable cgroup mount owned by the forge's uid. Without that, cells are
592	592	watched only. /system says which holds the cap, and why, and shows each
593	593	cell's memory beside its cap.
594	594
	595	Processes are held the same way. The watch stops a cell whose tree passes
	596	512 processes; where the pids controller is delegated with memory, each
	597	app and check cell also gets `pids.max` 512, so a fork bomb fails inside
	598	its cell before a tick of the watch, and the forge's process table is
	599	never exhausted (#10518). Crew runs get no process cap, as before.
	600	/system says beside the memory line who holds the process count, and why
	601	when it is the watch alone.
	602
	603	On a Kubernetes pod the forge is usually not at a cgroup namespace root
	604	(`/proc/self/cgroup` is not `0::/`) and `/sys/fs/cgroup` is read-only, so
	605	cells are watched only. The operator delegates one: run the container in
	606	its own cgroup namespace with a writable cgroup2 mount owned by the
	607	forge's uid, or set `FORGE_CELL_CGROUP` to a delegated subtree with
	608	`memory` (and `pids`) in its `cgroup.controllers`. /system then says the
	609	kernel holds the cap.
	610
595	611	The forge's own process outranks what it starts (#3319, `apps/background.ts`).
596	612	Every check step (install, test, check) and every crew run starts in the
597	613	background class: `SCHED_BATCH`, nice 10 (a CFS weight of 110 against the
		@@ -20,6 +20,7 @@
20	20	* cpu.weight and io.weight 1000 against the cells' 100
21	21	* R/cells/c<n>/ one per cell: memory.max, memory.swap.max = 0,
22	22	* memory.oom.group = 1 (the whole cell dies together);
	23	* pids.max the watch's process limit (#10518);
23	24	* a step's cpu.weight and io.weight 10, below an app's
24	25	* 100, and cpu.max its declared cores (`apps/background.ts`)
25	26	* R/cells/c<n>/ one per crew run, too: no memory cap, weight 10 and
		@@ -174,16 +175,18 @@ export function prepare(
174	175	const cells = join(root, "cells");
175	176	mkdirSync(cells, { recursive: true });
176	177	writeFileSync(join(cells, "cgroup.subtree_control"), "+memory");
177		// The forge outweighs every cell, and a step weighs less than an app
178		// (#3319). Asked for, never required: without the controller, nice and
179		// SCHED_BATCH still hold.
180		for (const controller of ["cpu", "io"]) {
	178	// A cell's process count is the kernel's too (`pids.max`, #10518), and
	179	// the forge outweighs every cell, a step less than an app (#3319).
	180	// Asked for, never required: without pids the watch still counts every
	181	// tick; without cpu or io, nice and SCHED_BATCH still hold.
	182	for (const controller of ["pids", "cpu", "io"]) {
181	183	if (!words(read(join(root, "cgroup.controllers"))).includes(controller)) continue;
182	184	try {
183	185	for (const dir of [root, cells])
184	186	if (!words(read(join(dir, "cgroup.subtree_control"))).includes(controller))
185	187	writeFileSync(join(dir, "cgroup.subtree_control"), `+${controller}`);
186		writeFileSync(join(root, "forge", `${controller}.weight`), `${controller === "io" ? "default " : ""}${FORGE_WEIGHT}`);
	188	if (controller !== "pids")
	189	writeFileSync(join(root, "forge", `${controller}.weight`), `${controller === "io" ? "default " : ""}${FORGE_WEIGHT}`);
187	190	} catch {
188	191	// not delegated here, or the forge's leaf is elsewhere: the memory cap still holds
189	192	}
		@@ -224,16 +227,28 @@ function removeDir(dir: string, tries = 10): void {
224	227	}
225	228	}
226	229
	230	/**
	231	* Whether cells get the pids controller here: `R/cells` lists it in its
	232	* `cgroup.subtree_control`, as cgroupfs reads it back. Then each cell's
	233	* `pids.max` is the kernel's; otherwise the watch alone counts processes.
	234	*/
	235	export function pidsDelegated(at: CgroupHost): boolean {
	236	return at.ok && words(read(join(at.root, "cells", "cgroup.subtree_control"))).includes("pids");
	237	}
	238
227	239	/**
228	240	* A cgroup of its own for one cell, capped at `memoryMiB` (`undefined`: no
229	241	* cap, for a crew run); `undefined` where the host has none. `share` is its
230	242	* `cpu.weight`, `io.weight` and `cpu.max` where those controllers reach it;
231		* unset leaves the defaults, 100 and no quota.
	243	* unset leaves the defaults, 100 and no quota. `processes` is its
	244	* `pids.max` where the pids controller reaches it (`pidsDelegated`); unset
	245	* or not a positive integer, no kernel cap.
232	246	*/
233	247	export function openCellCgroup(
234	248	memoryMiB: number \| undefined,
235	249	at: CgroupHost = cgroupHost(),
236	250	share?: CpuShare,
	251	processes?: number,
237	252	): CellCgroup \| undefined {
238	253	if (!at.ok) return undefined;
239	254	const dir = join(at.root, "cells", `c${nextCell++}-${process.pid}`);
		@@ -272,6 +287,13 @@ export function openCellCgroup(
272	287	}
273	288	}
274	289	}
	290	if (processes !== undefined && Number.isInteger(processes) && processes > 0 && pidsDelegated(at)) {
	291	try {
	292	writeFileSync(join(dir, "pids.max"), String(processes));
	293	} catch {
	294	// no pids controller here: the watch still counts the cell's processes
	295	}
	296	}
275	297	return {
276	298	dir,
277	299	maxBytes,
		@@ -38,7 +38,7 @@ import { dirname, join } from "node:path";
38	38	import { portInsteadOfSocket } from "../git/hook-address.ts";
39	39	import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
40	40	import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
41		import { cgroupHost, enterArgv, openCellCgroup } from "./cgroup.ts";
	41	import { cgroupHost, enterArgv, openCellCgroup, pidsDelegated, type CellCgroup, type CgroupHost } from "./cgroup.ts";
42	42
43	43	/** `ceiling`: a declaration was cut to the machine's maximum; `declared`: forge.json's own; `default`: neither. */
44	44	export type CeilingFrom = "env" \| "pod" \| "default";
		@@ -441,6 +441,23 @@ export function cellMounts(
441	441	return { hide, blank, reveal };
442	442	}
443	443
	444	/**
	445	* A cell's own cgroup on `at`: its memory cap, its share of the cores, and
	446	* `pids.max` at the watch's process limit where pids is delegated (#10518).
	447	*/
	448	export function cellCgroup(spec: CellSpec, at: CgroupHost = cgroupHost()): CellCgroup \| undefined {
	449	return openCellCgroup(
	450	spec.limits.memoryMiB,
	451	at,
	452	spec.background === true
	453	? { weight: BACKGROUND_CPU_WEIGHT, cpus: spec.limits.cpus }
	454	: spec.cpuQuota === true
	455	? { weight: 100, cpus: spec.limits.cpus }
	456	: undefined,
	457	PROCESSES_MAX,
	458	);
	459	}
	460
444	461	/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
445	462	export function openCell(
446	463	spec: CellSpec,
		@@ -486,15 +503,7 @@ export function openCell(
486	503	// small and bounded, and the ones that only exist to hide are read-only.
487	504	const SMALL = String(1 << 20);
488	505
489		const cgroup = openCellCgroup(
490		spec.limits.memoryMiB,
491		cgroupHost(),
492		spec.background === true
493		? { weight: BACKGROUND_CPU_WEIGHT, cpus: spec.limits.cpus }
494		: spec.cpuQuota === true
495		? { weight: 100, cpus: spec.limits.cpus }
496		: undefined,
497		);
	506	const cgroup = cellCgroup(spec);
498	507	const argv = (command: string): string[] => {
499	508	const inner = bwrapArgv(command);
500	509	return cgroup === undefined ? inner : enterArgv(join(cgroup.dir, "cgroup.procs"), inner);
		@@ -702,7 +711,8 @@ function readMountinfo(): string {
702	711
703	712	/** How often the process table is read, and over how long CPU is averaged. */
704	713	export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15 };
705		const PROCESSES_MAX = 512;
	714	/** A cell's process limit: the watch's every tick, and the kernel's `pids.max` where pids is delegated. */
	715	export const PROCESSES_MAX = 512;
706	716	const CLK_TCK = 100;
707	717	const DISK_WALK_MAX = 200_000;
708	718
		@@ -737,6 +747,14 @@ export function memoryCapHolder(): { by: "kernel" } \| { by: "watch"; reason: str
737	747	return host.ok ? { by: "kernel" } : { by: "watch", reason: host.reason };
738	748	}
739	749
	750	/** Whether cells' process counts are the kernel's here (`pids.max`), or why they are watched only. */
	751	export function processCapHolder(host: CgroupHost = cgroupHost()): { by: "kernel" } \| { by: "watch"; reason: string } {
	752	if (!host.ok) return { by: "watch", reason: host.reason };
	753	return pidsDelegated(host)
	754	? { by: "kernel" }
	755	: { by: "watch", reason: "the pids controller is not delegated to the cells" };
	756	}
	757
740	758	/** One labelled cell as its watch last read it: cumulative CPU and resident memory. */
741	759	export interface CellCounter {
742	760	/** Unique per watched tree, so a rate never spans two processes. */
		@@ -19,7 +19,7 @@
19	19	import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
20	20	import { telemetryReader } from "../../actions/telemetry.ts";
21	21	import type { Ctx } from "../../http/context.ts";
22		import { memoryCapHolder } from "../../apps/contain.ts";
	22	import { memoryCapHolder, PROCESSES_MAX, processCapHolder } from "../../apps/contain.ts";
23	23	import { bytes, spark, span } from "../../system/format.ts";
24	24	import { eventStream } from "../../http/sse.ts";
25	25	import type { SystemView } from "../../system/view.ts";
		@@ -239,11 +239,15 @@ export function systemPage(ctx: Ctx): Response {
239	239	const reader = timePhase(ctx.request, "reader", () => telemetryReader(ctx.app, ctx.auth.user!));
240	240	if (reader?.kind !== "owner") return notFoundPage(ctx);
241	241	const v = timePhase(ctx.request, "sample", () => live().read());
242		const holder = timePhase(ctx.request, "caps", () => memoryCapHolder());
	242	const [holder, pids] = timePhase(ctx.request, "caps", () => [memoryCapHolder(), processCapHolder()] as const);
243	243	const caps: Html =
244	244	holder.by === "kernel"
245	245	? html`<p class="small" id="cell-caps" data-by="kernel">Each cell's memory is capped by the kernel (its own cgroup's <code>memory.max</code>): a cell over it is killed alone.</p>`
246	246	: html`<p class="small" id="cell-caps" data-by="watch">Each cell's memory is watched once a second and the cell stopped over its cap; the kernel does not cap it (${holder.reason}).</p>`;
	247	const processCaps: Html =
	248	pids.by === "kernel"
	249	? html`<p class="small" id="cell-pids" data-by="kernel">Each cell's processes are capped by the kernel at ${PROCESSES_MAX} (its cgroup's <code>pids.max</code>; not a crew run's): a fork past it fails inside that cell.</p>`
	250	: html`<p class="small" id="cell-pids" data-by="watch">Each cell's processes are counted once a second and the cell stopped past ${PROCESSES_MAX}; the kernel does not cap them (${pids.reason}).</p>`;
247	251	const memory = memorySection(ctx);
248	252	const stalls = timePhase(ctx.request, "stalls", () => stallHour(loopWatch.tally(performance.now())));
249	253	const rendering = performance.now();
		@@ -264,6 +268,7 @@ export function systemPage(ctx: Ctx): Response {
264	268	<ul class="list" id="cores" aria-label="Per-core CPU, percent, with its last readings">${at("cores")}</ul>
265	269	<ul class="list" id="mem">${at("mem")}</ul>
266	270	${caps}
	271	${processCaps}
267	272	<p class="small mono" id="temps">${at("temps")}</p></section>
268	273	<section class="settings-block"><h2>Disks and network</h2>
269	274	<ul class="list" id="sys-disks">${at("sys-disks")}</ul></section>
		@@ -22,7 +22,7 @@ import {
22	22	openCellCgroup,
23	23	prepare,
24	24	} from "../src/apps/cgroup.ts";
25		import { cellsAvailable, type CellSpec } from "../src/apps/contain.ts";
	25	import { cellCgroup, cellsAvailable, processCapHolder, PROCESSES_MAX, type CellSpec } from "../src/apps/contain.ts";
26	26	import { childEnv, runStep } from "../src/apps/process.ts";
27	27	import type { Proc, Raw } from "../src/system/proc.ts";
28	28	import { buildSystem } from "../src/system/view.ts";
		@@ -88,8 +88,9 @@ describe("the layout, on a directory shaped like cgroupfs", () => {
88	88	expect(host).toEqual({ ok: true, root: dir });
89	89	// One pid a write: the last one written is what the plain file holds.
90	90	expect(readFileSync(join(dir, "forge", "cgroup.procs"), "utf8")).toBe("12");
91		expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+memory");
92		expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+memory");
	91	// The fake holds the last write; cgroupfs adds each: memory, then pids.
	92	expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+pids");
	93	expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+pids");
93	94	expect(existsSync(join(dir, "cells", `c7-${gone}`))).toBe(false);
94	95	expect(existsSync(join(dir, "cells", `c8-${process.pid}`))).toBe(true);
95	96	// Already laid out (cgroupfs reads back `memory`): nothing moves again.
		@@ -121,6 +122,54 @@ describe("the layout, on a directory shaped like cgroupfs", () => {
121	122	expect(app.cpuCapped).toBe(false);
122	123	});
123	124
	125	const spec = (dir: string): CellSpec => ({
	126	dir,
	127	writable: [],
	128	home: join(dir, ".home"),
	129	hide: [],
	130	egress: [],
	131	reach: [],
	132	limits: { memoryMiB: 64, cpus: 2, diskMiB: 512 },
	133	});
	134
	135	test("where pids is delegated, a cell's process count is the kernel's: pids.max at the watch's limit, and /system says so", () => {
	136	const dir = fake("pids", "memory pids\n");
	137	const host = prepare({ ok: true, root: dir });
	138	expect(host.ok).toBe(true);
	139	expect(existsSync(join(dir, "forge", "pids.weight"))).toBe(false);
	140	// cgroupfs reads back every controller enabled, without the `+`.
	141	writeFileSync(join(dir, "cells", "cgroup.subtree_control"), "memory pids\n");
	142	expect(processCapHolder(host)).toEqual({ by: "kernel" });
	143	// What openCell opens: its own spec, the watch's limit as pids.max.
	144	const cell = cellCgroup(spec(join(dir, "app")), host)!;
	145	expect(readFileSync(join(cell.dir, "memory.max"), "utf8")).toBe(String(64 * 1024 * 1024));
	146	expect(readFileSync(join(cell.dir, "pids.max"), "utf8")).toBe(String(PROCESSES_MAX));
	147	expect(PROCESSES_MAX).toBe(512);
	148	// Not asked for (a crew run), or not a positive integer: no file.
	149	for (const processes of [undefined, 0, -1, 1.5, Number.NaN]) {
	150	const run = openCellCgroup(undefined, host, { weight: 10, cpus: 4 }, processes)!;
	151	expect(existsSync(join(run.dir, "pids.max"))).toBe(false);
	152	}
	153	});
	154
	155	test("no pids controller: the cell still opens with its memory cap, no pids.max, and /system says the watch counts", () => {
	156	const dir = fake("nopids", "memory\n");
	157	const host = prepare({ ok: true, root: dir });
	158	expect(host.ok).toBe(true);
	159	// Memory is still what the cells are given.
	160	expect(readFileSync(join(dir, "cgroup.subtree_control"), "utf8")).toBe("+memory");
	161	expect(readFileSync(join(dir, "cells", "cgroup.subtree_control"), "utf8")).toBe("+memory");
	162	const cell = cellCgroup(spec(join(dir, "app")), host)!;
	163	expect(cell).toBeDefined();
	164	expect(readFileSync(join(cell.dir, "memory.max"), "utf8")).toBe(String(64 * 1024 * 1024));
	165	expect(existsSync(join(cell.dir, "pids.max"))).toBe(false);
	166	const holder = processCapHolder(host);
	167	expect(holder.by).toBe("watch");
	168	if (holder.by === "watch") expect(holder.reason).toContain("pids controller");
	169	// No cgroup at all: the host's own reason.
	170	expect(processCapHolder({ ok: false, reason: "FORGE_CELL_CGROUP=off" })).toEqual({ by: "watch", reason: "FORGE_CELL_CGROUP=off" });
	171	});
	172
124	173	test("a crew run's cell has no memory cap, weighs 10 and is held to its cores", () => {
125	174	const dir = fake("crew", "cpu io memory\n");
126	175	const host = prepare({ ok: true, root: dir });
		@@ -306,6 +306,8 @@ describe("telemetry:read: the owner grants it, an agent reads with it, nobody el
306	306	// Each check cell's memory beside its cap, and who holds the cap (#2410).
307	307	expect(html).toContain('data-cap="8589934592" data-cap-by="kernel">of 8.0 GiB (kernel cap)');
308	308	expect(html).toMatch(/id="cell-caps" data-by="(kernel\|watch)"/);
	309	// And who holds each cell's process count (#10518).
	310	expect(html).toMatch(/id="cell-pids" data-by="(kernel\|watch)"/);
309	311	// Built from the system's own parts: pills, meters and the process table grouped by kind (#2721), no style attribute (#2691).
310	312	expect(html).toMatch(/id="summary"[^>]>\s<span class="pill"/);
311	313	expect(html).toContain('<tbody data-kind="check"');