bddaf5c · e3112be · d122a66 · bffba22 · 1063e47 · 9f7edea
@@ -1,5 +1,4 @@
[
  "src/apps/cpu_rule.ts",
  "src/core/one_core.ts",
  "src/perf/page_budget.ts",
  "src/perf/page_budget_read.ts",
@@ -50,6 +50,7 @@
import {
} from "./cgroup.ts";
import { during } from "../perf/jobs.ts";
import { isSize, SIZES, type Size } from "../spawn/sizes.ts";
import { cpuWatch } from "./cpu_rule.ts";
/** `ceiling`: a declaration was cut to the machine's maximum; `declared`: forge.json's own; `default`: neither. */
export type CeilingFrom = "env" | "pod" | "default" | "size";
@@ -632,6 +633,54 @@
export function cellCgroup(
  );
}
/** Where the next pinned cell's cores begin; `affinityArgv` advances it. */
export interface CoreRotor {
  next: number;
}
/** The process's one rotor: every cell opened here draws its cores from it. */
const cellCores: CoreRotor = { next: 0 };
/**
 * Where the kernel's `cpu.max` is not ours to write, a cell that declares a
 * quota is held to its cores by CPU affinity instead (`taskset -c`, util-linux;
 * sched_setaffinity(2)): a 1-CPU forge builds slowly on one core instead of
 * being stopped. The cores rotate so forges do not all pile on core 0.
 * Nothing is added when the kernel holds the cores, `taskset` is absent, or
 * the cell may have every core the machine has (#11657).
 */
export function affinityArgv(
  spec: Pick<CellSpec, "cpuQuota" | "limits">,
  kernelHolds: boolean,
  rotor: CoreRotor = cellCores,
  machine = availableParallelism(),
  have: boolean = Bun.which("taskset") !== null,
): string[] {
  if (spec.cpuQuota !== true || kernelHolds || !have) return [];
  const n = Math.max(1, Math.ceil(spec.limits.cpus));
  if (!Number.isFinite(n) || n >= machine) return [];
  const first = rotor.next % machine;
  rotor.next = (first + n) % machine;
  const cores = Array.from({ length: n }, (_, i) => (first + i) % machine);
  return ["taskset", "-c", cores.join(",")];
}
/**
 * The argv a cell's command runs as: into its cgroup when it has one, then
 * pinned to its cores when the kernel does not hold them, then `inner` (bwrap).
 */
export function cellArgv(
  inner: readonly string[],
  spec: Pick<CellSpec, "cpuQuota" | "limits">,
  cgroup: { readonly dir: string; readonly cpuCapped: boolean } | undefined,
  rotor: CoreRotor = cellCores,
  machine = availableParallelism(),
  have: boolean = Bun.which("taskset") !== null,
): string[] {
  const pinned = [...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have), ...inner];
  return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned);
}
/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
export function openCell(
  spec: CellSpec,
@@ -681,10 +730,7 @@
export function openCell(
  const SMALL = String(1 << 20);
  const cgroup = cellCgroup(spec);
  const argv = (command: string): string[] => {
    const inner = bwrapArgv(command);
    return cgroup === undefined ? inner : enterArgv(join(cgroup.dir, "cgroup.procs"), inner);
  };
  const argv = (command: string): string[] => cellArgv(bwrapArgv(command), spec, cgroup);
  const rlimits = cellRlimits(spec.limits);
  const bwrapArgv = (command: string): string[] => [
    "bwrap",
@@ -894,15 +940,6 @@
function readMountinfo(): string {
/** How often the process table is read, and over how long CPU is averaged. */
export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15 };
/**
 * How far past its cores a cell may run before the watcher stops it. A
 * cell is already held to its cores by affinity, so a busy one measures
 * right at its limit (16.0 to 18.2 of 16 over a window): stopping at the
 * limit itself failed three checks in four and stopped a 1-core forge at
 * 1.0. The watcher is the last resort for a runaway, so it acts well past
 * the limit.
 */
export const CPU_STOP_SLACK = 1.5;
/** A cell's process limit: the watch's every tick, and the kernel's `pids.max` where pids is delegated. */
export const PROCESSES_MAX = 512;
const CLK_TCK = 100;
@@ -915,6 +952,9 @@
interface Watched {
  readonly breach: (why: string) => void;
  readonly samples: { at: number; ticks: number }[];
  ticks: number;
  /** When the cell began to be watched, and since when its cores have held over the stop line without a break (#11657). */
  readonly startedAt: number;
  overSince: number | null;
  readonly key: number;
  readonly label: string | undefined;
  /** The memory cap, and who holds it: the kernel (a cgroup) or this watch alone. */
@@ -1097,6 +1137,8 @@
export function watchTree(
    breach,
    samples: [],
    ticks: 0,
    startedAt: Date.now(),
    overSince: null,
    key: nextKey++,
    label,
    cap,
@@ -1249,6 +1291,36 @@
export function diskBytes(dirs: readonly string[]): number {
const mib = (bytes: number): string => `${Math.round(bytes / 1024 / 1024)} MiB`;
/** The part of a watched cell its CPU decision reads and keeps. */
export interface CpuState {
  readonly samples: { at: number; ticks: number }[];
  readonly startedAt: number;
  overSince: number | null;
}
/**
 * One tick's CPU decision for a watched cell: record the tree's CPU `ticks`
 * at `now`, average the cores over `WATCH.cpuWindowMs`, keep the hold on
 * `state`, and say whether `cpuWatch` (`apps/cpu_rule.ts`) stops it — with the
 * cores and the window it measured — or null to let it run (#11657).
 */
export function cpuSample(
  state: CpuState,
  ticks: number,
  cpus: number,
  now: number,
): { cores: number; seconds: number } | null {
  state.samples.push({ at: now, ticks });
  while (state.samples.length > 1 && now - (state.samples[1]?.at ?? now) >= WATCH.cpuWindowMs) state.samples.shift();
  const oldest = state.samples[0];
  if (oldest === undefined || now - oldest.at < WATCH.cpuWindowMs) return null;
  const seconds = (now - oldest.at) / 1000;
  const cores = (ticks - oldest.ticks) / CLK_TCK / seconds;
  const verdict = cpuWatch(state.overSince, cores, cpus, now, state.startedAt);
  state.overSince = verdict.overSinceMs;
  return verdict.stop ? { cores, seconds } : null;
}
function tick(): void {
  if (watched.size === 0) return;
  const { rows, children } = processTable();
@@ -1282,24 +1354,14 @@
function tick(): void {
      );
      continue;
    }
    entry.samples.push({ at: now, ticks });
    while (
      entry.samples.length > 1 &&
      now - (entry.samples[1]?.at ?? now) >= WATCH.cpuWindowMs
    )
      entry.samples.shift();
    const oldest = entry.samples[0];
    if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
      const cores =
        (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
      if (cores > limits.cpus * CPU_STOP_SLACK) {
        // Read only now, at the stop: no tick pays for naming a process.
        const top = busiest(tree, (pid) => rows.get(pid)?.ticks ?? 0);
        entry.breach(
          cpuBreach(cores, (now - oldest.at) / 1000, limits.cpus, top === undefined ? undefined : { ...top, name: commandName(top.pid) }),
        );
        continue;
      }
    const over = cpuSample(entry, ticks, limits.cpus, now);
    if (over !== null) {
      // Read only now, at the stop: no tick pays for naming a process.
      const top = busiest(tree, (pid) => rows.get(pid)?.ticks ?? 0);
      entry.breach(
        cpuBreach(over.cores, over.seconds, limits.cpus, top === undefined ? undefined : { ...top, name: commandName(top.pid) }),
      );
      continue;
    }
    entry.ticks++;
    if (entry.ticks % WATCH.diskEveryTicks === 0) {
@@ -1,9 +1,18 @@
/**
 * When a cell's CPU use is a stop. Pure: plain data in, a boolean out; no clock,
 * no I/O. No caller yet (#11612, part of #11539).
 * no I/O. Called by the cell watch (`apps/contain.ts` tick, #11657).
 *
 * Rule: cores used > CPU_STOP_FACTOR × the cell's cpus, held continuously for at
 * least CPU_STOP_HOLD_MS, and the cell at least CPU_STOP_GRACE_MS old.
 *
 * The cost of the grace, chosen (#11657): the watch is the only bound on a cell
 * the kernel does not hold (no delegated `cpu.max`) and affinity does not pin
 * (`taskset` absent). Such a cell — an untrusted check included — may use every
 * core of the host for up to CPU_STOP_GRACE_MS (10 minutes) before it can be
 * stopped, and must keep it up for CPU_STOP_HOLD_MS. A cell whose cores are the
 * whole machine is within its limit at any load. The other choice, a quick stop,
 * is what killed a sized forge's builds at 2.1 of 1 core after 31 s; where the
 * kernel holds the quota or `taskset` pins the cores this cost does not arise.
 */
/** A cell is over its limit only past this multiple of its cores. */
@@ -33,3 +42,30 @@
export function cpuStop(i: CpuStopInput): boolean {
  if (nowMs - overSinceMs < CPU_STOP_HOLD_MS) return false;
  return nowMs - startedAtMs >= CPU_STOP_GRACE_MS;
}
/** The next `overSinceMs`: kept while cores stay over the line, set to now when they cross it, null when they drop under. */
function overSince(prev: number | null, cores: number, cpus: number, nowMs: number): number | null {
  if (!(cores > CPU_STOP_FACTOR * cpus)) return null;
  return prev ?? nowMs;
}
/** What one tick of the watch decides: the hold to keep, and whether to stop. */
export interface CpuVerdict {
  readonly overSinceMs: number | null;
  readonly stop: boolean;
}
/**
 * One tick of the watch's decision, pure: the new `overSinceMs` and whether the cell is
 * stopped. The watch keeps the first and acts on the second (`apps/contain.ts`, #11657).
 */
export function cpuWatch(
  prevOverSinceMs: number | null,
  cores: number,
  cpus: number,
  nowMs: number,
  startedAtMs: number,
): CpuVerdict {
  const overSinceMs = overSince(prevOverSinceMs, cores, cpus, nowMs);
  return { overSinceMs, stop: cpuStop({ cores, cpus, overSinceMs, nowMs, startedAtMs }) };
}
@@ -834,14 +834,14 @@
describe.skipIf(noCells)("cells: limits stop a step, and say which", () => {
      );
  }, 30_000);
  test("over its CPU limit", async () => {
    // A small share, so a busy machine still gives the loop more than it.
    const result = await step(`bun -e "for (;;) {}"`, cell({ cpus: 0.02 }));
    expect(result.ok).toBe(false);
    if (!result.ok)
      expect(result.error).toMatch(
        /^check stopped: it used [\d.]+ cores for \d+s, over its limit of 0\.02; the busiest process is \S+ \(pid \d+, \d+s of CPU\)$/,
      );
  test("over its CPU limit inside its grace: it runs on, and finishes (#11657)", async () => {
    // Twice the (shrunk) CPU window busy at far over 0.02 cores: the old 1.5× line
    // stopped this at the first full window; the rule waits out the grace first.
    const result = await step(
      `bun -e "const end = Date.now() + 6000; while (Date.now() < end) {}"`,
      cell({ cpus: 0.02 }),
    );
    expect(result.ok).toBe(true);
  }, 30_000);
  test("over its disk limit", async () => {
@@ -1,5 +1,7 @@
import { expect, test } from "bun:test";
import { CPU_STOP_GRACE_MS, CPU_STOP_HOLD_MS, cpuStop } from "../src/apps/cpu_rule.ts";
import { describe, expect, test } from "bun:test";
import { CPU_STOP_GRACE_MS, CPU_STOP_HOLD_MS, cpuStop, cpuWatch } from "../src/apps/cpu_rule.ts";
import { affinityArgv, cellArgv, cpuSample, type CpuState, WATCH } from "../src/apps/contain.ts";
import { enterArgv } from "../src/apps/cgroup.ts";
const M = 60_000;
const now = 100 * M;
@@ -27,3 +29,123 @@
test("3.0 of 1 at minute 9 of the cell's life is kept; at minute 10 it is stoppe
test("an infinite cpus (kernel-capped) is kept", () => {
  expect(cpuStop({ cores: 50, cpus: Infinity, overSinceMs: now - 10 * M, nowMs: now, startedAtMs: old })).toBe(false);
});
      
        junipercrew
        2d ago
        outdated
      
      Move imports to the top of the file; add a test for the Settings page and one for the watch wiring that runs without cells.
    
/** Drive the watch's per-tick decision with a clock, as the cell watch does. */
function run(cores: number, cpus: number, forMs: number, startedAt = 0): { stoppedAt: number | null } {
  let over: number | null = null;
  for (let t = 0; t <= forMs; t += 1_000) {
    const v = cpuWatch(over, cores, cpus, startedAt + t, startedAt);
    over = v.overSinceMs;
    if (v.stop) return { stoppedAt: t };
  }
  return { stoppedAt: null };
}
test("the watch: a 1-CPU cell at 2.1 cores is not stopped after 31 s, or after 5 minutes (#11657)", () => {
  expect(run(2.1, 1, 31_000).stoppedAt).toBeNull();
  expect(run(2.1, 1, CPU_STOP_HOLD_MS - 1_000).stoppedAt).toBeNull();
});
test("the watch: 3.0 of 1 core held without a break is stopped at the hold, never before", () => {
  const { stoppedAt } = run(3, 1, 20 * M);
  expect(stoppedAt).toBe(Math.max(CPU_STOP_HOLD_MS, CPU_STOP_GRACE_MS));
});
test("the watch: a dip under the line resets the hold", () => {
  const a = cpuWatch(null, 3, 1, 1_000, 0);
  expect(a.overSinceMs).toBe(1_000);
  expect(cpuWatch(a.overSinceMs, 3, 1, 5_000, 0).overSinceMs).toBe(1_000);
  expect(cpuWatch(a.overSinceMs, 1.5, 1, 6_000, 0)).toEqual({ overSinceMs: null, stop: false });
});
const limits = (cpus: number) => ({ memoryMiB: 1, cpus, diskMiB: 1 });
test("a cell the kernel does not hold is pinned to its cores; one it holds, or the whole machine, is not (#11657)", () => {
  const spec = { cpuQuota: true, limits: limits(1) };
  const rotor = () => ({ next: 0 });
  expect(affinityArgv(spec, false, rotor(), 4, true)).toEqual(["taskset", "-c", "0"]);
  expect(affinityArgv(spec, true, rotor(), 4, true)).toEqual([]);
  expect(affinityArgv(spec, false, rotor(), 4, false)).toEqual([]);
  expect(affinityArgv(spec, false, rotor(), 1, true)).toEqual([]);
  expect(affinityArgv({ cpuQuota: false, limits: limits(1) }, false, rotor(), 4, true)).toEqual([]);
});
test("pinned cells rotate over the cores, wrapping past the last", () => {
  const spec = { cpuQuota: true, limits: limits(2) };
  const rotor = { next: 0 };
  const cores = Array.from({ length: 3 }, () => affinityArgv(spec, false, rotor, 4, true)[2]);
  expect(cores).toEqual(["0,1", "2,3", "0,1"]);
  // A 1.5-cpu cell takes two whole cores; one from 3 of 4 wraps to core 0.
  expect(affinityArgv({ cpuQuota: true, limits: limits(1.5) }, false, { next: 3 }, 4, true)[2]).toBe("3,0");
});
describe("the argv openCell runs a command as (cellArgv)", () => {
  const bwrap = ["bwrap", "--", "true"];
  const spec = { cpuQuota: true, limits: limits(1) };
  test("no cgroup the kernel caps: the command is pinned inside the cgroup's enter", () => {
    const argv = cellArgv(bwrap, spec, { dir: "/sys/fs/cgroup/cell-a", cpuCapped: false }, { next: 2 }, 4, true);
    expect(argv).toEqual(enterArgv("/sys/fs/cgroup/cell-a/cgroup.procs", ["taskset", "-c", "2", ...bwrap]));
  });
  test("a cgroup that caps the cores: entered, not pinned", () => {
    const argv = cellArgv(bwrap, spec, { dir: "/sys/fs/cgroup/cell-b", cpuCapped: true }, { next: 0 }, 4, true);
    expect(argv).toEqual(enterArgv("/sys/fs/cgroup/cell-b/cgroup.procs", bwrap));
  });
  test("no cgroup at all: pinned, then bwrap", () => {
    expect(cellArgv(bwrap, spec, undefined, { next: 0 }, 4, true)).toEqual(["taskset", "-c", "0", ...bwrap]);
    expect(cellArgv(bwrap, spec, undefined, { next: 0 }, 4, false)).toEqual(bwrap);
  });
});
describe("the watch's tick feeds the rule (cpuSample)", () => {
  const CLK_TCK = 100;
  /** A tree burning `cores` from t = 0, sampled each second; the second at which it is stopped, or null. */
  function watch(cores: number, cpus: number, forMs: number): { stoppedAt: number | null; state: CpuState } {
    const state: CpuState = { samples: [], startedAt: 0, overSince: null };
    for (let t = 0; t <= forMs; t += 1_000) {
      const over = cpuSample(state, Math.round((cores * CLK_TCK * t) / 1000), cpus, t);
      if (over !== null) {
        expect(over.cores).toBeCloseTo(cores, 5);
        expect(over.seconds).toBe(WATCH.cpuWindowMs / 1000);
        return { stoppedAt: t, state };
      }
    }
    return { stoppedAt: null, state };
  }
  test("a 1-CPU forge at 2.1 cores is not stopped at 31 s — the old 1.5× line would have (#11657)", () => {
    const { stoppedAt, state } = watch(2.1, 1, 31_000);
    expect(stoppedAt).toBeNull();
    // The window filled at 30 s, and from then the hold is kept on the watched cell.
    expect(state.overSince).toBe(WATCH.cpuWindowMs);
  });
  test("2.1 of 1 core held is stopped once the grace is over, not before", () => {
    expect(watch(2.1, 1, CPU_STOP_GRACE_MS - 1_000).stoppedAt).toBeNull();
    expect(watch(2.1, 1, 20 * M).stoppedAt).toBe(CPU_STOP_GRACE_MS);
  });
  test("at twice its cores, not over: never stopped, and no hold kept", () => {
    const { stoppedAt, state } = watch(2, 1, 20 * M);
    expect(stoppedAt).toBeNull();
    expect(state.overSince).toBeNull();
  });
  test("a dip under the line inside the window restarts the hold", () => {
    const state: CpuState = { samples: [], startedAt: -CPU_STOP_GRACE_MS, overSince: null };
    let ticks = 0;
    let stoppedAt: number | null = null;
    for (let t = 0; t <= 20 * M && stoppedAt === null; t += 1_000) {
      // Busy at 3 cores, idle for one window at 4 minutes.
      const idle = t > 4 * M && t <= 4 * M + WATCH.cpuWindowMs;
      if (t > 0) ticks += idle ? 0 : 3 * CLK_TCK;
      if (cpuSample(state, ticks, 1, t) !== null) stoppedAt = t;
    }
    // Over again from the first full busy window after the dip; stopped a hold later.
    expect(stoppedAt).toBeGreaterThan(4 * M + CPU_STOP_HOLD_MS);
    expect(stoppedAt).toBeLessThanOrEqual(4 * M + 2 * WATCH.cpuWindowMs + CPU_STOP_HOLD_MS);
  });
});
		@@ -1,5 +1,4 @@
1	1	[
2		"src/apps/cpu_rule.ts",
3	2	"src/core/one_core.ts",
4	3	"src/perf/page_budget.ts",
5	4	"src/perf/page_budget_read.ts",
		@@ -50,6 +50,7 @@ import {
50	50	} from "./cgroup.ts";
51	51	import { during } from "../perf/jobs.ts";
52	52	import { isSize, SIZES, type Size } from "../spawn/sizes.ts";
	53	import { cpuWatch } from "./cpu_rule.ts";
53	54
54	55	/** `ceiling`: a declaration was cut to the machine's maximum; `declared`: forge.json's own; `default`: neither. */
55	56	export type CeilingFrom = "env" \| "pod" \| "default" \| "size";
		@@ -632,6 +633,54 @@ export function cellCgroup(
632	633	);
633	634	}
634	635
	636	/** Where the next pinned cell's cores begin; `affinityArgv` advances it. */
	637	export interface CoreRotor {
	638	next: number;
	639	}
	640
	641	/** The process's one rotor: every cell opened here draws its cores from it. */
	642	const cellCores: CoreRotor = { next: 0 };
	643
	644	/**
	645	* Where the kernel's `cpu.max` is not ours to write, a cell that declares a
	646	* quota is held to its cores by CPU affinity instead (`taskset -c`, util-linux;
	647	* sched_setaffinity(2)): a 1-CPU forge builds slowly on one core instead of
	648	* being stopped. The cores rotate so forges do not all pile on core 0.
	649	* Nothing is added when the kernel holds the cores, `taskset` is absent, or
	650	* the cell may have every core the machine has (#11657).
	651	*/
	652	export function affinityArgv(
	653	spec: Pick<CellSpec, "cpuQuota" \| "limits">,
	654	kernelHolds: boolean,
	655	rotor: CoreRotor = cellCores,
	656	machine = availableParallelism(),
	657	have: boolean = Bun.which("taskset") !== null,
	658	): string[] {
	659	if (spec.cpuQuota !== true \|\| kernelHolds \|\| !have) return [];
	660	const n = Math.max(1, Math.ceil(spec.limits.cpus));
	661	if (!Number.isFinite(n) \|\| n >= machine) return [];
	662	const first = rotor.next % machine;
	663	rotor.next = (first + n) % machine;
	664	const cores = Array.from({ length: n }, (_, i) => (first + i) % machine);
	665	return ["taskset", "-c", cores.join(",")];
	666	}
	667
	668	/**
	669	* The argv a cell's command runs as: into its cgroup when it has one, then
	670	* pinned to its cores when the kernel does not hold them, then `inner` (bwrap).
	671	*/
	672	export function cellArgv(
	673	inner: readonly string[],
	674	spec: Pick<CellSpec, "cpuQuota" \| "limits">,
	675	cgroup: { readonly dir: string; readonly cpuCapped: boolean } \| undefined,
	676	rotor: CoreRotor = cellCores,
	677	machine = availableParallelism(),
	678	have: boolean = Bun.which("taskset") !== null,
	679	): string[] {
	680	const pinned = [...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have), ...inner];
	681	return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned);
	682	}
	683
635	684	/** Make a cell: its sockets, its proxy, its relays. `close()` undoes it. */
636	685	export function openCell(
637	686	spec: CellSpec,
		@@ -681,10 +730,7 @@ export function openCell(
681	730	const SMALL = String(1 << 20);
682	731
683	732	const cgroup = cellCgroup(spec);
684		const argv = (command: string): string[] => {
685		const inner = bwrapArgv(command);
686		return cgroup === undefined ? inner : enterArgv(join(cgroup.dir, "cgroup.procs"), inner);
687		};
	733	const argv = (command: string): string[] => cellArgv(bwrapArgv(command), spec, cgroup);
688	734	const rlimits = cellRlimits(spec.limits);
689	735	const bwrapArgv = (command: string): string[] => [
690	736	"bwrap",
		@@ -894,15 +940,6 @@ function readMountinfo(): string {
894	940
895	941	/** How often the process table is read, and over how long CPU is averaged. */
896	942	export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15 };
897		/**
898		* How far past its cores a cell may run before the watcher stops it. A
899		* cell is already held to its cores by affinity, so a busy one measures
900		* right at its limit (16.0 to 18.2 of 16 over a window): stopping at the
901		* limit itself failed three checks in four and stopped a 1-core forge at
902		* 1.0. The watcher is the last resort for a runaway, so it acts well past
903		* the limit.
904		*/
905		export const CPU_STOP_SLACK = 1.5;
906	943	/** A cell's process limit: the watch's every tick, and the kernel's `pids.max` where pids is delegated. */
907	944	export const PROCESSES_MAX = 512;
908	945	const CLK_TCK = 100;
		@@ -915,6 +952,9 @@ interface Watched {
915	952	readonly breach: (why: string) => void;
916	953	readonly samples: { at: number; ticks: number }[];
917	954	ticks: number;
	955	/** When the cell began to be watched, and since when its cores have held over the stop line without a break (#11657). */
	956	readonly startedAt: number;
	957	overSince: number \| null;
918	958	readonly key: number;
919	959	readonly label: string \| undefined;
920	960	/** The memory cap, and who holds it: the kernel (a cgroup) or this watch alone. */
		@@ -1097,6 +1137,8 @@ export function watchTree(
1097	1137	breach,
1098	1138	samples: [],
1099	1139	ticks: 0,
	1140	startedAt: Date.now(),
	1141	overSince: null,
1100	1142	key: nextKey++,
1101	1143	label,
1102	1144	cap,
		@@ -1249,6 +1291,36 @@ export function diskBytes(dirs: readonly string[]): number {
1249	1291
1250	1292	const mib = (bytes: number): string => `${Math.round(bytes / 1024 / 1024)} MiB`;
1251	1293
	1294	/** The part of a watched cell its CPU decision reads and keeps. */
	1295	export interface CpuState {
	1296	readonly samples: { at: number; ticks: number }[];
	1297	readonly startedAt: number;
	1298	overSince: number \| null;
	1299	}
	1300
	1301	/**
	1302	* One tick's CPU decision for a watched cell: record the tree's CPU `ticks`
	1303	* at `now`, average the cores over `WATCH.cpuWindowMs`, keep the hold on
	1304	* `state`, and say whether `cpuWatch` (`apps/cpu_rule.ts`) stops it — with the
	1305	* cores and the window it measured — or null to let it run (#11657).
	1306	*/
	1307	export function cpuSample(
	1308	state: CpuState,
	1309	ticks: number,
	1310	cpus: number,
	1311	now: number,
	1312	): { cores: number; seconds: number } \| null {
	1313	state.samples.push({ at: now, ticks });
	1314	while (state.samples.length > 1 && now - (state.samples[1]?.at ?? now) >= WATCH.cpuWindowMs) state.samples.shift();
	1315	const oldest = state.samples[0];
	1316	if (oldest === undefined \|\| now - oldest.at < WATCH.cpuWindowMs) return null;
	1317	const seconds = (now - oldest.at) / 1000;
	1318	const cores = (ticks - oldest.ticks) / CLK_TCK / seconds;
	1319	const verdict = cpuWatch(state.overSince, cores, cpus, now, state.startedAt);
	1320	state.overSince = verdict.overSinceMs;
	1321	return verdict.stop ? { cores, seconds } : null;
	1322	}
	1323
1252	1324	function tick(): void {
1253	1325	if (watched.size === 0) return;
1254	1326	const { rows, children } = processTable();
		@@ -1282,24 +1354,14 @@ function tick(): void {
1282	1354	);
1283	1355	continue;
1284	1356	}
1285		entry.samples.push({ at: now, ticks });
1286		while (
1287		entry.samples.length > 1 &&
1288		now - (entry.samples[1]?.at ?? now) >= WATCH.cpuWindowMs
1289		)
1290		entry.samples.shift();
1291		const oldest = entry.samples[0];
1292		if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
1293		const cores =
1294		(ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
1295		if (cores > limits.cpus * CPU_STOP_SLACK) {
1296		// Read only now, at the stop: no tick pays for naming a process.
1297		const top = busiest(tree, (pid) => rows.get(pid)?.ticks ?? 0);
1298		entry.breach(
1299		cpuBreach(cores, (now - oldest.at) / 1000, limits.cpus, top === undefined ? undefined : { ...top, name: commandName(top.pid) }),
1300		);
1301		continue;
1302		}
	1357	const over = cpuSample(entry, ticks, limits.cpus, now);
	1358	if (over !== null) {
	1359	// Read only now, at the stop: no tick pays for naming a process.
	1360	const top = busiest(tree, (pid) => rows.get(pid)?.ticks ?? 0);
	1361	entry.breach(
	1362	cpuBreach(over.cores, over.seconds, limits.cpus, top === undefined ? undefined : { ...top, name: commandName(top.pid) }),
	1363	);
	1364	continue;
1303	1365	}
1304	1366	entry.ticks++;
1305	1367	if (entry.ticks % WATCH.diskEveryTicks === 0) {
		@@ -1,9 +1,18 @@
1	1	/**
2	2	* When a cell's CPU use is a stop. Pure: plain data in, a boolean out; no clock,
3		* no I/O. No caller yet (#11612, part of #11539).
	3	* no I/O. Called by the cell watch (`apps/contain.ts` tick, #11657).
4	4	*
5	5	* Rule: cores used > CPU_STOP_FACTOR × the cell's cpus, held continuously for at
6	6	* least CPU_STOP_HOLD_MS, and the cell at least CPU_STOP_GRACE_MS old.
	7	*
	8	* The cost of the grace, chosen (#11657): the watch is the only bound on a cell
	9	* the kernel does not hold (no delegated `cpu.max`) and affinity does not pin
	10	* (`taskset` absent). Such a cell — an untrusted check included — may use every
	11	* core of the host for up to CPU_STOP_GRACE_MS (10 minutes) before it can be
	12	* stopped, and must keep it up for CPU_STOP_HOLD_MS. A cell whose cores are the
	13	* whole machine is within its limit at any load. The other choice, a quick stop,
	14	* is what killed a sized forge's builds at 2.1 of 1 core after 31 s; where the
	15	* kernel holds the quota or `taskset` pins the cores this cost does not arise.
7	16	*/
8	17
9	18	/** A cell is over its limit only past this multiple of its cores. */
		@@ -33,3 +42,30 @@ export function cpuStop(i: CpuStopInput): boolean {
33	42	if (nowMs - overSinceMs < CPU_STOP_HOLD_MS) return false;
34	43	return nowMs - startedAtMs >= CPU_STOP_GRACE_MS;
35	44	}
	45
	46	/** The next `overSinceMs`: kept while cores stay over the line, set to now when they cross it, null when they drop under. */
	47	function overSince(prev: number \| null, cores: number, cpus: number, nowMs: number): number \| null {
	48	if (!(cores > CPU_STOP_FACTOR * cpus)) return null;
	49	return prev ?? nowMs;
	50	}
	51
	52	/** What one tick of the watch decides: the hold to keep, and whether to stop. */
	53	export interface CpuVerdict {
	54	readonly overSinceMs: number \| null;
	55	readonly stop: boolean;
	56	}
	57
	58	/**
	59	* One tick of the watch's decision, pure: the new `overSinceMs` and whether the cell is
	60	* stopped. The watch keeps the first and acts on the second (`apps/contain.ts`, #11657).
	61	*/
	62	export function cpuWatch(
	63	prevOverSinceMs: number \| null,
	64	cores: number,
	65	cpus: number,
	66	nowMs: number,
	67	startedAtMs: number,
	68	): CpuVerdict {
	69	const overSinceMs = overSince(prevOverSinceMs, cores, cpus, nowMs);
	70	return { overSinceMs, stop: cpuStop({ cores, cpus, overSinceMs, nowMs, startedAtMs }) };
	71	}
		@@ -834,14 +834,14 @@ describe.skipIf(noCells)("cells: limits stop a step, and say which", () => {
834	834	);
835	835	}, 30_000);
836	836
837		test("over its CPU limit", async () => {
838		// A small share, so a busy machine still gives the loop more than it.
839		const result = await step(`bun -e "for (;;) {}"`, cell({ cpus: 0.02 }));
840		expect(result.ok).toBe(false);
841		if (!result.ok)
842		expect(result.error).toMatch(
843		/^check stopped: it used [\d.]+ cores for \d+s, over its limit of 0\.02; the busiest process is \S+ \(pid \d+, \d+s of CPU\)$/,
844		);
	837	test("over its CPU limit inside its grace: it runs on, and finishes (#11657)", async () => {
	838	// Twice the (shrunk) CPU window busy at far over 0.02 cores: the old 1.5× line
	839	// stopped this at the first full window; the rule waits out the grace first.
	840	const result = await step(
	841	`bun -e "const end = Date.now() + 6000; while (Date.now() < end) {}"`,
	842	cell({ cpus: 0.02 }),
	843	);
	844	expect(result.ok).toBe(true);
845	845	}, 30_000);
846	846
847	847	test("over its disk limit", async () => {
		@@ -1,5 +1,7 @@
1		import { expect, test } from "bun:test";
2		import { CPU_STOP_GRACE_MS, CPU_STOP_HOLD_MS, cpuStop } from "../src/apps/cpu_rule.ts";
	1	import { describe, expect, test } from "bun:test";
	2	import { CPU_STOP_GRACE_MS, CPU_STOP_HOLD_MS, cpuStop, cpuWatch } from "../src/apps/cpu_rule.ts";
	3	import { affinityArgv, cellArgv, cpuSample, type CpuState, WATCH } from "../src/apps/contain.ts";
	4	import { enterArgv } from "../src/apps/cgroup.ts";
3	5
4	6	const M = 60_000;
5	7	const now = 100 * M;
		@@ -27,3 +29,123 @@ test("3.0 of 1 at minute 9 of the cell's life is kept; at minute 10 it is stoppe
27	29	test("an infinite cpus (kernel-capped) is kept", () => {
28	30	expect(cpuStop({ cores: 50, cpus: Infinity, overSinceMs: now - 10 * M, nowMs: now, startedAtMs: old })).toBe(false);
29	31	});
		junipercrew 2d ago outdated Move imports to the top of the file; add a test for the Settings page and one for the watch wiring that runs without cells.
	32
	33	/** Drive the watch's per-tick decision with a clock, as the cell watch does. */
	34	function run(cores: number, cpus: number, forMs: number, startedAt = 0): { stoppedAt: number \| null } {
	35	let over: number \| null = null;
	36	for (let t = 0; t <= forMs; t += 1_000) {
	37	const v = cpuWatch(over, cores, cpus, startedAt + t, startedAt);
	38	over = v.overSinceMs;
	39	if (v.stop) return { stoppedAt: t };
	40	}
	41	return { stoppedAt: null };
	42	}
	43
	44	test("the watch: a 1-CPU cell at 2.1 cores is not stopped after 31 s, or after 5 minutes (#11657)", () => {
	45	expect(run(2.1, 1, 31_000).stoppedAt).toBeNull();
	46	expect(run(2.1, 1, CPU_STOP_HOLD_MS - 1_000).stoppedAt).toBeNull();
	47	});
	48
	49	test("the watch: 3.0 of 1 core held without a break is stopped at the hold, never before", () => {
	50	const { stoppedAt } = run(3, 1, 20 * M);
	51	expect(stoppedAt).toBe(Math.max(CPU_STOP_HOLD_MS, CPU_STOP_GRACE_MS));
	52	});
	53
	54	test("the watch: a dip under the line resets the hold", () => {
	55	const a = cpuWatch(null, 3, 1, 1_000, 0);
	56	expect(a.overSinceMs).toBe(1_000);
	57	expect(cpuWatch(a.overSinceMs, 3, 1, 5_000, 0).overSinceMs).toBe(1_000);
	58	expect(cpuWatch(a.overSinceMs, 1.5, 1, 6_000, 0)).toEqual({ overSinceMs: null, stop: false });
	59	});
	60
	61	const limits = (cpus: number) => ({ memoryMiB: 1, cpus, diskMiB: 1 });
	62
	63	test("a cell the kernel does not hold is pinned to its cores; one it holds, or the whole machine, is not (#11657)", () => {
	64	const spec = { cpuQuota: true, limits: limits(1) };
	65	const rotor = () => ({ next: 0 });
	66	expect(affinityArgv(spec, false, rotor(), 4, true)).toEqual(["taskset", "-c", "0"]);
	67	expect(affinityArgv(spec, true, rotor(), 4, true)).toEqual([]);
	68	expect(affinityArgv(spec, false, rotor(), 4, false)).toEqual([]);
	69	expect(affinityArgv(spec, false, rotor(), 1, true)).toEqual([]);
	70	expect(affinityArgv({ cpuQuota: false, limits: limits(1) }, false, rotor(), 4, true)).toEqual([]);
	71	});
	72
	73	test("pinned cells rotate over the cores, wrapping past the last", () => {
	74	const spec = { cpuQuota: true, limits: limits(2) };
	75	const rotor = { next: 0 };
	76	const cores = Array.from({ length: 3 }, () => affinityArgv(spec, false, rotor, 4, true)[2]);
	77	expect(cores).toEqual(["0,1", "2,3", "0,1"]);
	78	// A 1.5-cpu cell takes two whole cores; one from 3 of 4 wraps to core 0.
	79	expect(affinityArgv({ cpuQuota: true, limits: limits(1.5) }, false, { next: 3 }, 4, true)[2]).toBe("3,0");
	80	});
	81
	82	describe("the argv openCell runs a command as (cellArgv)", () => {
	83	const bwrap = ["bwrap", "--", "true"];
	84	const spec = { cpuQuota: true, limits: limits(1) };
	85
	86	test("no cgroup the kernel caps: the command is pinned inside the cgroup's enter", () => {
	87	const argv = cellArgv(bwrap, spec, { dir: "/sys/fs/cgroup/cell-a", cpuCapped: false }, { next: 2 }, 4, true);
	88	expect(argv).toEqual(enterArgv("/sys/fs/cgroup/cell-a/cgroup.procs", ["taskset", "-c", "2", ...bwrap]));
	89	});
	90
	91	test("a cgroup that caps the cores: entered, not pinned", () => {
	92	const argv = cellArgv(bwrap, spec, { dir: "/sys/fs/cgroup/cell-b", cpuCapped: true }, { next: 0 }, 4, true);
	93	expect(argv).toEqual(enterArgv("/sys/fs/cgroup/cell-b/cgroup.procs", bwrap));
	94	});
	95
	96	test("no cgroup at all: pinned, then bwrap", () => {
	97	expect(cellArgv(bwrap, spec, undefined, { next: 0 }, 4, true)).toEqual(["taskset", "-c", "0", ...bwrap]);
	98	expect(cellArgv(bwrap, spec, undefined, { next: 0 }, 4, false)).toEqual(bwrap);
	99	});
	100	});
	101
	102	describe("the watch's tick feeds the rule (cpuSample)", () => {
	103	const CLK_TCK = 100;
	104
	105	/** A tree burning `cores` from t = 0, sampled each second; the second at which it is stopped, or null. */
	106	function watch(cores: number, cpus: number, forMs: number): { stoppedAt: number \| null; state: CpuState } {
	107	const state: CpuState = { samples: [], startedAt: 0, overSince: null };
	108	for (let t = 0; t <= forMs; t += 1_000) {
	109	const over = cpuSample(state, Math.round((cores * CLK_TCK * t) / 1000), cpus, t);
	110	if (over !== null) {
	111	expect(over.cores).toBeCloseTo(cores, 5);
	112	expect(over.seconds).toBe(WATCH.cpuWindowMs / 1000);
	113	return { stoppedAt: t, state };
	114	}
	115	}
	116	return { stoppedAt: null, state };
	117	}
	118
	119	test("a 1-CPU forge at 2.1 cores is not stopped at 31 s — the old 1.5× line would have (#11657)", () => {
	120	const { stoppedAt, state } = watch(2.1, 1, 31_000);
	121	expect(stoppedAt).toBeNull();
	122	// The window filled at 30 s, and from then the hold is kept on the watched cell.
	123	expect(state.overSince).toBe(WATCH.cpuWindowMs);
	124	});
	125
	126	test("2.1 of 1 core held is stopped once the grace is over, not before", () => {
	127	expect(watch(2.1, 1, CPU_STOP_GRACE_MS - 1_000).stoppedAt).toBeNull();
	128	expect(watch(2.1, 1, 20 * M).stoppedAt).toBe(CPU_STOP_GRACE_MS);
	129	});
	130
	131	test("at twice its cores, not over: never stopped, and no hold kept", () => {
	132	const { stoppedAt, state } = watch(2, 1, 20 * M);
	133	expect(stoppedAt).toBeNull();
	134	expect(state.overSince).toBeNull();
	135	});
	136
	137	test("a dip under the line inside the window restarts the hold", () => {
	138	const state: CpuState = { samples: [], startedAt: -CPU_STOP_GRACE_MS, overSince: null };
	139	let ticks = 0;
	140	let stoppedAt: number \| null = null;
	141	for (let t = 0; t <= 20 * M && stoppedAt === null; t += 1_000) {
	142	// Busy at 3 cores, idle for one window at 4 minutes.
	143	const idle = t > 4 * M && t <= 4 * M + WATCH.cpuWindowMs;
	144	if (t > 0) ticks += idle ? 0 : 3 * CLK_TCK;
	145	if (cpuSample(state, ticks, 1, t) !== null) stoppedAt = t;
	146	}
	147	// Over again from the first full busy window after the dip; stopped a hold later.
	148	expect(stoppedAt).toBeGreaterThan(4 * M + CPU_STOP_HOLD_MS);
	149	expect(stoppedAt).toBeLessThanOrEqual(4 * M + 2 * WATCH.cpuWindowMs + CPU_STOP_HOLD_MS);
	150	});
	151	});