trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

A forge is stopped for CPU only far past its limit for minutes, never at its limit and never in its first ten minutes #11590

torquecrew opened this

Check · Review · Land · Live — No review yet. Next: reviewer, no review yet.

ConversationFiles
Ready to land

torque/forge:torque/11558-a-forge-is-stopped-for-cpu-only-far-past → main 4 commits fork

Review

Sign in to review this change.
4 files changed +108 −7
src/apps/contain.ts +15 −4
@@ -38,6 +38,7 @@ import { dirname, join } from "node:path";
3838 import { portInsteadOfSocket } from "../git/hook-address.ts";
3939 import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
4040 import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
41 import { CPU_RULE, cpuVerdict } from "./cpu_breach.ts";
4142 import {
4243 cgroupHost,
4344 diskDevice,
@@ -856,7 +857,7 @@ function readMountinfo(): string {
856857 // ---- the watch ----
857858
858859 /** How often the process table is read, and over how long CPU is averaged. */
859 export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15 };
860 export const WATCH = { tickMs: 1_000, cpuWindowMs: 30_000, diskEveryTicks: 15, cpuRule: { ...CPU_RULE } };
860861 /** A cell's process limit: the watch's every tick, and the kernel's `pids.max` where pids is delegated. */
861862 export const PROCESSES_MAX = 512;
862863 const CLK_TCK = 100;
@@ -869,6 +870,9 @@ interface Watched {
869870 readonly breach: (why: string) => void;
870871 readonly samples: { at: number; ticks: number }[];
871872 ticks: number;
873 readonly bornAt: number;
874 /** Since when the window has stayed over the CPU rule's line; null when not. */
875 overSince: number | null;
872876 readonly key: number;
873877 readonly label: string | undefined;
874878 /** The memory cap, and who holds it: the kernel (a cgroup) or this watch alone. */
@@ -1051,6 +1055,8 @@ export function watchTree(
10511055 breach,
10521056 samples: [],
10531057 ticks: 0,
1058 bornAt: Date.now(),
1059 overSince: null,
10541060 key: nextKey++,
10551061 label,
10561062 cap,
@@ -1140,7 +1146,7 @@ export function busiest(
11401146
11411147 /**
11421148 * Why a cell was stopped for CPU: cores = Δticks / CLK_TCK / Δseconds over
1143 * the window, against its limit, and the busiest process when there is
1149 * the window, against twice its limit (`cpu_breach.ts`), and the busiest process when there is
11441150 * one (its CPU seconds = ticks / CLK_TCK, over its life).
11451151 */
11461152 export function cpuBreach(
@@ -1149,7 +1155,7 @@ export function cpuBreach(
11491155 cpus: number,
11501156 top: { pid: number; ticks: number; name: string } | undefined,
11511157 ): string {
1152 const why = `used ${cores.toFixed(1)} cores for ${Math.round(seconds)}s, over its limit of ${cpus}`;
1158 const why = `used ${cores.toFixed(1)} cores for ${Math.round(seconds)}s, over twice its limit of ${cpus}`;
11531159 if (top === undefined) return why;
11541160 const spent = Math.round(top.ticks / CLK_TCK);
11551161 return `${why}; the busiest process is ${top.name} (pid ${top.pid}, ${spent}s of CPU)`;
@@ -1246,7 +1252,12 @@ function tick(): void {
12461252 if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
12471253 const cores =
12481254 (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
1249 if (cores > limits.cpus) {
1255 const verdict = cpuVerdict(
1256 { cores, cpus: limits.cpus, overSince: entry.overSince, bornAt: entry.bornAt, now },
1257 WATCH.cpuRule,
1258 );
1259 entry.overSince = verdict.overSince;
1260 if (verdict.stop) {
12501261 // Read only now, at the stop: no tick pays for naming a process.
12511262 const top = busiest(tree, (pid) => rows.get(pid)?.ticks ?? 0);
12521263 entry.breach(
src/apps/cpu_breach.ts added +41
@@ -0,0 +1,41 @@
1 /**
2 * When a cell is stopped for CPU (#11558): only when it held more than twice
3 * its cores for five minutes straight, and never in its first ten minutes.
4 * A cell at, or just over, its limit is kept: a sampled 1.04 of 1 is the
5 * sampling, and a build's burst is not a runaway.
6 *
7 * Pure: plain data in, a verdict out. The watch keeps `overSince` per cell.
8 */
9
10 /** The rule's numbers; `WATCH` in `contain.ts` holds a mutable copy for tests. */
11 export const CPU_RULE = {
12 /** Stop only above this multiple of the cell's cores. */
13 factor: 2,
14 /** … held this long, ms, without a tick at or under it. */
15 holdMs: 5 * 60_000,
16 /** … and never while the cell is younger than this, ms. */
17 graceMs: 10 * 60_000,
18 };
19
20 /** Cores as the stop line prints them: one decimal. The rule compares this, so the line never contradicts it. */
21 export const shownCores = (cores: number): number => Math.round(cores * 10) / 10;
22
23 export interface CpuVerdict {
24 /** When the cell first went over and has stayed over, ms; null when it is not over now. */
25 readonly overSince: number | null;
26 readonly stop: boolean;
27 }
28
29 /**
30 * One tick's verdict. `cores` is the window's average, `cpus` the limit,
31 * `overSince` the last verdict's, `bornAt` and `now` epoch ms.
32 */
33 export function cpuVerdict(
34 input: { cores: number; cpus: number; overSince: number | null; bornAt: number; now: number },
35 rule: typeof CPU_RULE = CPU_RULE,
36 ): CpuVerdict {
37 const { cores, cpus, bornAt, now } = input;
38 if (!(shownCores(cores) > rule.factor * cpus)) return { overSince: null, stop: false };
39 const overSince = input.overSince ?? now;
40 return { overSince, stop: now - overSince >= rule.holdMs && now - bornAt >= rule.graceMs };
41 }
test/contain.test.ts +2 −1
@@ -793,6 +793,7 @@ describe.skipIf(noCells)("cells: limits stop a step, and say which", () => {
793793 WATCH.tickMs = 200;
794794 WATCH.cpuWindowMs = 3_000;
795795 WATCH.diskEveryTicks = 1;
796 WATCH.cpuRule = { factor: 2, holdMs: 1_000, graceMs: 0 };
796797 });
797798
798799 afterAll(async () => {
@@ -840,7 +841,7 @@ describe.skipIf(noCells)("cells: limits stop a step, and say which", () => {
840841 expect(result.ok).toBe(false);
841842 if (!result.ok)
842843 expect(result.error).toMatch(
843 /^check stopped: it used [\d.]+ cores for \d+s, over its limit of 0\.02; the busiest process is \S+ \(pid \d+, \d+s of CPU\)$/,
844 /^check stopped: it used [\d.]+ cores for \d+s, over twice its limit of 0\.02; the busiest process is \S+ \(pid \d+, \d+s of CPU\)$/,
844845 );
845846 }, 30_000);
846847
test/cpu-breach.test.ts +50 −2
@@ -4,6 +4,9 @@
44 */
55 import { describe, expect, test } from "bun:test";
66 import { busiest, cpuBreach } from "../src/apps/contain.ts";
7 import { cpuVerdict, shownCores } from "../src/apps/cpu_breach.ts";
8
9 const MIN = 60_000;
710
811 describe("a cell stopped for CPU", () => {
912 test("the busiest process is the one with the most ticks in the tree", () => {
@@ -20,11 +23,56 @@ describe("a cell stopped for CPU", () => {
2023 // 9_300 ticks over 30 s at 100 ticks a second is 3.1 cores.
2124 const cores = 9_300 / 100 / 30;
2225 expect(cpuBreach(cores, 30.4, 1, { pid: 4242, ticks: 9_150, name: "chrome" })).toBe(
23 "used 3.1 cores for 30s, over its limit of 1; the busiest process is chrome (pid 4242, 92s of CPU)",
26 "used 3.1 cores for 30s, over twice its limit of 1; the busiest process is chrome (pid 4242, 92s of CPU)",
2427 );
2528 });
2629
2730 test("with no process to name, the line is the cores alone", () => {
28 expect(cpuBreach(3.1, 30, 1, undefined)).toBe("used 3.1 cores for 30s, over its limit of 1");
31 expect(cpuBreach(3.1, 30, 1, undefined)).toBe("used 3.1 cores for 30s, over twice its limit of 1");
32 });
33 });
34
35 /** Run a cell at `cores` for `minutes`, from `from` minutes into its life, one tick a second; true when stopped. */
36 function run(cores: number, minutes: number, from = 0, cpus = 1): boolean {
37 let overSince: number | null = null;
38 for (let t = from * MIN; t <= (from + minutes) * MIN; t += 1000) {
39 const v = cpuVerdict({ cores, cpus, overSince, bornAt: 0, now: t });
40 if (v.stop) return true;
41 overSince = v.overSince;
42 }
43 return false;
44 }
45
46 describe("cpu breach rule", () => {
47 test("at its limit: kept", () => expect(run(1.0, 30, 11)).toBe(false));
48 test("1.9x for 10 minutes: kept", () => expect(run(1.9, 10, 11)).toBe(false));
49 test("2.1x for 4 minutes: kept", () => expect(run(2.1, 4, 11)).toBe(false));
50 test("2.1x for 5 minutes: stopped", () => expect(run(2.1, 5, 11)).toBe(true));
51 test("3x at minute 9 of the cell's life: kept", () => expect(run(3, 4, 5)).toBe(false));
52 test("3x since minute 5 is stopped at minute 10, not before", () => {
53 expect(run(3, 4.9, 5)).toBe(false);
54 expect(run(3, 5, 5)).toBe(true);
55 });
56 test("over since birth: the hold ends at minute 5, only the ten-minute grace keeps it", () => {
57 expect(run(3, 9.9, 0)).toBe(false);
58 expect(run(3, 10, 0)).toBe(true);
59 });
60 test("a dip under the line starts the five minutes again", () => {
61 let overSince: number | null = null;
62 const at = (cores: number, now: number) => {
63 const v = cpuVerdict({ cores, cpus: 1, overSince, bornAt: 0, now });
64 overSince = v.overSince;
65 return v.stop;
66 };
67 expect(at(3, 11 * MIN)).toBe(false);
68 expect(at(1, 15 * MIN)).toBe(false);
69 expect(at(3, 16 * MIN)).toBe(false);
70 expect(at(3, 20 * MIN)).toBe(false);
71 expect(at(3, 21 * MIN)).toBe(true);
72 });
73 test("the line's rounding is the rule's: 2.04 prints 2.0 and is kept", () => {
74 expect(shownCores(2.04)).toBe(2);
75 expect(run(2.04, 10, 11)).toBe(false);
76 expect(cpuBreach(2.1, 300, 1, undefined)).toBe("used 2.1 cores for 300s, over twice its limit of 1");
2977 });
3078 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs