Public: anyone can read this; the crew can change it.

A forge that reaches its CPU limit is slowed, not stopped: a crew run on a 1-CPU forge finishes #11547

Parked opened this in answer to
Closed without merging

scout/forge:scout/11539-a-forge-that-reaches-its-cpu-limit-is-sl → main 2 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
4 files changed +201 −2
docs/features/cpu-throttle.md added +14
@@ -0,0 +1,14 @@
1 # A forge at its CPU limit is slowed, not stopped (#11539)
2
3 Where the kernel cannot hold a child forge to its cores (`cpu.max` not
4 delegated), the watch in `src/apps/contain.ts` holds it with the bucket in
5 `src/apps/throttle.ts`: CPU seconds in, CPU seconds out, the tree paused
6 (SIGSTOP) while in debt and resumed (SIGCONT) when paid back. Using exactly the
7 limit is never over it, and a ten-second burst is free.
8
9 The cell is stopped only when the throttle cannot hold it: over twice its cores
10 across five minutes, and never before it is ten minutes old.
11
12 Not done here: scheduling inside the cell (looks/checks waiting for the crew
13 run), resuming an interrupted run without double charge, the owner's home-page
14 notice, and the size measurements for the plans.
src/apps/contain.ts +40 −2
@@ -38,6 +38,7 @@ import { dirname, join } from "node:path";
3838 import { portInsteadOfSocket } from "../git/hook-address.ts";
3939 import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
4040 import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
41 import { advance, holdFailed, LAST_RESORT_WINDOW_MS, newBucket, type Bucket } from "./throttle.ts";
4142 import {
4243 cgroupHost,
4344 diskDevice,
@@ -717,7 +718,7 @@ export function openCell(
717718 breach = why;
718719 log(`stopped: ${why}`);
719720 stop(why);
720 });
721 }, spec.cpuQuota === true && cgroup?.cpuCapped !== true ? spec.limits.cpus : undefined);
721722 };
722723 return {
723724 argv,
@@ -868,6 +869,8 @@ interface Watched {
868869 readonly writable: readonly string[];
869870 readonly breach: (why: string) => void;
870871 readonly samples: { at: number; ticks: number }[];
872 /** Held to its cores by slowing, not stopping (`apps/throttle.ts`), when set. */
873 readonly throttle?: { readonly cpus: number; readonly bucket: Bucket; readonly startedAt: number; readonly long: { at: number; seconds: number }[]; lastAt: number; lastTicks: number } | undefined;
871874 ticks: number;
872875 readonly key: number;
873876 readonly label: string | undefined;
@@ -969,6 +972,16 @@ export function killTree(root: number): void {
969972 ).unref();
970973 }
971974
975 function signalAll(pids: readonly number[], signal: NodeJS.Signals): void {
976 for (const pid of pids) {
977 try {
978 process.kill(pid, signal);
979 } catch {
980 // already gone
981 }
982 }
983 }
984
972985 /** Send `signal` to every process under `root`, not `root` (bubblewrap) itself. */
973986 export function signalTree(root: number, signal: NodeJS.Signals): void {
974987 for (const pid of descendants(root)) {
@@ -1043,8 +1056,14 @@ export function watchTree(
10431056 writable: readonly string[],
10441057 label: string | undefined,
10451058 breach: (why: string) => void,
1059 throttled?: number,
10461060 ): () => void {
1061 const now = Date.now();
10471062 const entry: Watched = {
1063 throttle:
1064 throttled === undefined || !Number.isFinite(throttled)
1065 ? undefined
1066 : { cpus: throttled, bucket: newBucket(throttled), startedAt: now, long: [], lastAt: now, lastTicks: 0 },
10481067 root,
10491068 limits,
10501069 writable,
@@ -1058,6 +1077,7 @@ export function watchTree(
10581077 watched.add(entry);
10591078 timer ??= setInterval(() => during("cell-watch", tick), WATCH.tickMs);
10601079 return () => {
1080 if (entry.throttle?.bucket.paused === true) signalTree(root, "SIGCONT");
10611081 watched.delete(entry);
10621082 if (watched.size === 0 && timer !== undefined) {
10631083 clearInterval(timer);
@@ -1236,6 +1256,24 @@ function tick(): void {
12361256 );
12371257 continue;
12381258 }
1259 const held = entry.throttle;
1260 if (held !== undefined) {
1261 const used = (ticks - held.lastTicks) / CLK_TCK;
1262 const verdict = advance(held.bucket, used, (now - held.lastAt) / 1000, held.cpus);
1263 held.lastAt = now;
1264 held.lastTicks = ticks;
1265 if (verdict === "pause" || (verdict === "hold" && held.bucket.paused)) signalAll(tree, "SIGSTOP");
1266 else if (verdict === "resume") signalAll(tree, "SIGCONT");
1267 held.long.push({ at: now, seconds: ticks / CLK_TCK });
1268 while (held.long.length > 1 && now - (held.long[0]?.at ?? now) > LAST_RESORT_WINDOW_MS + 5_000) held.long.shift();
1269 const cores = holdFailed(held.long, now, held.startedAt, held.cpus);
1270 if (cores !== undefined) {
1271 held.bucket.paused = false;
1272 signalAll(tree, "SIGCONT");
1273 entry.breach(`used ${cores.toFixed(1)} cores for ${Math.round(LAST_RESORT_WINDOW_MS / 1000)}s though slowed, over twice its limit of ${held.cpus}`);
1274 continue;
1275 }
1276 }
12391277 entry.samples.push({ at: now, ticks });
12401278 while (
12411279 entry.samples.length > 1 &&
@@ -1243,7 +1281,7 @@ function tick(): void {
12431281 )
12441282 entry.samples.shift();
12451283 const oldest = entry.samples[0];
1246 if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
1284 if (entry.throttle === undefined && oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
12471285 const cores =
12481286 (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
12491287 if (cores > limits.cpus) {
src/apps/throttle.ts added +70
@@ -0,0 +1,70 @@
1 /**
2 * CPU held by slowing, not stopping (#11539). Where the kernel cannot hold
3 * a cell to its cores (`cpu.max` not delegated), the watch holds it the
4 * nearest way there is: a token bucket of CPU seconds. Each tick adds
5 * `cpus × seconds` and takes what the tree used; while the bucket is in
6 * debt the tree is paused (SIGSTOP) and resumed (SIGCONT) when it is paid
7 * back, so a tree that wants N cores runs `cpus / N` of the time and is
8 * never killed. Plain data in, a verdict out: the watch sends the signals.
9 *
10 * Stopping the cell remains for one case: a tree the throttle cannot hold,
11 * over twice its cores across five minutes, and only once it is ten
12 * minutes old.
13 */
14
15 /** A burst of this many seconds at the limit is free (a check's shards, an install). */
16 export const BURST_SECONDS = 10;
17 /** Debt is capped at this many seconds of the limit, so a pause is short. */
18 export const DEBT_SECONDS = 2;
19 /** The last resort: over `LAST_RESORT_FACTOR` × cores for this long... */
20 export const LAST_RESORT_WINDOW_MS = 5 * 60_000;
21 export const LAST_RESORT_FACTOR = 2;
22 /** ...and never for a cell younger than this. */
23 export const LAST_RESORT_MIN_AGE_MS = 10 * 60_000;
24
25 export interface Bucket {
26 /** CPU seconds in hand; negative is debt. */
27 seconds: number;
28 paused: boolean;
29 }
30
31 export const newBucket = (cpus: number): Bucket => ({ seconds: cpus * BURST_SECONDS, paused: false });
32
33 /**
34 * One tick: `used` CPU seconds spent over `elapsed` seconds against `cpus`.
35 * Returns what to do to the tree: "pause", "resume" or "hold" (no change).
36 */
37 export function advance(bucket: Bucket, used: number, elapsed: number, cpus: number): "pause" | "resume" | "hold" {
38 if (!Number.isFinite(cpus) || cpus <= 0) return "hold";
39 bucket.seconds = Math.min(
40 cpus * BURST_SECONDS,
41 Math.max(-cpus * DEBT_SECONDS, bucket.seconds + cpus * elapsed - Math.max(0, used)),
42 );
43 if (bucket.seconds < 0 && !bucket.paused) {
44 bucket.paused = true;
45 return "pause";
46 }
47 if (bucket.seconds >= 0 && bucket.paused) {
48 bucket.paused = false;
49 return "resume";
50 }
51 return "hold";
52 }
53
54 /**
55 * Whether a throttled cell is one to stop: `samples` are (time, cumulative
56 * CPU seconds), oldest first, spanning the last-resort window.
57 */
58 export function holdFailed(
59 samples: readonly { at: number; seconds: number }[],
60 now: number,
61 startedAt: number,
62 cpus: number,
63 ): number | undefined {
64 if (now - startedAt < LAST_RESORT_MIN_AGE_MS) return undefined;
65 const first = samples[0];
66 const last = samples[samples.length - 1];
67 if (first === undefined || last === undefined || last.at - first.at < LAST_RESORT_WINDOW_MS * 0.95) return undefined;
68 const cores = (last.seconds - first.seconds) / ((last.at - first.at) / 1000);
69 return cores > cpus * LAST_RESORT_FACTOR ? cores : undefined;
70 }
test/cpu-throttle.test.ts added +77
@@ -0,0 +1,77 @@
1 /** A cell at its CPU limit is slowed, not stopped (#11539). */
2 import { describe, expect, test } from "bun:test";
3 import { readFileSync } from "node:fs";
4 import { watchTree } from "../src/apps/contain.ts";
5 import { advance, BURST_SECONDS, holdFailed, newBucket } from "../src/apps/throttle.ts";
6
7 describe("the bucket", () => {
8 test("a tree that wants 4 cores on 1 runs about a quarter of the time, and is never stopped", () => {
9 const bucket = newBucket(1);
10 let running = 0;
11 let paused = false;
12 const total = 600;
13 for (let t = 0; t < total; t++) {
14 const used = paused ? 0 : 4;
15 if (!paused) running++;
16 const verdict = advance(bucket, used, 1, 1);
17 if (verdict === "pause") paused = true;
18 if (verdict === "resume") paused = false;
19 }
20 expect(running / total).toBeGreaterThan(0.2);
21 expect(running / total).toBeLessThan(0.35);
22 });
23
24 test("exactly at the limit is never over it", () => {
25 const bucket = newBucket(1);
26 for (let t = 0; t < 100; t++) expect(advance(bucket, 1, 1, 1)).toBe("hold");
27 });
28
29 test("a burst up to the allowance runs free", () => {
30 const bucket = newBucket(2);
31 for (let t = 0; t < BURST_SECONDS; t++) expect(advance(bucket, 4, 1, 2)).not.toBe("pause");
32 });
33 });
34
35 describe("the last resort", () => {
36 const span = (cores: number, ms: number) => [
37 { at: 0, seconds: 0 },
38 { at: ms, seconds: (cores * ms) / 1000 },
39 ];
40 test("not at or near the limit", () => {
41 expect(holdFailed(span(1.0, 300_000), 3_600_000, 0, 1)).toBeUndefined();
42 expect(holdFailed(span(1.9, 300_000), 3_600_000, 0, 1)).toBeUndefined();
43 });
44 test("over twice the limit for five minutes", () => {
45 expect(holdFailed(span(2.5, 300_000), 3_600_000, 0, 1)).toBeCloseTo(2.5);
46 });
47 test("never for a cell younger than ten minutes", () => {
48 expect(holdFailed(span(4, 300_000), 300_000, 0, 1)).toBeUndefined();
49 });
50 test("not on a window shorter than five minutes", () => {
51 expect(holdFailed(span(4, 60_000), 3_600_000, 0, 1)).toBeUndefined();
52 });
53 });
54
55 const cpuSeconds = (pid: number): number => {
56 const f = readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]!.split(" ");
57 return (Number(f[11]) + Number(f[12])) / 100;
58 };
59
60 describe("a busy process under the watch", () => {
61 test("is slowed to its quota and not killed", async () => {
62 const proc = Bun.spawn(["bun", "-e", "for (;;) {}"], { stdout: "ignore", stderr: "ignore" });
63 let why: string | undefined;
64 const unwatch = watchTree(proc.pid, { memoryMiB: 4096, cpus: 0.1, diskMiB: 1024 }, { bytes: Infinity, by: "watch" }, [], undefined, (w) => {
65 why = w;
66 }, 0.1);
67 await Bun.sleep(7_000);
68 const spent = cpuSeconds(proc.pid);
69 unwatch();
70 const alive = proc.exitCode === null;
71 proc.kill("SIGKILL");
72 expect(why).toBeUndefined();
73 expect(alive).toBe(true);
74 // The burst (1 s) plus 0.1 core for the rest, not 7 s.
75 expect(spent).toBeLessThan(3.5);
76 }, 20_000);
77 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs