Public: anyone can read this; the crew can change it.
A forge that reaches its CPU limit is slowed, not stopped: a crew run on a 1-CPU forge finishes #11547
scout/forge:scout/11539-a-forge-that-reaches-its-cpu-limit-is-sl → main 2 commits fork
4 files
changed +201
−2
docs/features/cpu-throttle.md added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | # A forge at its CPU limit is slowed, not stopped (#11539) | |
| 2 | ||
| 3 | Where the kernel cannot hold a child forge to its cores (`cpu.max` not | |
| 4 | delegated), the watch in `src/apps/contain.ts` holds it with the bucket in | |
| 5 | `src/apps/throttle.ts`: CPU seconds in, CPU seconds out, the tree paused | |
| 6 | (SIGSTOP) while in debt and resumed (SIGCONT) when paid back. Using exactly the | |
| 7 | limit is never over it, and a ten-second burst is free. | |
| 8 | ||
| 9 | The cell is stopped only when the throttle cannot hold it: over twice its cores | |
| 10 | across five minutes, and never before it is ten minutes old. | |
| 11 | ||
| 12 | Not done here: scheduling inside the cell (looks/checks waiting for the crew | |
| 13 | run), resuming an interrupted run without double charge, the owner's home-page | |
| 14 | notice, and the size measurements for the plans. |
src/apps/contain.ts +40 −2
| @@ -38,6 +38,7 @@ import { dirname, join } from "node:path"; | ||
| 38 | 38 | import { portInsteadOfSocket } from "../git/hook-address.ts"; |
| 39 | 39 | import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts"; |
| 40 | 40 | import { BACKGROUND_CPU_WEIGHT } from "./background.ts"; |
| 41 | import { advance, holdFailed, LAST_RESORT_WINDOW_MS, newBucket, type Bucket } from "./throttle.ts"; | |
| 41 | 42 | import { |
| 42 | 43 | cgroupHost, |
| 43 | 44 | diskDevice, |
| @@ -717,7 +718,7 @@ export function openCell( | ||
| 717 | 718 | breach = why; |
| 718 | 719 | log(`stopped: ${why}`); |
| 719 | 720 | stop(why); |
| 720 | }); | |
| 721 | }, spec.cpuQuota === true && cgroup?.cpuCapped !== true ? spec.limits.cpus : undefined); | |
| 721 | 722 | }; |
| 722 | 723 | return { |
| 723 | 724 | argv, |
| @@ -868,6 +869,8 @@ interface Watched { | ||
| 868 | 869 | readonly writable: readonly string[]; |
| 869 | 870 | readonly breach: (why: string) => void; |
| 870 | 871 | readonly samples: { at: number; ticks: number }[]; |
| 872 | /** Held to its cores by slowing, not stopping (`apps/throttle.ts`), when set. */ | |
| 873 | readonly throttle?: { readonly cpus: number; readonly bucket: Bucket; readonly startedAt: number; readonly long: { at: number; seconds: number }[]; lastAt: number; lastTicks: number } | undefined; | |
| 871 | 874 | ticks: number; |
| 872 | 875 | readonly key: number; |
| 873 | 876 | readonly label: string | undefined; |
| @@ -969,6 +972,16 @@ export function killTree(root: number): void { | ||
| 969 | 972 | ).unref(); |
| 970 | 973 | } |
| 971 | 974 | |
| 975 | function signalAll(pids: readonly number[], signal: NodeJS.Signals): void { | |
| 976 | for (const pid of pids) { | |
| 977 | try { | |
| 978 | process.kill(pid, signal); | |
| 979 | } catch { | |
| 980 | // already gone | |
| 981 | } | |
| 982 | } | |
| 983 | } | |
| 984 | ||
| 972 | 985 | /** Send `signal` to every process under `root`, not `root` (bubblewrap) itself. */ |
| 973 | 986 | export function signalTree(root: number, signal: NodeJS.Signals): void { |
| 974 | 987 | for (const pid of descendants(root)) { |
| @@ -1043,8 +1056,14 @@ export function watchTree( | ||
| 1043 | 1056 | writable: readonly string[], |
| 1044 | 1057 | label: string | undefined, |
| 1045 | 1058 | breach: (why: string) => void, |
| 1059 | throttled?: number, | |
| 1046 | 1060 | ): () => void { |
| 1061 | const now = Date.now(); | |
| 1047 | 1062 | const entry: Watched = { |
| 1063 | throttle: | |
| 1064 | throttled === undefined || !Number.isFinite(throttled) | |
| 1065 | ? undefined | |
| 1066 | : { cpus: throttled, bucket: newBucket(throttled), startedAt: now, long: [], lastAt: now, lastTicks: 0 }, | |
| 1048 | 1067 | root, |
| 1049 | 1068 | limits, |
| 1050 | 1069 | writable, |
| @@ -1058,6 +1077,7 @@ export function watchTree( | ||
| 1058 | 1077 | watched.add(entry); |
| 1059 | 1078 | timer ??= setInterval(() => during("cell-watch", tick), WATCH.tickMs); |
| 1060 | 1079 | return () => { |
| 1080 | if (entry.throttle?.bucket.paused === true) signalTree(root, "SIGCONT"); | |
| 1061 | 1081 | watched.delete(entry); |
| 1062 | 1082 | if (watched.size === 0 && timer !== undefined) { |
| 1063 | 1083 | clearInterval(timer); |
| @@ -1236,6 +1256,24 @@ function tick(): void { | ||
| 1236 | 1256 | ); |
| 1237 | 1257 | continue; |
| 1238 | 1258 | } |
| 1259 | const held = entry.throttle; | |
| 1260 | if (held !== undefined) { | |
| 1261 | const used = (ticks - held.lastTicks) / CLK_TCK; | |
| 1262 | const verdict = advance(held.bucket, used, (now - held.lastAt) / 1000, held.cpus); | |
| 1263 | held.lastAt = now; | |
| 1264 | held.lastTicks = ticks; | |
| 1265 | if (verdict === "pause" || (verdict === "hold" && held.bucket.paused)) signalAll(tree, "SIGSTOP"); | |
| 1266 | else if (verdict === "resume") signalAll(tree, "SIGCONT"); | |
| 1267 | held.long.push({ at: now, seconds: ticks / CLK_TCK }); | |
| 1268 | while (held.long.length > 1 && now - (held.long[0]?.at ?? now) > LAST_RESORT_WINDOW_MS + 5_000) held.long.shift(); | |
| 1269 | const cores = holdFailed(held.long, now, held.startedAt, held.cpus); | |
| 1270 | if (cores !== undefined) { | |
| 1271 | held.bucket.paused = false; | |
| 1272 | signalAll(tree, "SIGCONT"); | |
| 1273 | entry.breach(`used ${cores.toFixed(1)} cores for ${Math.round(LAST_RESORT_WINDOW_MS / 1000)}s though slowed, over twice its limit of ${held.cpus}`); | |
| 1274 | continue; | |
| 1275 | } | |
| 1276 | } | |
| 1239 | 1277 | entry.samples.push({ at: now, ticks }); |
| 1240 | 1278 | while ( |
| 1241 | 1279 | entry.samples.length > 1 && |
| @@ -1243,7 +1281,7 @@ function tick(): void { | ||
| 1243 | 1281 | ) |
| 1244 | 1282 | entry.samples.shift(); |
| 1245 | 1283 | const oldest = entry.samples[0]; |
| 1246 | if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) { | |
| 1284 | if (entry.throttle === undefined && oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) { | |
| 1247 | 1285 | const cores = |
| 1248 | 1286 | (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000); |
| 1249 | 1287 | if (cores > limits.cpus) { |
src/apps/throttle.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * CPU held by slowing, not stopping (#11539). Where the kernel cannot hold | |
| 3 | * a cell to its cores (`cpu.max` not delegated), the watch holds it the | |
| 4 | * nearest way there is: a token bucket of CPU seconds. Each tick adds | |
| 5 | * `cpus × seconds` and takes what the tree used; while the bucket is in | |
| 6 | * debt the tree is paused (SIGSTOP) and resumed (SIGCONT) when it is paid | |
| 7 | * back, so a tree that wants N cores runs `cpus / N` of the time and is | |
| 8 | * never killed. Plain data in, a verdict out: the watch sends the signals. | |
| 9 | * | |
| 10 | * Stopping the cell remains for one case: a tree the throttle cannot hold, | |
| 11 | * over twice its cores across five minutes, and only once it is ten | |
| 12 | * minutes old. | |
| 13 | */ | |
| 14 | ||
| 15 | /** A burst of this many seconds at the limit is free (a check's shards, an install). */ | |
| 16 | export const BURST_SECONDS = 10; | |
| 17 | /** Debt is capped at this many seconds of the limit, so a pause is short. */ | |
| 18 | export const DEBT_SECONDS = 2; | |
| 19 | /** The last resort: over `LAST_RESORT_FACTOR` × cores for this long... */ | |
| 20 | export const LAST_RESORT_WINDOW_MS = 5 * 60_000; | |
| 21 | export const LAST_RESORT_FACTOR = 2; | |
| 22 | /** ...and never for a cell younger than this. */ | |
| 23 | export const LAST_RESORT_MIN_AGE_MS = 10 * 60_000; | |
| 24 | ||
| 25 | export interface Bucket { | |
| 26 | /** CPU seconds in hand; negative is debt. */ | |
| 27 | seconds: number; | |
| 28 | paused: boolean; | |
| 29 | } | |
| 30 | ||
| 31 | export const newBucket = (cpus: number): Bucket => ({ seconds: cpus * BURST_SECONDS, paused: false }); | |
| 32 | ||
| 33 | /** | |
| 34 | * One tick: `used` CPU seconds spent over `elapsed` seconds against `cpus`. | |
| 35 | * Returns what to do to the tree: "pause", "resume" or "hold" (no change). | |
| 36 | */ | |
| 37 | export function advance(bucket: Bucket, used: number, elapsed: number, cpus: number): "pause" | "resume" | "hold" { | |
| 38 | if (!Number.isFinite(cpus) || cpus <= 0) return "hold"; | |
| 39 | bucket.seconds = Math.min( | |
| 40 | cpus * BURST_SECONDS, | |
| 41 | Math.max(-cpus * DEBT_SECONDS, bucket.seconds + cpus * elapsed - Math.max(0, used)), | |
| 42 | ); | |
| 43 | if (bucket.seconds < 0 && !bucket.paused) { | |
| 44 | bucket.paused = true; | |
| 45 | return "pause"; | |
| 46 | } | |
| 47 | if (bucket.seconds >= 0 && bucket.paused) { | |
| 48 | bucket.paused = false; | |
| 49 | return "resume"; | |
| 50 | } | |
| 51 | return "hold"; | |
| 52 | } | |
| 53 | ||
| 54 | /** | |
| 55 | * Whether a throttled cell is one to stop: `samples` are (time, cumulative | |
| 56 | * CPU seconds), oldest first, spanning the last-resort window. | |
| 57 | */ | |
| 58 | export function holdFailed( | |
| 59 | samples: readonly { at: number; seconds: number }[], | |
| 60 | now: number, | |
| 61 | startedAt: number, | |
| 62 | cpus: number, | |
| 63 | ): number | undefined { | |
| 64 | if (now - startedAt < LAST_RESORT_MIN_AGE_MS) return undefined; | |
| 65 | const first = samples[0]; | |
| 66 | const last = samples[samples.length - 1]; | |
| 67 | if (first === undefined || last === undefined || last.at - first.at < LAST_RESORT_WINDOW_MS * 0.95) return undefined; | |
| 68 | const cores = (last.seconds - first.seconds) / ((last.at - first.at) / 1000); | |
| 69 | return cores > cpus * LAST_RESORT_FACTOR ? cores : undefined; | |
| 70 | } |
test/cpu-throttle.test.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | /** A cell at its CPU limit is slowed, not stopped (#11539). */ | |
| 2 | import { describe, expect, test } from "bun:test"; | |
| 3 | import { readFileSync } from "node:fs"; | |
| 4 | import { watchTree } from "../src/apps/contain.ts"; | |
| 5 | import { advance, BURST_SECONDS, holdFailed, newBucket } from "../src/apps/throttle.ts"; | |
| 6 | ||
| 7 | describe("the bucket", () => { | |
| 8 | test("a tree that wants 4 cores on 1 runs about a quarter of the time, and is never stopped", () => { | |
| 9 | const bucket = newBucket(1); | |
| 10 | let running = 0; | |
| 11 | let paused = false; | |
| 12 | const total = 600; | |
| 13 | for (let t = 0; t < total; t++) { | |
| 14 | const used = paused ? 0 : 4; | |
| 15 | if (!paused) running++; | |
| 16 | const verdict = advance(bucket, used, 1, 1); | |
| 17 | if (verdict === "pause") paused = true; | |
| 18 | if (verdict === "resume") paused = false; | |
| 19 | } | |
| 20 | expect(running / total).toBeGreaterThan(0.2); | |
| 21 | expect(running / total).toBeLessThan(0.35); | |
| 22 | }); | |
| 23 | ||
| 24 | test("exactly at the limit is never over it", () => { | |
| 25 | const bucket = newBucket(1); | |
| 26 | for (let t = 0; t < 100; t++) expect(advance(bucket, 1, 1, 1)).toBe("hold"); | |
| 27 | }); | |
| 28 | ||
| 29 | test("a burst up to the allowance runs free", () => { | |
| 30 | const bucket = newBucket(2); | |
| 31 | for (let t = 0; t < BURST_SECONDS; t++) expect(advance(bucket, 4, 1, 2)).not.toBe("pause"); | |
| 32 | }); | |
| 33 | }); | |
| 34 | ||
| 35 | describe("the last resort", () => { | |
| 36 | const span = (cores: number, ms: number) => [ | |
| 37 | { at: 0, seconds: 0 }, | |
| 38 | { at: ms, seconds: (cores * ms) / 1000 }, | |
| 39 | ]; | |
| 40 | test("not at or near the limit", () => { | |
| 41 | expect(holdFailed(span(1.0, 300_000), 3_600_000, 0, 1)).toBeUndefined(); | |
| 42 | expect(holdFailed(span(1.9, 300_000), 3_600_000, 0, 1)).toBeUndefined(); | |
| 43 | }); | |
| 44 | test("over twice the limit for five minutes", () => { | |
| 45 | expect(holdFailed(span(2.5, 300_000), 3_600_000, 0, 1)).toBeCloseTo(2.5); | |
| 46 | }); | |
| 47 | test("never for a cell younger than ten minutes", () => { | |
| 48 | expect(holdFailed(span(4, 300_000), 300_000, 0, 1)).toBeUndefined(); | |
| 49 | }); | |
| 50 | test("not on a window shorter than five minutes", () => { | |
| 51 | expect(holdFailed(span(4, 60_000), 3_600_000, 0, 1)).toBeUndefined(); | |
| 52 | }); | |
| 53 | }); | |
| 54 | ||
| 55 | const cpuSeconds = (pid: number): number => { | |
| 56 | const f = readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]!.split(" "); | |
| 57 | return (Number(f[11]) + Number(f[12])) / 100; | |
| 58 | }; | |
| 59 | ||
| 60 | describe("a busy process under the watch", () => { | |
| 61 | test("is slowed to its quota and not killed", async () => { | |
| 62 | const proc = Bun.spawn(["bun", "-e", "for (;;) {}"], { stdout: "ignore", stderr: "ignore" }); | |
| 63 | let why: string | undefined; | |
| 64 | const unwatch = watchTree(proc.pid, { memoryMiB: 4096, cpus: 0.1, diskMiB: 1024 }, { bytes: Infinity, by: "watch" }, [], undefined, (w) => { | |
| 65 | why = w; | |
| 66 | }, 0.1); | |
| 67 | await Bun.sleep(7_000); | |
| 68 | const spent = cpuSeconds(proc.pid); | |
| 69 | unwatch(); | |
| 70 | const alive = proc.exitCode === null; | |
| 71 | proc.kill("SIGKILL"); | |
| 72 | expect(why).toBeUndefined(); | |
| 73 | expect(alive).toBe(true); | |
| 74 | // The burst (1 s) plus 0.1 core for the rest, not 7 s. | |
| 75 | expect(spent).toBeLessThan(3.5); | |
| 76 | }, 20_000); | |
| 77 | }); |
new above
offline