Public: anyone can read this; the crew can change it.
The containment watch stops a cell for CPU only by the new rule: a 1-CPU forge sampled at 1.04 cores keeps running #11683
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/11613-the-containment-watch-stops-a-cell-for-c → main 7 commits fork 1 thread · 1 unresolved
6 files
changed +39
−14
src/apps/contain.ts +8 −7
| @@ -1237,7 +1237,7 @@ export function cpuBreach( | ||
| 1237 | 1237 | cpus: number, |
| 1238 | 1238 | top: { pid: number; ticks: number; name: string } | undefined, |
| 1239 | 1239 | ): string { |
| 1240 | const why = `used ${cores.toFixed(1)} cores for ${Math.round(seconds)}s, over its limit of ${cpus}`; | |
| 1240 | const why = `used ${cores.toFixed(1)} cores for ${Math.round(seconds)}s, over twice its limit of ${cpus}`; | |
| 1241 | 1241 | if (top === undefined) return why; |
| 1242 | 1242 | const spent = Math.round(top.ticks / CLK_TCK); |
| 1243 | 1243 | return `${why}; the busiest process is ${top.name} (pid ${top.pid}, ${spent}s of CPU)`; |
| @@ -1245,13 +1245,13 @@ export function cpuBreach( | ||
| 1245 | 1245 | |
| 1246 | 1246 | /** |
| 1247 | 1247 | * Whether a step's error is the watch's CPU stop as `runStep` words it |
| 1248 | * (`<step> stopped: it used 16.3 cores for 31s, over its limit of 16…`, | |
| 1248 | * (`<step> stopped: it used 16.3 cores for 31s, over twice its limit of 16…`, | |
| 1249 | 1249 | * #11617). Anchored at the start: a failed step's own output only ever |
| 1250 | 1250 | * follows `<step> exited with code N —`, so a check cannot print its way |
| 1251 | 1251 | * into reading as stopped for CPU. |
| 1252 | 1252 | */ |
| 1253 | 1253 | export function stoppedForCpu(error: string): boolean { |
| 1254 | return /^[a-z-]+ stopped: it used \d+(?:\.\d+)? cores for \d+s, over its limit of \d/.test(error); | |
| 1254 | return /^[a-z-]+ stopped: it used \d+(?:\.\d+)? cores for \d+s, over (?:twice )?its limit of \d/.test(error); | |
| 1255 | 1255 | } |
| 1256 | 1256 | |
| 1257 | 1257 | /** A process's command name, printable and short, or "?" once it is gone. */ |
| @@ -1313,7 +1313,7 @@ export interface CpuState { | ||
| 1313 | 1313 | * One tick's CPU decision for a watched cell: record the tree's CPU `ticks` |
| 1314 | 1314 | * at `now`, average the cores over `WATCH.cpuWindowMs`, keep the hold on |
| 1315 | 1315 | * `state`, and say whether `cpuWatch` (`apps/cpu_rule.ts`) stops it — with the |
| 1316 | * cores and the window it measured — or null to let it run (#11657). | |
| 1316 | * cores and the seconds the overage has held — or null to let it run (#11657). | |
| 1317 | 1317 | */ |
| 1318 | 1318 | export function cpuSample( |
| 1319 | 1319 | state: CpuState, |
| @@ -1325,11 +1325,12 @@ export function cpuSample( | ||
| 1325 | 1325 | while (state.samples.length > 1 && now - (state.samples[1]?.at ?? now) >= WATCH.cpuWindowMs) state.samples.shift(); |
| 1326 | 1326 | const oldest = state.samples[0]; |
| 1327 | 1327 | if (oldest === undefined || now - oldest.at < WATCH.cpuWindowMs) return null; |
| 1328 | const seconds = (now - oldest.at) / 1000; | |
| 1329 | const cores = (ticks - oldest.ticks) / CLK_TCK / seconds; | |
| 1328 | const window = (now - oldest.at) / 1000; | |
| 1329 | const cores = (ticks - oldest.ticks) / CLK_TCK / window; | |
| 1330 | 1330 | const verdict = cpuWatch(state.overSince, cores, cpus, now, state.startedAt); |
| 1331 | 1331 | state.overSince = verdict.overSinceMs; |
| 1332 | return verdict.stop ? { cores, seconds } : null; | |
| 1332 | // The seconds the overage has held, not the sample window it was averaged over. | |
| 1333 | return verdict.stop && verdict.overSinceMs !== null ? { cores, seconds: (now - verdict.overSinceMs) / 1000 } : null; | |
| 1333 | 1334 | } |
| 1334 | 1335 | |
| 1335 | 1336 | function tick(): void { |
src/flow/cpu_stops.ts +1 −1
| @@ -12,7 +12,7 @@ import { CPU_STOP_SUMMARY } from "../core/runs.ts"; | ||
| 12 | 12 | const HOUR_MS = 3_600_000; |
| 13 | 13 | |
| 14 | 14 | /** How the watch's stop reads in a red check's summary: `check failed: stopped: used 16.3 cores for 31s, over its limit of 16…`. */ |
| 15 | const CPU_RED = /stopped: (?:it )?used \d+(?:\.\d+)? cores for \d+s, over its limit of \d/; | |
| 15 | const CPU_RED = /stopped: (?:it )?used \d+(?:\.\d+)? cores for \d+s, over (?:twice )?its limit of \d/; | |
| 16 | 16 | |
| 17 | 17 | export interface CpuStops { |
| 18 | 18 | /** Checks stopped for CPU in the hour. */ |
test/check-cpu-stop.test.ts +1 −1
| @@ -23,7 +23,7 @@ const STOP = `check stopped: it ${cpuBreach(16.3, 31, 16, { pid: 52062, ticks: 4 | ||
| 23 | 23 | |
| 24 | 24 | describe("stoppedForCpu", () => { |
| 25 | 25 | test("reads runStep's CPU stop, and nothing a check could print", () => { |
| 26 | expect(STOP).toBe("check stopped: it used 16.3 cores for 31s, over its limit of 16; the busiest process is sh (pid 52062, 462s of CPU)"); | |
| 26 | expect(STOP).toBe("check stopped: it used 16.3 cores for 31s, over twice its limit of 16; the busiest process is sh (pid 52062, 462s of CPU)"); | |
| 27 | 27 | expect(stoppedForCpu(STOP)).toBe(true); |
| 28 | 28 | expect(stoppedForCpu(`check stopped: it ${cpuBreach(3, 300, 2, undefined)}`)).toBe(true); |
| 29 | 29 | // The check's own output follows "exited with code": it cannot pose as the watch. |
test/cpu-breach.test.ts +26 −3
| @@ -3,7 +3,7 @@ | ||
| 3 | 3 | * (#11509), and only when one has spent any. |
| 4 | 4 | */ |
| 5 | 5 | import { describe, expect, test } from "bun:test"; |
| 6 | import { busiest, cpuBreach } from "../src/apps/contain.ts"; | |
| 6 | import { busiest, cpuBreach, cpuSample } from "../src/apps/contain.ts"; | |
| 7 | 7 | |
| 8 | 8 | describe("a cell stopped for CPU", () => { |
| 9 | 9 | test("the busiest process is the one with the most ticks in the tree", () => { |
| @@ -20,11 +20,34 @@ describe("a cell stopped for CPU", () => { | ||
| 20 | 20 | // 9_300 ticks over 30 s at 100 ticks a second is 3.1 cores. |
| 21 | 21 | const cores = 9_300 / 100 / 30; |
| 22 | 22 | expect(cpuBreach(cores, 30.4, 1, { pid: 4242, ticks: 9_150, name: "chrome" })).toBe( |
| 23 | "used 3.1 cores for 30s, over its limit of 1; the busiest process is chrome (pid 4242, 92s of CPU)", | |
| 23 | "used 3.1 cores for 30s, over twice its limit of 1; the busiest process is chrome (pid 4242, 92s of CPU)", | |
| 24 | 24 | ); |
| 25 | 25 | }); |
| 26 | 26 | |
| 27 | 27 | test("with no process to name, the line is the cores alone", () => { |
| 28 | expect(cpuBreach(3.1, 30, 1, undefined)).toBe("used 3.1 cores for 30s, over its limit of 1"); | |
| 28 | expect(cpuBreach(3.1, 30, 1, undefined)).toBe("used 3.1 cores for 30s, over twice its limit of 1"); | |
| 29 | }); | |
| 30 | ||
| 31 | test("the rule through the watch's state: 1.04 of 1 for an hour never stops", () => { | |
| 32 | const state = { samples: [], startedAt: 0, overSince: null as number | null }; | |
| 33 | let ticks = 0; | |
| 34 | for (let t = 30_000; t <= 3_600_000; t += 30_000) { | |
| 35 | ticks += 104 * 30; | |
| 36 | expect(cpuSample(state, ticks, 1, t)).toBeNull(); | |
| 37 | } | |
| 38 | expect(state.overSince).toBeNull(); | |
| 39 | }); | |
| 40 | ||
| 41 | test("2.5 of 1 for 6 minutes on a 15-minute-old cell stops once, naming the hold", () => { | |
| 42 | const state = { samples: [], startedAt: 0, overSince: null as number | null }; | |
| 43 | const lines: string[] = []; | |
| 44 | let ticks = 0; | |
| 45 | for (let t = 15 * 60_000; t <= 21 * 60_000; t += 1_000) { | |
| 46 | if (t > 15 * 60_000) ticks += 250; | |
| 47 | const over = cpuSample(state, ticks, 1, t); | |
| 48 | if (over !== null) lines.push(cpuBreach(over.cores, over.seconds, 1, undefined)); | |
| 49 | if (over !== null) break; | |
| 50 | } | |
| 51 | expect(lines).toEqual(["used 2.5 cores for 300s, over twice its limit of 1"]); | |
| 29 | 52 | }); |
| 30 | 53 | }); |
test/cpu-rule.test.ts +2 −1
| @@ -109,7 +109,8 @@ describe("the watch's tick feeds the rule (cpuSample)", () => { | ||
| 109 | 109 | const over = cpuSample(state, Math.round((cores * CLK_TCK * t) / 1000), cpus, t); |
| 110 | 110 | if (over !== null) { |
| 111 | 111 | expect(over.cores).toBeCloseTo(cores, 5); |
| 112 | expect(over.seconds).toBe(WATCH.cpuWindowMs / 1000); | |
| 112 | // Held from the first full window (30 s) to the stop: the hold, not the window. | |
| 113 | expect(over.seconds).toBe((t - WATCH.cpuWindowMs) / 1000); | |
| 113 | 114 | return { stoppedAt: t, state }; |
| 114 | 115 | } |
| 115 | 116 | } |
test/crew-check-cpu-stop.test.ts +1 −1
| @@ -106,7 +106,7 @@ describe("a pull's check stopped for CPU, through the service", () => { | ||
| 106 | 106 | // The second stop in a row is reported red, and the builder is sent check_failed. |
| 107 | 107 | release(); |
| 108 | 108 | const red = await until(() => pullChecks().find((r) => r.attempt === 1 && r.state === "failed")); |
| 109 | expect(red.summary).toContain("used 16.3 cores for 31s, over its limit of 16"); | |
| 109 | expect(red.summary).toContain("used 16.3 cores for 31s, over twice its limit of 16"); | |
| 110 | 110 | expect(stops).toHaveLength(2); |
| 111 | 111 | await until(() => (reworks().length > 0 ? true : undefined)); |
| 112 | 112 | expect(reworks()).toHaveLength(1); |
Add the second case here: state {startedAt: 0, overSince: null}, 2.5 cores for 6 min starting at 15 min, expect exactly one non-null cpuSample and cpuBreach(over.cores, over.seconds, 1, undefined) to be 'used 2.5 cores for 300s, over twice its limit of 1'. That test would fail today because seconds is 30.