Public: anyone can read this; the crew can change it.
A machine's memory and CPU ceilings are declared in forge.json and enforced on the crew run #9479
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9469-a-machine-s-memory-and-cpu-ceilings-are → main 1 commit fork 1 thread · 1 unresolved
6 files
changed +232
−12
docs/features/machines.md +13 −1
| @@ -25,6 +25,18 @@ cannot widen its own machine by editing the file on its branch): | ||
| 25 | 25 | role with none runs in `standard`, which is today's sandbox exactly. |
| 26 | 26 | - `standard` and `security` are built in. `security` carries the investigation |
| 27 | 27 | tools, aimed at the forge's own copies only. |
| 28 | - `memory` (whole MiB, 256–65536) and `cpus` (cores, 0.25–64) are the run's | |
| 29 | ceilings, e.g. `"audit": { "memory": 2048, "cpus": 2 }`. They only narrow: | |
| 30 | `cpus` is never above the forge's own `FORGE_CREW_CPUS`, and a machine that | |
| 31 | sets neither (`standard`, `security`) runs exactly as today, uncapped in | |
| 32 | memory. The kernel holds them in the run's cgroup cell | |
| 33 | (`src/crew/run-cell.ts`, `src/apps/cgroup.ts`): past `memory` the run is | |
| 34 | OOM-killed, never the forge. A run with a memory cap enters its cell | |
| 35 | strictly, so it never runs uncapped in a cgroup made for it. | |
| 36 | - Where the host cannot hold a ceiling (no delegated cgroup v2, no cpu | |
| 37 | controller), the run's log says so in one line, e.g. | |
| 38 | `machine: memory 2048 MiB not applied: no cgroup v2 here`; where it can, | |
| 39 | `machine: held to memory 2048 MiB and 2 CPUs`. | |
| 28 | 40 | - An invalid declaration falls back to `standard`; it never stops the crew. |
| 29 | 41 | |
| 30 | 42 | ## What a run gets |
| @@ -34,5 +46,5 @@ line in its brief naming the tools its machine carries, and its `apps` reach. | ||
| 34 | 46 | |
| 35 | 47 | ## Not done yet (split from #9418) |
| 36 | 48 | |
| 37 | Memory and CPU ceilings for a machine wait for the cgroup work (#9460); the tools are declared, not yet installed in an image; the browser | |
| 49 | Disk ceilings are not declared (#9460); the tools are declared, not yet installed in an image; the browser | |
| 38 | 50 | shell (ghostty-web, tmux, WebSocket, session records) is not started. |
src/crew/claude.ts +10 −8
| @@ -13,9 +13,9 @@ | ||
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | 15 | import { measureOf, type Measure } from "../core/measure.ts"; |
| 16 | import { BACKGROUND_CPU_WEIGHT, backgroundArgv, crewCpus } from "../apps/background.ts"; | |
| 17 | import { cgroupHost, openCellCgroup, tryEnterArgv } from "../apps/cgroup.ts"; | |
| 18 | import { join } from "node:path"; | |
| 16 | import { backgroundArgv, crewCpus } from "../apps/background.ts"; | |
| 17 | import { cgroupHost } from "../apps/cgroup.ts"; | |
| 18 | import { openRunCell, type RunLimits } from "./run-cell.ts"; | |
| 19 | 19 | import { watchLabelled } from "../apps/contain.ts"; |
| 20 | 20 | import { childEnv, pump } from "../apps/process.ts"; |
| 21 | 21 | import { spellings, stripForgeDirs } from "../runs/paths.ts"; |
| @@ -32,6 +32,8 @@ export interface ClaudeInput { | ||
| 32 | 32 | readonly resume?: string | undefined; |
| 33 | 33 | readonly env: Record<string, string>; |
| 34 | 34 | readonly sandbox: SandboxPolicy; |
| 35 | /** The run's machine's memory and CPU ceilings (`crew/run-cell.ts`); none is today's cell. */ | |
| 36 | readonly limits?: RunLimits | undefined; | |
| 35 | 37 | /** What `/system` calls this run's process tree (`crew:<name>`); unlabelled when absent. */ |
| 36 | 38 | readonly label?: string | undefined; |
| 37 | 39 | readonly timeoutMs: number; |
| @@ -213,13 +215,13 @@ export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> { | ||
| 213 | 215 | // there is spawned bare, so it fails as itself (ENOENT, caught by the |
| 214 | 216 | // caller as a failed run) and not as `nice`'s exit 127. Where the forge |
| 215 | 217 | // holds a cgroup, the run gets a cell of its own, weighed and held to |
| 216 | // `FORGE_CREW_CPUS` cores, so it no longer shares the forge's leaf. | |
| 218 | // `FORGE_CREW_CPUS` cores, so it no longer shares the forge's leaf; its | |
| 219 | // machine may narrow that cell with a memory cap and fewer cores. | |
| 217 | 220 | const argv = [input.binary, ...claudeArgs(input)]; |
| 218 | 221 | const present = Bun.which(input.binary) !== null; |
| 219 | const cgroup = present | |
| 220 | ? openCellCgroup(undefined, cgroupHost(), { weight: BACKGROUND_CPU_WEIGHT, cpus: crewCpus() }) | |
| 221 | : undefined; | |
| 222 | const contained = cgroup === undefined ? argv : tryEnterArgv(join(cgroup.dir, "cgroup.procs"), argv); | |
| 222 | const cell = present ? openRunCell(input.limits ?? {}, cgroupHost(), crewCpus(), input.log) : undefined; | |
| 223 | const cgroup = cell?.cgroup; | |
| 224 | const contained = cell === undefined ? argv : cell.wrap(argv); | |
| 223 | 225 | const spawn = (): ReturnType<typeof Bun.spawn<"pipe", "pipe", "pipe">> => |
| 224 | 226 | Bun.spawn(present ? backgroundArgv(contained) : argv, { |
| 225 | 227 | cwd: input.cwd, |
src/crew/machines.ts +31 −1
| @@ -11,7 +11,7 @@ | ||
| 11 | 11 | * the default branch's tip, never from the agent's own branch (an agent |
| 12 | 12 | * cannot hand itself a bigger machine): |
| 13 | 13 | * |
| 14 | * "machines": { "audit": { "tools": ["gitleaks"], "reach": [] } }, | |
| 14 | * "machines": { "audit": { "tools": ["gitleaks"], "reach": [], "memory": 2048, "cpus": 2 } }, | |
| 15 | 15 | * "machineFor": { "reviewer": "audit" } |
| 16 | 16 | * |
| 17 | 17 | * `standard` (today's sandbox) and `security` ship built in. Pure: text in, |
| @@ -49,8 +49,16 @@ export interface Machine { | ||
| 49 | 49 | readonly label: string; |
| 50 | 50 | readonly tools: readonly string[]; |
| 51 | 51 | readonly reach: readonly Reach[]; |
| 52 | /** The run's memory ceiling, MiB (`forge.json` `memory`); none is today's uncapped run. */ | |
| 53 | readonly memoryMiB?: number | undefined; | |
| 54 | /** The run's CPU ceiling, cores (`forge.json` `cpus`); never above the forge's own `FORGE_CREW_CPUS`. */ | |
| 55 | readonly cpus?: number | undefined; | |
| 52 | 56 | } |
| 53 | 57 | |
| 58 | /** Bounds on a machine's ceilings: enough to run `bun test`, and no more than a pod holds. */ | |
| 59 | export const MEMORY_MIB = { min: 256, max: 65536 } as const; | |
| 60 | export const CPUS = { min: 0.25, max: 64 } as const; | |
| 61 | ||
| 54 | 62 | /** Today's sandbox, unchanged: the forge and the repository's own app. */ |
| 55 | 63 | export const STANDARD: Machine = { |
| 56 | 64 | name: "standard", |
| @@ -150,6 +158,18 @@ function parseMachine(name: string, value: unknown): Result<Machine, string> { | ||
| 150 | 158 | reach.some((r) => !(REACHES as readonly unknown[]).includes(r)) |
| 151 | 159 | ) |
| 152 | 160 | return invalid(`machines.${name}.reach: pick from ${REACHES.join(", ")} (a machine cannot name a host)`); |
| 161 | const memory = value["memory"]; | |
| 162 | if ( | |
| 163 | memory !== undefined && | |
| 164 | (typeof memory !== "number" || !Number.isInteger(memory) || memory < MEMORY_MIB.min || memory > MEMORY_MIB.max) | |
| 165 | ) | |
| 166 | return invalid(`machines.${name}.memory is whole MiB from ${MEMORY_MIB.min} to ${MEMORY_MIB.max}`); | |
| 167 | const cpus = value["cpus"]; | |
| 168 | if ( | |
| 169 | cpus !== undefined && | |
| 170 | (typeof cpus !== "number" || !Number.isFinite(cpus) || cpus < CPUS.min || cpus > CPUS.max) | |
| 171 | ) | |
| 172 | return invalid(`machines.${name}.cpus is a number of cores from ${CPUS.min} to ${CPUS.max}`); | |
| 153 | 173 | const label = value["label"]; |
| 154 | 174 | if (label !== undefined && (typeof label !== "string" || label.length > 80)) |
| 155 | 175 | return invalid(`machines.${name}.label must be short text`); |
| @@ -158,6 +178,8 @@ function parseMachine(name: string, value: unknown): Result<Machine, string> { | ||
| 158 | 178 | label: typeof label === "string" && label !== "" ? label : name, |
| 159 | 179 | tools: [...new Set(tools as string[])], |
| 160 | 180 | reach: [...new Set(reach as Reach[])], |
| 181 | ...(memory === undefined ? {} : { memoryMiB: memory }), | |
| 182 | ...(cpus === undefined ? {} : { cpus }), | |
| 161 | 183 | }); |
| 162 | 184 | } |
| 163 | 185 | |
| @@ -175,6 +197,14 @@ export function reachableApps(machine: Machine, appUrls: readonly string[]): str | ||
| 175 | 197 | return machine.reach.includes("apps") ? [...appUrls] : []; |
| 176 | 198 | } |
| 177 | 199 | |
| 200 | /** The ceilings a machine asks of its run; none for `standard`. */ | |
| 201 | export function machineLimits(machine: Machine): { memoryMiB?: number; cpus?: number } { | |
| 202 | return { | |
| 203 | ...(machine.memoryMiB === undefined ? {} : { memoryMiB: machine.memoryMiB }), | |
| 204 | ...(machine.cpus === undefined ? {} : { cpus: machine.cpus }), | |
| 205 | }; | |
| 206 | } | |
| 207 | ||
| 178 | 208 | /** The run's environment for its machine: names only, nothing secret. */ |
| 179 | 209 | export function machineEnv(machine: Machine): Record<string, string> { |
| 180 | 210 | return { |
src/crew/run-cell.ts added +93
| @@ -0,0 +1,93 @@ | ||
| 1 | /** | |
| 2 | * A crew run's cell (#9469): the cgroup `runClaude` puts the agent in, held | |
| 3 | * to its machine's ceilings (`crew/machines.ts`). A machine can only narrow | |
| 4 | * it: memory is capped where today it is not, and cores are the lower of the | |
| 5 | * machine's and the forge's own `FORGE_CREW_CPUS`. A machine that declares | |
| 6 | * nothing (`standard`) gets exactly today's cell: no memory cap, the crew's | |
| 7 | * cores. Where the host cannot hold a declared ceiling, the run's log says | |
| 8 | * so in one line rather than the run silently going uncapped. | |
| 9 | */ | |
| 10 | ||
| 11 | import { readFileSync } from "node:fs"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { BACKGROUND_CPU_WEIGHT } from "../apps/background.ts"; | |
| 14 | import { cpuMax, enterArgv, openCellCgroup, tryEnterArgv, type CellCgroup, type CgroupHost } from "../apps/cgroup.ts"; | |
| 15 | ||
| 16 | export interface RunLimits { | |
| 17 | readonly memoryMiB?: number | undefined; | |
| 18 | readonly cpus?: number | undefined; | |
| 19 | } | |
| 20 | ||
| 21 | /** What the run's cell is asked for: the machine's ceilings, never above the forge's own cores. */ | |
| 22 | export function cellPlan(limits: RunLimits, crewCpus: number): { memoryMiB: number | undefined; cpus: number } { | |
| 23 | return { | |
| 24 | memoryMiB: limits.memoryMiB, | |
| 25 | cpus: limits.cpus === undefined ? crewCpus : Math.min(limits.cpus, crewCpus), | |
| 26 | }; | |
| 27 | } | |
| 28 | ||
| 29 | const declared = (limits: RunLimits): string[] => [ | |
| 30 | ...(limits.memoryMiB === undefined ? [] : [`memory ${limits.memoryMiB} MiB`]), | |
| 31 | ...(limits.cpus === undefined ? [] : [`${limits.cpus} CPUs`]), | |
| 32 | ]; | |
| 33 | ||
| 34 | const read = (file: string): string | undefined => { | |
| 35 | try { | |
| 36 | return readFileSync(file, "utf8").trim(); | |
| 37 | } catch { | |
| 38 | return undefined; | |
| 39 | } | |
| 40 | }; | |
| 41 | ||
| 42 | export interface RunCell { | |
| 43 | readonly cgroup: CellCgroup | undefined; | |
| 44 | /** The argv that runs `argv` inside the cell (as it is, without one). */ | |
| 45 | readonly wrap: (argv: readonly string[]) => string[]; | |
| 46 | } | |
| 47 | ||
| 48 | /** | |
| 49 | * Open the run's cell and say, in one log line, which declared ceiling it | |
| 50 | * holds or why the host cannot. Nothing is logged for a machine that | |
| 51 | * declares none. A declared memory cap is entered strictly (the run refuses | |
| 52 | * to start uncapped in a cgroup it was made for); otherwise entering is | |
| 53 | * best effort, as it always was. | |
| 54 | */ | |
| 55 | export function openRunCell( | |
| 56 | limits: RunLimits, | |
| 57 | host: CgroupHost, | |
| 58 | crewCpus: number, | |
| 59 | log: (line: string) => void, | |
| 60 | ): RunCell { | |
| 61 | const wanted = declared(limits); | |
| 62 | const plan = cellPlan(limits, crewCpus); | |
| 63 | const bare: RunCell = { cgroup: undefined, wrap: (argv) => [...argv] }; | |
| 64 | if (!host.ok) { | |
| 65 | if (wanted.length > 0) log(`machine: ${wanted.join(" and ")} not applied: ${host.reason}`); | |
| 66 | return bare; | |
| 67 | } | |
| 68 | const cgroup = openCellCgroup(plan.memoryMiB, host, { weight: BACKGROUND_CPU_WEIGHT, cpus: plan.cpus }); | |
| 69 | if (cgroup === undefined) { | |
| 70 | if (wanted.length > 0) log(`machine: ${wanted.join(" and ")} not applied: the run's cgroup could not be made`); | |
| 71 | return bare; | |
| 72 | } | |
| 73 | if (wanted.length > 0) { | |
| 74 | const held: string[] = []; | |
| 75 | const missed: string[] = []; | |
| 76 | if (limits.memoryMiB !== undefined) held.push(`memory ${limits.memoryMiB} MiB`); | |
| 77 | if (limits.cpus !== undefined) { | |
| 78 | const label = `${plan.cpus} CPUs`; | |
| 79 | (read(join(cgroup.dir, "cpu.max")) === cpuMax(plan.cpus) ? held : missed).push(label); | |
| 80 | } | |
| 81 | log( | |
| 82 | missed.length === 0 | |
| 83 | ? `machine: held to ${held.join(" and ")}` | |
| 84 | : `machine: ${missed.join(" and ")} not applied: no cpu controller delegated` + | |
| 85 | (held.length === 0 ? "" : `; held to ${held.join(" and ")}`), | |
| 86 | ); | |
| 87 | } | |
| 88 | const procs = join(cgroup.dir, "cgroup.procs"); | |
| 89 | return { | |
| 90 | cgroup, | |
| 91 | wrap: (argv) => (limits.memoryMiB === undefined ? tryEnterArgv(procs, argv) : enterArgv(procs, argv)), | |
| 92 | }; | |
| 93 | } |
src/crew/runner.ts +2 −1
| @@ -35,7 +35,7 @@ import { previewEnv } from "../apps/data.ts"; | ||
| 35 | 35 | import { appsRuntime } from "../apps/runtime.ts"; |
| 36 | 36 | import { chooseModel, modelFacts } from "./model_choice.ts"; |
| 37 | 37 | import { ownApp, ownAppUrls } from "./app_hosts.ts"; |
| 38 | import { machineEnv, machineFor, reachableApps } from "./machines.ts"; | |
| 38 | import { machineEnv, machineFor, machineLimits, reachableApps } from "./machines.ts"; | |
| 39 | 39 | import { repoMachines } from "./machines_read.ts"; |
| 40 | 40 | import { authorsInRepoPull } from "./authors_pull.ts"; |
| 41 | 41 | import { crewRole, getAgent, type Agent } from "../core/agents.ts"; |
| @@ -1402,6 +1402,7 @@ async function work( | ||
| 1402 | 1402 | resume, |
| 1403 | 1403 | env, |
| 1404 | 1404 | sandbox: shellPolicy(), |
| 1405 | limits: machineLimits(machine), | |
| 1405 | 1406 | label: `crew:${agent.user.username}`, |
| 1406 | 1407 | timeoutMs: Math.max(deadline - Date.now(), 0), |
| 1407 | 1408 | log: write, |
test/crew-machines.test.ts +83 −1
| @@ -1,8 +1,14 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 1 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { prepare } from "../src/apps/cgroup.ts"; | |
| 6 | import { cellPlan, openRunCell } from "../src/crew/run-cell.ts"; | |
| 2 | 7 | import { |
| 3 | 8 | BUILT_IN, |
| 4 | 9 | machineEnv, |
| 5 | 10 | machineFor, |
| 11 | machineLimits, | |
| 6 | 12 | machineNote, |
| 7 | 13 | NO_CONFIG, |
| 8 | 14 | parseMachines, |
| @@ -86,3 +92,79 @@ describe("machines", () => { | ||
| 86 | 92 | expect(machineNote(SECURITY)).toContain("forge's own copies"); |
| 87 | 93 | }); |
| 88 | 94 | }); |
| 95 | ||
| 96 | describe("a machine's memory and CPU ceilings (#9469)", () => { | |
| 97 | test("memory (whole MiB) and cpus (cores) parse within bounds", () => { | |
| 98 | const c = config({ machines: { big: { memory: 2048, cpus: 1.5 } }, machineFor: { builder: "big" } }); | |
| 99 | const big = machineFor(c, "builder"); | |
| 100 | expect(big.memoryMiB).toBe(2048); | |
| 101 | expect(big.cpus).toBe(1.5); | |
| 102 | expect(machineLimits(big)).toEqual({ memoryMiB: 2048, cpus: 1.5 }); | |
| 103 | }); | |
| 104 | ||
| 105 | test("an out-of-bounds or ill-typed ceiling is refused, so the crew runs in standard", () => { | |
| 106 | for (const bad of [{ memory: 64 }, { memory: 1e6 }, { memory: 512.5 }, { memory: "2G" }, { cpus: 0 }, { cpus: 0.1 }, { cpus: 65 }, { cpus: "2" }]) | |
| 107 | expect(refusal({ machines: { m: bad } })).toMatch(/machines\.m\.(memory|cpus)/); | |
| 108 | }); | |
| 109 | ||
| 110 | test("standard declares no ceiling: today's run exactly", () => { | |
| 111 | expect(machineLimits(STANDARD)).toEqual({}); | |
| 112 | expect(machineLimits(SECURITY)).toEqual({}); | |
| 113 | expect(cellPlan({}, 4)).toEqual({ memoryMiB: undefined, cpus: 4 }); | |
| 114 | }); | |
| 115 | ||
| 116 | test("a machine only narrows the forge's own cores", () => { | |
| 117 | expect(cellPlan({ cpus: 2 }, 4).cpus).toBe(2); | |
| 118 | expect(cellPlan({ cpus: 16 }, 4).cpus).toBe(4); | |
| 119 | }); | |
| 120 | }); | |
| 121 | ||
| 122 | describe("a crew run's cell holds its machine's ceilings", () => { | |
| 123 | const root = mkdtempSync(join(tmpdir(), "run-cell-")); | |
| 124 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 125 | let n = 0; | |
| 126 | const host = () => { | |
| 127 | const dir = join(root, `h${n++}`); | |
| 128 | mkdirSync(dir); | |
| 129 | writeFileSync(join(dir, "cgroup.controllers"), "cpu io memory\n"); | |
| 130 | writeFileSync(join(dir, "cgroup.procs"), ""); | |
| 131 | writeFileSync(join(dir, "cgroup.subtree_control"), ""); | |
| 132 | return prepare({ ok: true, root: dir }); | |
| 133 | }; | |
| 134 | ||
| 135 | test("declared ceilings land in the cell's cgroup, the run enters it strictly, and the log says so", () => { | |
| 136 | const lines: string[] = []; | |
| 137 | const cell = openRunCell({ memoryMiB: 1024, cpus: 2 }, host(), 4, (l) => lines.push(l)); | |
| 138 | const dir = cell.cgroup!.dir; | |
| 139 | expect(readFileSync(join(dir, "memory.max"), "utf8")).toBe(String(1024 * 1024 * 1024)); | |
| 140 | expect(readFileSync(join(dir, "cpu.max"), "utf8")).toBe("200000 100000"); | |
| 141 | const argv = cell.wrap(["claude", "-p"]); | |
| 142 | expect(argv).toContain(join(dir, "cgroup.procs")); | |
| 143 | expect(argv.slice(-2)).toEqual(["claude", "-p"]); | |
| 144 | // Strict: a run made for a cap refuses to start outside it. | |
| 145 | expect(argv[2]).toContain("exit 125"); | |
| 146 | expect(lines).toEqual(["machine: held to memory 1024 MiB and 2 CPUs"]); | |
| 147 | cell.cgroup!.remove(); | |
| 148 | }); | |
| 149 | ||
| 150 | test("standard's cell is today's: no memory cap, the crew's cores, best-effort entry, nothing logged", () => { | |
| 151 | const lines: string[] = []; | |
| 152 | const cell = openRunCell({}, host(), 4, (l) => lines.push(l)); | |
| 153 | expect(existsSync(join(cell.cgroup!.dir, "memory.max"))).toBe(false); | |
| 154 | expect(readFileSync(join(cell.cgroup!.dir, "cpu.max"), "utf8")).toBe("400000 100000"); | |
| 155 | expect(cell.wrap(["claude"])[2]).not.toContain("exit 125"); | |
| 156 | expect(lines).toEqual([]); | |
| 157 | cell.cgroup!.remove(); | |
| 158 | }); | |
| 159 | ||
| 160 | test("a host that cannot hold a ceiling says so in one line, and the run goes as before", () => { | |
| 161 | const lines: string[] = []; | |
| 162 | const cell = openRunCell({ memoryMiB: 512 }, { ok: false, reason: "no cgroup v2 here" }, 4, (l) => lines.push(l)); | |
| 163 | expect(cell.cgroup).toBeUndefined(); | |
| 164 | expect(cell.wrap(["claude"])).toEqual(["claude"]); | |
| 165 | expect(lines).toEqual(["machine: memory 512 MiB not applied: no cgroup v2 here"]); | |
| 166 | const quiet: string[] = []; | |
| 167 | openRunCell({}, { ok: false, reason: "no cgroup v2 here" }, 4, (l) => quiet.push(l)); | |
| 168 | expect(quiet).toEqual([]); | |
| 169 | }); | |
| 170 | }); |
Memory is reported held without reading memory.max back, unlike cpus. Verify it the same way so a missing memory controller is logged as not applied.