Public: anyone can read this; the crew can change it.
A check cell's memory ceiling is the pod's memory minus the forge's own use, so a check dies before the forge does #12540
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12519-a-check-cell-s-memory-ceiling-is-the-pod → main 4 commits fork
9 files
changed +312
−17
docs/OPERATING.md +1 −1
| @@ -386,7 +386,7 @@ The forge setting `signup` (`src/settings/signup.ts`) is `open`, `invite` or `cl | ||
| 386 | 386 | |
| 387 | 387 | ### A cell's memory ceiling |
| 388 | 388 | |
| 389 | A check, test or app cell asks for memory in `forge.json` (`check.limits.memory`, `app.limits.memory`, MiB) and gets it up to the machine's ceiling. The ceiling is `FORGE_CELL_MEMORY_MAX_MIB` when set; otherwise 75% of the pod's `/sys/fs/cgroup/memory.max`; otherwise 4096 where the host states no limit. A declaration above the ceiling is held to it, and a check's log shows (before its command runs) `warning: forge.json declares memory 8192 MiB; this machine grants at most 4096 (…)`. A cell stopped for memory says whether its limit was the ceiling, `forge.json` or the default. The warning is for checks only: an app declaring `app.limits.memory` above the ceiling is held to it without a line saying so, and the settings page does not show the cut either (#3134). | |
| 389 | A check, test or app cell asks for memory in `forge.json` (`check.limits.memory`, `app.limits.memory`, MiB) and gets it up to the machine's ceiling. The ceiling is `FORGE_CELL_MEMORY_MAX_MIB` when set; otherwise the pod's `/sys/fs/cgroup/memory.max` less the forge's own resident size (counted at no less than 192 MiB) less a 64 MiB reserve, never above 75% of it (`cellMemoryCeilingMiB` in `src/core/capacity.ts`: 384 MiB gives 128, 512 gives 256, 2048 gives 1536); otherwise 4096 where the host states no limit. When the ceiling read from the pod is under 128 MiB, a check is not started: its log says `too small to check here: …`, and it is the machine's, not a red (#12519). A declaration above the ceiling is held to it, and a check's log shows (before its command runs) `warning: forge.json declares memory 8192 MiB; this machine grants at most 4096 (…)`. A cell stopped for memory says whether its limit was the ceiling, `forge.json` or the default. The warning is for checks only: an app declaring `app.limits.memory` above the ceiling is held to it without a line saying so, and the settings page does not show the cut either (#3134). | |
| 390 | 390 | |
| 391 | 391 | ## Check tiers |
| 392 | 392 |
src/apps/check-memory.ts +2 −1
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | import { readFileAt } from "./checkout.ts"; |
| 9 | 9 | import { memoryCeilingSource, OWN_SIZE_SHARE, POD_MEMORY_FILE, POD_MEMORY_SHARE, resolveLimits, type CeilingFrom, type Limits } from "./contain.ts"; |
| 10 | 10 | import { parseCheck } from "./manifest.ts"; |
| 11 | import { CELL_MEMORY_RESERVE_MIB } from "../core/capacity.ts"; | |
| 11 | 12 | |
| 12 | 13 | interface Held { |
| 13 | 14 | readonly limits: Limits; |
| @@ -53,7 +54,7 @@ export function ceilingWords(from: CeilingFrom): string { | ||
| 53 | 54 | : from === "size" |
| 54 | 55 | ? `${Math.round(OWN_SIZE_SHARE * 100)}% of this child forge's own size` |
| 55 | 56 | : from === "pod" |
| 56 | ? `${Math.round(POD_MEMORY_SHARE * 100)}% of the pod's memory.max` | |
| 57 | ? `the pod's memory.max less this forge's resident size and a ${CELL_MEMORY_RESERVE_MIB} MiB reserve, at most ${Math.round(POD_MEMORY_SHARE * 100)}% of it` | |
| 57 | 58 | : "the machine's default ceiling; the pod states no memory limit"; |
| 58 | 59 | } |
| 59 | 60 |
src/apps/contain.ts +38 −8
| @@ -53,6 +53,7 @@ import { during } from "../perf/jobs.ts"; | ||
| 53 | 53 | import { diskBytes, diskWalk, type DiskWalk } from "./disk_walk.ts"; |
| 54 | 54 | import { below, readTable, readTrees, treeOfTable, type ProcessTable, type TreeAsk, type TreeReading } from "./tree_read.ts"; |
| 55 | 55 | import { isSize, SIZES, type Size } from "../spawn/sizes.ts"; |
| 56 | import { CELL_MEMORY_FLOOR_MIB, CELL_MEMORY_RESERVE_MIB, cellMemoryCeilingMiB } from "../core/capacity.ts"; | |
| 56 | 57 | import { cpuWatch } from "./cpu_rule.ts"; |
| 57 | 58 | import { dutyTick, newDuty, resumeNow, type Duty, type DutyLater, type Signaller } from "./cpu_duty.ts"; |
| 58 | 59 | |
| @@ -130,10 +131,18 @@ export const POD_MEMORY_SHARE = 0.75; | ||
| 130 | 131 | */ |
| 131 | 132 | export const FORGE_OWN_MIB = 192; |
| 132 | 133 | |
| 133 | /** A share of the pod's bytes for one cell, never more than the pod less the forge's own room. */ | |
| 134 | export function podShareMiB(bytes: number): number { | |
| 135 | const mib = bytes / 1048576; | |
| 136 | return Math.max(1, Math.floor(Math.min(mib * POD_MEMORY_SHARE, mib - FORGE_OWN_MIB))); | |
| 134 | /** The forge's own resident size now, MiB, never counted under {@link FORGE_OWN_MIB}. */ | |
| 135 | export function forgeResidentMiB(): number { | |
| 136 | return Math.max(FORGE_OWN_MIB, Math.ceil(process.memoryUsage.rss() / 1048576)); | |
| 137 | } | |
| 138 | ||
| 139 | /** | |
| 140 | * A share of the pod's bytes for one cell: the pod less the forge's resident | |
| 141 | * size less the reserve, never above {@link POD_MEMORY_SHARE} of it | |
| 142 | * (`cellMemoryCeilingMiB`, #12519). At least 1, so a cell is never unlimited. | |
| 143 | */ | |
| 144 | export function podShareMiB(bytes: number, forgeMiB = forgeResidentMiB()): number { | |
| 145 | return Math.max(1, cellMemoryCeilingMiB(bytes / 1048576, forgeMiB, POD_MEMORY_SHARE)); | |
| 137 | 146 | } |
| 138 | 147 | |
| 139 | 148 | /** |
| @@ -145,8 +154,27 @@ export function memoryCeiling( | ||
| 145 | 154 | env: string | undefined, |
| 146 | 155 | podFile = POD_MEMORY_FILE, |
| 147 | 156 | size = ownSize(), |
| 157 | forgeMiB?: number, | |
| 148 | 158 | ): number { |
| 149 | return ceiling(env, podFile, size).mib; | |
| 159 | return ceiling(env, podFile, size, forgeMiB).mib; | |
| 160 | } | |
| 161 | ||
| 162 | /** | |
| 163 | * The line that refuses a check before it starts, when the pod's memory less | |
| 164 | * the forge's own leaves a cell under {@link CELL_MEMORY_FLOOR_MIB}; null when | |
| 165 | * it does not. Only a ceiling read from the pod is held to the floor: a | |
| 166 | * `FORGE_CELL_MEMORY_MAX_MIB` an admin set, or a child's size, is theirs. | |
| 167 | */ | |
| 168 | export function tooSmallToCheck( | |
| 169 | env: string | undefined, | |
| 170 | podFile = POD_MEMORY_FILE, | |
| 171 | size = ownSize(), | |
| 172 | forgeMiB?: number, | |
| 173 | ): string | null { | |
| 174 | const held = ceiling(env, podFile, size, forgeMiB); | |
| 175 | return held.from === "pod" && held.mib < CELL_MEMORY_FLOOR_MIB | |
| 176 | ? `too small to check here: the pod's memory less the forge's own and a ${CELL_MEMORY_RESERVE_MIB} MiB reserve leaves a check cell ${held.mib} MiB, under the ${CELL_MEMORY_FLOOR_MIB} MiB a check needs; give the pod more memory, or set FORGE_CELL_MEMORY_MAX_MIB` | |
| 177 | : null; | |
| 150 | 178 | } |
| 151 | 179 | |
| 152 | 180 | /** Where the ceiling comes from: the variable, the pod's `memory.max`, or the fixed default. */ |
| @@ -163,8 +191,9 @@ function ceiling( | ||
| 163 | 191 | env: string | undefined, |
| 164 | 192 | podFile: string, |
| 165 | 193 | size: Size | undefined, |
| 194 | forgeMiB?: number, | |
| 166 | 195 | ): { mib: number; from: CeilingFrom } { |
| 167 | const machine = machineCeiling(env, podFile); | |
| 196 | const machine = machineCeiling(env, podFile, forgeMiB); | |
| 168 | 197 | if (size === undefined) return machine; |
| 169 | 198 | const own = ownShare(SIZES[size].memoryMiB); |
| 170 | 199 | return own < machine.mib ? { mib: own, from: "size" } : machine; |
| @@ -173,13 +202,14 @@ function ceiling( | ||
| 173 | 202 | function machineCeiling( |
| 174 | 203 | env: string | undefined, |
| 175 | 204 | podFile: string, |
| 205 | forgeMiB?: number, | |
| 176 | 206 | ): { mib: number; from: CeilingFrom } { |
| 177 | 207 | const set = positive(env); |
| 178 | 208 | if (set !== undefined) return { mib: set, from: "env" }; |
| 179 | 209 | try { |
| 180 | 210 | const bytes = Number(readFileSync(podFile, "utf8").trim()); |
| 181 | 211 | if (Number.isFinite(bytes) && bytes > 0) |
| 182 | return { mib: podShareMiB(bytes), from: "pod" }; | |
| 212 | return { mib: podShareMiB(bytes, forgeMiB), from: "pod" }; | |
| 183 | 213 | } catch { |
| 184 | 214 | // no cgroup v2 here, or "max" (unlimited): the fixed default below |
| 185 | 215 | } |
| @@ -205,7 +235,7 @@ function ceilingText(from: CeilingFrom | undefined): string { | ||
| 205 | 235 | : from === "size" |
| 206 | 236 | ? `${OWN_SIZE_SHARE * 100}% of this child forge's own size` |
| 207 | 237 | : from === "pod" |
| 208 | ? `${POD_MEMORY_SHARE * 100}% of the pod's memory.max, less the forge's own ${FORGE_OWN_MIB} MiB` | |
| 238 | ? `${POD_MEMORY_SHARE * 100}% of the pod's memory.max, at most the pod less the forge's resident size and a ${CELL_MEMORY_RESERVE_MIB} MiB reserve` | |
| 209 | 239 | : "the 4096 MiB default, the host states no memory limit"; |
| 210 | 240 | } |
| 211 | 241 |
src/checks/pod_room.ts added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | /** | |
| 2 | * Whether this pod has room for a check cell (#12519): the pod's memory.max | |
| 3 | * less the forge's resident size and a reserve (`core/capacity.ts`). Under | |
| 4 | * the floor the check is refused before any step starts, so the forge is | |
| 5 | * never the one the OOM killer takes; the refusal is the machine's | |
| 6 | * (`infrastructure`), never the pull's red. | |
| 7 | */ | |
| 8 | ||
| 9 | import { POD_MEMORY_FILE, tooSmallToCheck } from "../apps/contain.ts"; | |
| 10 | ||
| 11 | /** Where the pod's limit is read and how big the forge is; unset, the real ones. */ | |
| 12 | export interface PodRoom { | |
| 13 | /** FORGE_CELL_MEMORY_MAX_MIB as set; an admin's ceiling is never refused. */ | |
| 14 | readonly env: string | undefined; | |
| 15 | readonly podFile?: string | undefined; | |
| 16 | /** The forge's resident size in MiB; unset, measured now. */ | |
| 17 | readonly forgeMiB?: number | undefined; | |
| 18 | } | |
| 19 | ||
| 20 | export interface TooSmall { | |
| 21 | readonly ok: false; | |
| 22 | readonly error: string; | |
| 23 | readonly infrastructure: true; | |
| 24 | } | |
| 25 | ||
| 26 | /** The refusal, its line already written to the check's log; null when there is room. */ | |
| 27 | export function refuseSmallPod(room: PodRoom, write: (line: string) => void): TooSmall | null { | |
| 28 | const line = tooSmallToCheck(room.env, room.podFile ?? POD_MEMORY_FILE, undefined, room.forgeMiB); | |
| 29 | if (line === null) return null; | |
| 30 | write(line); | |
| 31 | return { ok: false, error: line, infrastructure: true }; | |
| 32 | } |
src/checks/service.ts +14
| @@ -129,6 +129,7 @@ import { | ||
| 129 | 129 | import { summariseCheckFailure } from "./summary.ts"; |
| 130 | 130 | import type { Result } from "../kernel/result.ts"; |
| 131 | 131 | import { placementFromEnv, type Placement } from "./placement.ts"; |
| 132 | import { refuseSmallPod, type PodRoom } from "./pod_room.ts"; | |
| 132 | 133 | import { JOB_VANISHED, runJobStep } from "./job_step.ts"; |
| 133 | 134 | import { |
| 134 | 135 | cellInstall, |
| @@ -193,6 +194,11 @@ export interface ChecksOptions { | ||
| 193 | 194 | readonly installRetryMs?: number | undefined; |
| 194 | 195 | /** Runs the check step itself; tests stand one in for a stop the cell's watch makes (#11617). @default runStep */ |
| 195 | 196 | readonly checkStep?: typeof runStep | undefined; |
| 197 | /** | |
| 198 | * Where the pod's memory.max is read and the forge's size | |
| 199 | * (`checks/pod_room.ts`); tests stand in a small pod. @default the real ones | |
| 200 | */ | |
| 201 | readonly podRoom?: Omit<PodRoom, "env"> | undefined; | |
| 196 | 202 | /** |
| 197 | 203 | * How often the default branch is verified after landing at most |
| 198 | 204 | * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds |
| @@ -1380,6 +1386,14 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 1380 | 1386 | } |
| 1381 | 1387 | : undefined; |
| 1382 | 1388 | try { |
| 1389 | // The pod's memory less the forge's own leaves too little for a cell | |
| 1390 | // (#12519): refused before a step starts, so the forge is never the one | |
| 1391 | // the OOM killer takes. The machine's, not the pull's: no red. A Job's | |
| 1392 | // memory is the cluster's, not this pod's. | |
| 1393 | const small = job === undefined | |
| 1394 | ? refuseSmallPod({ env: process.env["FORGE_CELL_MEMORY_MAX_MIB"], ...options.podRoom }, write) | |
| 1395 | : null; | |
| 1396 | if (small !== null) return small; | |
| 1383 | 1397 | for (const line of limitWarnings(manifest.limits)) write(line); |
| 1384 | 1398 | const cellCpus = resolveLimits("step", manifest.limits).cpus; |
| 1385 | 1399 | const env = childEnv({ |
src/core/capacity.ts +35
| @@ -649,3 +649,38 @@ export function capacityChanges(db: Database, limit = 20): CapacityChange[] { | ||
| 649 | 649 | reason: r.reason, |
| 650 | 650 | })); |
| 651 | 651 | } |
| 652 | ||
| 653 | /** | |
| 654 | * Room the pod keeps beside the forge and one check cell, MiB: the kernel's | |
| 655 | * own pages, the page cache a git checkout needs, a burst in the forge's | |
| 656 | * heap while the check runs (#12519). | |
| 657 | */ | |
| 658 | export const CELL_MEMORY_RESERVE_MIB = 64; | |
| 659 | ||
| 660 | /** | |
| 661 | * Below this a check cell is not started, MiB: bun alone takes about 50 and a | |
| 662 | * test runner as much again. A cell held under it dies on its first install, | |
| 663 | * which reads as a red the pull never earned (#12519). | |
| 664 | */ | |
| 665 | export const CELL_MEMORY_FLOOR_MIB = 128; | |
| 666 | ||
| 667 | /** | |
| 668 | * The most memory one check cell may be given in a pod of `podMiB`, MiB: | |
| 669 | * | |
| 670 | * ceiling = ⌊min(podMiB × share, podMiB − forgeMiB − reserve)⌋, at least 0 | |
| 671 | * | |
| 672 | * where `forgeMiB` is the forge's own resident size. On 384 MiB with the | |
| 673 | * forge at 192: min(288, 384 − 192 − 64) = 128, so the forge and the cell | |
| 674 | * together never reach the pod's limit and the OOM killer takes the check, | |
| 675 | * not the forge (#12092, finding 1). `share` keeps one cell from a big pod's | |
| 676 | * whole memory, as before. | |
| 677 | */ | |
| 678 | export function cellMemoryCeilingMiB( | |
| 679 | podMiB: number, | |
| 680 | forgeMiB: number, | |
| 681 | share = 0.75, | |
| 682 | reserveMiB = CELL_MEMORY_RESERVE_MIB, | |
| 683 | ): number { | |
| 684 | const n = Math.floor(Math.min(podMiB * share, podMiB - forgeMiB - reserveMiB)); | |
| 685 | return Number.isFinite(n) && n > 0 ? n : 0; | |
| 686 | } |
test/check-pod-too-small.test.ts added +102
| @@ -0,0 +1,102 @@ | ||
| 1 | /** | |
| 2 | * A pod too small for a check cell is refused through the checks service | |
| 3 | * before any step runs (#12519): the check step is never called, the log | |
| 4 | * says why, and the run is the machine's (queued once more), never a red | |
| 5 | * the first time. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { writeFileSync } from "node:fs"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 14 | import type { runStep } from "../src/apps/process.ts"; | |
| 15 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 16 | import { isTerminal, listRuns } from "../src/core/runs.ts"; | |
| 17 | import { createUser } from "../src/core/users.ts"; | |
| 18 | import { ok } from "../src/kernel/result.ts"; | |
| 19 | import { readRunLog } from "../src/runs/log.ts"; | |
| 20 | ||
| 21 | process.env["FORGE_CELLS"] = "off"; | |
| 22 | const savedCeiling = process.env["FORGE_CELL_MEMORY_MAX_MIB"]; | |
| 23 | delete process.env["FORGE_CELL_MEMORY_MAX_MIB"]; | |
| 24 | ||
| 25 | let dataDir: string; | |
| 26 | let app: App; | |
| 27 | let checks: ChecksHandle; | |
| 28 | let stepsRun = 0; | |
| 29 | ||
| 30 | const checkStep: typeof runStep = async () => { | |
| 31 | stepsRun++; | |
| 32 | return ok(); | |
| 33 | }; | |
| 34 | ||
| 35 | beforeAll(async () => { | |
| 36 | dataDir = await mkdtemp(join(tmpdir(), "forge-podsmall-")); | |
| 37 | const podFile = join(dataDir, "memory.max"); | |
| 38 | writeFileSync(podFile, String(384 * 1048576)); | |
| 39 | app = createApp({ dataDir, port: 0 }); | |
| 40 | checks = startChecks(app, { | |
| 41 | forgeUrl: "http://forge.test", | |
| 42 | concurrency: 1, | |
| 43 | checkStep, | |
| 44 | // 384 − 230 − 64 = 90 MiB, under the 128 MiB floor. | |
| 45 | podRoom: { podFile, forgeMiB: 230 }, | |
| 46 | }); | |
| 47 | }); | |
| 48 | ||
| 49 | afterAll(async () => { | |
| 50 | await checks.stop(); | |
| 51 | app.close(); | |
| 52 | await rm(dataDir, { recursive: true, force: true }); | |
| 53 | if (savedCeiling === undefined) delete process.env["FORGE_CELL_MEMORY_MAX_MIB"]; | |
| 54 | else process.env["FORGE_CELL_MEMORY_MAX_MIB"] = savedCeiling; | |
| 55 | }); | |
| 56 | ||
| 57 | test("no step runs; the first refusal is queued again, the log names the reason", async () => { | |
| 58 | const u = await createUser(app.db, { | |
| 59 | origin: "seed", | |
| 60 | trusted: true, | |
| 61 | username: "alice", | |
| 62 | email: "[email protected]", | |
| 63 | password: "a-good-passphrase", | |
| 64 | adminIfFirst: true, | |
| 65 | }); | |
| 66 | if (!u.ok) throw new Error(u.error); | |
| 67 | const made = await createRepo(app, u.value, { name: "small", visibility: "public" }); | |
| 68 | if (!made.ok) throw new Error(String(made.error)); | |
| 69 | const repo = made.value; | |
| 70 | const pushed = await commitToBranch(app, u.value, repo, { | |
| 71 | message: "A check the pod has no room for", | |
| 72 | files: [{ path: "forge.json", content: `{"check": {"run": "true", "install": "true"}}\n` }], | |
| 73 | }); | |
| 74 | if (!pushed.ok) throw new Error(JSON.stringify(pushed.error)); | |
| 75 | const oid = pushed.value.commitOid; | |
| 76 | ||
| 77 | const mine = () => | |
| 78 | listRuns(app.db, repo.id, { kind: "check", limit: 50 }) | |
| 79 | .filter((r) => r.commitOid === oid && isTerminal(r.state)) | |
| 80 | .sort((a, b) => a.attempt - b.attempt); | |
| 81 | const deadline = Date.now() + 30_000; | |
| 82 | // A green run means a step ran: stop waiting, the asserts below say so. | |
| 83 | const waiting = () => mine().length < 2 && !mine().some((r) => r.state === "ok"); | |
| 84 | while (Date.now() < deadline && waiting()) { | |
| 85 | await Bun.sleep(50); // real-timer: the checks service clones and queues with real git on its own ticks | |
| 86 | } | |
| 87 | if (waiting()) throw new Error("timed out waiting for a second terminal check run"); | |
| 88 | const runs = mine(); | |
| 89 | expect(runs.length).toBeGreaterThanOrEqual(2); | |
| 90 | ||
| 91 | const [first, second] = runs; | |
| 92 | // The first refusal is the machine's: cancelled and queued once more, not red. | |
| 93 | expect(first?.state).toBe("cancelled"); | |
| 94 | // The second is said as any red is, with the reason. | |
| 95 | expect(second?.state).toBe("failed"); | |
| 96 | for (const run of [first, second]) { | |
| 97 | const lines = readRunLog(dataDir, run!.id).lines; | |
| 98 | expect(lines.some((l) => l.startsWith("too small to check here") && l.includes("90 MiB"))).toBe(true); | |
| 99 | expect(lines.some((l) => l.startsWith("$ "))).toBe(false); | |
| 100 | } | |
| 101 | expect(stepsRun).toBe(0); | |
| 102 | }, 60_000); |
test/memory-ceiling.test.ts +86 −5
| @@ -3,7 +3,9 @@ import { describe, expect, test } from "bun:test"; | ||
| 3 | 3 | import { mkdtempSync, writeFileSync } from "node:fs"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { join } from "node:path"; |
| 6 | import { limitWarnings, memoryCeiling, memorySource, resolveLimits } from "../src/apps/contain.ts"; | |
| 6 | import { limitWarnings, memoryCeiling, memorySource, resolveLimits, tooSmallToCheck } from "../src/apps/contain.ts"; | |
| 7 | import { refuseSmallPod } from "../src/checks/pod_room.ts"; | |
| 8 | import { CELL_MEMORY_FLOOR_MIB, CELL_MEMORY_RESERVE_MIB, cellMemoryCeilingMiB } from "../src/core/capacity.ts"; | |
| 7 | 9 | |
| 8 | 10 | const pod = (text: string) => { |
| 9 | 11 | const f = join(mkdtempSync(join(tmpdir(), "mem-")), "memory.max"); |
| @@ -20,10 +22,64 @@ describe("the memory ceiling is never a silent clamp (#3128)", () => { | ||
| 20 | 22 | }); |
| 21 | 23 | |
| 22 | 24 | test("a small pod leaves the forge its own room: the check dies, not the forge (#12346)", () => { |
| 23 | expect(memoryCeiling(undefined, pod(String(384 * 1048576)))).toBe(192); | |
| 24 | expect(memoryCeiling(undefined, pod(String(512 * 1048576)))).toBe(320); | |
| 25 | expect(memoryCeiling(undefined, pod(String(256 * 1048576)))).toBe(64); | |
| 26 | expect(memoryCeiling(undefined, pod(String(2048 * 1048576)))).toBe(1536); | |
| 25 | // The forge at its measured 192 MiB, so the answer does not hang on this test's own size. | |
| 26 | const at = (mib: number) => memoryCeiling(undefined, pod(String(mib * 1048576)), undefined, 192); | |
| 27 | expect(at(384)).toBe(128); | |
| 28 | expect(at(512)).toBe(256); | |
| 29 | expect(at(256)).toBe(1); | |
| 30 | expect(at(2048)).toBe(1536); | |
| 31 | }); | |
| 32 | }); | |
| 33 | ||
| 34 | describe("a check cell's ceiling is the pod less the forge's resident size less a reserve (#12519)", () => { | |
| 35 | test("the reserve and floor are the stated numbers", () => { | |
| 36 | expect(CELL_MEMORY_RESERVE_MIB).toBe(64); | |
| 37 | expect(CELL_MEMORY_FLOOR_MIB).toBe(128); | |
| 38 | }); | |
| 39 | ||
| 40 | test("384, 512 and 2048 MiB pods, by hand", () => { | |
| 41 | // 384: min(0.75 × 384 = 288, 384 − 192 − 64 = 128) = 128 | |
| 42 | expect(cellMemoryCeilingMiB(384, 192)).toBe(128); | |
| 43 | // the same pod with the forge grown to 230: 384 − 230 − 64 = 90 | |
| 44 | expect(cellMemoryCeilingMiB(384, 230)).toBe(90); | |
| 45 | // 512: min(384, 512 − 192 − 64 = 256) = 256; forge at 300: 148 | |
| 46 | expect(cellMemoryCeilingMiB(512, 192)).toBe(256); | |
| 47 | expect(cellMemoryCeilingMiB(512, 300)).toBe(148); | |
| 48 | // 2048: min(1536, 2048 − 192 − 64 = 1792) = 1536, the share binds; forge at 600: 1384 | |
| 49 | expect(cellMemoryCeilingMiB(2048, 192)).toBe(1536); | |
| 50 | expect(cellMemoryCeilingMiB(2048, 600)).toBe(1384); | |
| 51 | }); | |
| 52 | ||
| 53 | test("the forge and the cell together never reach the pod's limit", () => { | |
| 54 | for (const podMiB of [384, 512, 2048]) | |
| 55 | for (const forge of [150, 192, 250, 300]) | |
| 56 | expect(forge + cellMemoryCeilingMiB(podMiB, forge)).toBeLessThanOrEqual(podMiB - CELL_MEMORY_RESERVE_MIB); | |
| 57 | }); | |
| 58 | ||
| 59 | test("a pod the forge fills leaves 0, never a negative number", () => { | |
| 60 | expect(cellMemoryCeilingMiB(256, 192)).toBe(0); | |
| 61 | expect(cellMemoryCeilingMiB(200, 192)).toBe(0); | |
| 62 | expect(cellMemoryCeilingMiB(Number.NaN, 192)).toBe(0); | |
| 63 | }); | |
| 64 | ||
| 65 | test("under the floor the check is refused before it starts, with the reason", () => { | |
| 66 | const p = pod(String(384 * 1048576)); | |
| 67 | expect(tooSmallToCheck(undefined, p, undefined, 192)).toBeNull(); | |
| 68 | const line = tooSmallToCheck(undefined, p, undefined, 230); | |
| 69 | expect(line).toStartWith("too small to check here"); | |
| 70 | expect(line).toContain("90 MiB"); | |
| 71 | expect(line).toContain("128 MiB"); | |
| 72 | expect(tooSmallToCheck(undefined, pod(String(2048 * 1048576)), undefined, 192)).toBeNull(); | |
| 73 | }); | |
| 74 | ||
| 75 | test("FORGE_CELL_MEMORY_MAX_MIB still overrides: an admin's ceiling is neither recomputed nor refused", () => { | |
| 76 | const p = pod(String(384 * 1048576)); | |
| 77 | expect(memoryCeiling("100", p, undefined, 230)).toBe(100); | |
| 78 | expect(tooSmallToCheck("100", p, undefined, 230)).toBeNull(); | |
| 79 | }); | |
| 80 | ||
| 81 | test("no stated pod limit is never refused", () => { | |
| 82 | expect(tooSmallToCheck(undefined, pod("max\n"), undefined, 10_000)).toBeNull(); | |
| 27 | 83 | }); |
| 28 | 84 | |
| 29 | 85 | test("a declared memory above the ceiling produces a warning; within it, none", () => { |
| @@ -89,3 +145,28 @@ describe("the memory ceiling is never a silent clamp (#3128)", () => { | ||
| 89 | 145 | } |
| 90 | 146 | }); |
| 91 | 147 | }); |
| 148 | ||
| 149 | describe("the check service's refuse-or-go decision (#12519)", () => { | |
| 150 | test("a 384 MiB pod with the forge at 230 is refused as the machine's, its line written once", () => { | |
| 151 | const lines: string[] = []; | |
| 152 | const refused = refuseSmallPod({ env: undefined, podFile: pod(String(384 * 1048576)), forgeMiB: 230 }, (l) => lines.push(l)); | |
| 153 | expect(refused).not.toBeNull(); | |
| 154 | expect(refused?.ok).toBe(false); | |
| 155 | expect(refused?.infrastructure).toBe(true); | |
| 156 | expect(refused?.error).toStartWith("too small to check here"); | |
| 157 | expect(refused?.error).toContain("90 MiB"); | |
| 158 | expect(lines).toEqual([refused?.error ?? ""]); | |
| 159 | }); | |
| 160 | ||
| 161 | test("the same pod with the forge at 192 goes, and writes nothing", () => { | |
| 162 | const lines: string[] = []; | |
| 163 | expect(refuseSmallPod({ env: undefined, podFile: pod(String(384 * 1048576)), forgeMiB: 192 }, (l) => lines.push(l))).toBeNull(); | |
| 164 | expect(lines).toEqual([]); | |
| 165 | }); | |
| 166 | ||
| 167 | test("an admin's FORGE_CELL_MEMORY_MAX_MIB goes, however small the pod", () => { | |
| 168 | const lines: string[] = []; | |
| 169 | expect(refuseSmallPod({ env: "100", podFile: pod(String(384 * 1048576)), forgeMiB: 230 }, (l) => lines.push(l))).toBeNull(); | |
| 170 | expect(lines).toEqual([]); | |
| 171 | }); | |
| 172 | }); |
test/settings-check-memory.test.ts +2 −2
| @@ -100,7 +100,7 @@ test("with no variable set, a cut is blamed on the pod, not on the variable", as | ||
| 100 | 100 | const body = await repoWith("pod", check('{"memory":65536}')); |
| 101 | 101 | expect(body).toContain("forge.json asks 65536 MiB"); |
| 102 | 102 | expect(body).toContain("6144 MiB"); |
| 103 | expect(body).toContain("75% of the pod's memory.max"); | |
| 103 | expect(body).toContain("the pod's memory.max less this forge's resident size and a 64 MiB reserve, at most 75% of it"); | |
| 104 | 104 | expect(body).not.toContain("FORGE_CELL_MEMORY_MAX_MIB"); |
| 105 | 105 | } finally { |
| 106 | 106 | restore(); |
| @@ -121,7 +121,7 @@ test("the ceiling's origin is named from the environment, then the pod file", as | ||
| 121 | 121 | test("each origin has its own words", async () => { |
| 122 | 122 | const { ceilingWords } = await import("../src/apps/check-memory.ts"); |
| 123 | 123 | expect(ceilingWords("env")).toContain("FORGE_CELL_MEMORY_MAX_MIB"); |
| 124 | expect(ceilingWords("pod")).toBe("75% of the pod's memory.max"); | |
| 124 | expect(ceilingWords("pod")).toBe("the pod's memory.max less this forge's resident size and a 64 MiB reserve, at most 75% of it"); | |
| 125 | 125 | expect(ceilingWords("default")).not.toContain("FORGE_CELL_MEMORY_MAX_MIB"); |
| 126 | 126 | }); |
| 127 | 127 |
new above
offline