Public: anyone can read this; the crew can change it.
Sentinel: red:01m23vygyffx5hg2f5vm8r74sr was degraded 122 minutes today: trevato/forge: main is red at 1aecf60 — stopped: used 16.1 cores for 30s, over its limit of 16; 5 pull checks wait for … #2162
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/2161-sentinel-red-01m23vygyffx5hg2f5vm8r74sr → main 5 commits fork
5 files
changed +80
−10
scripts/gate-summary.ts +1 −1
| @@ -66,7 +66,7 @@ export type GateSummary = { | ||
| 66 | 66 | files: number; |
| 67 | 67 | /** `FORGE_CHECK_SHARDS` as it was set, if it was. */ |
| 68 | 68 | override: string | undefined; |
| 69 | /** The memory `shardCount` was handed: the least of the machine's and the cgroup's. */ | |
| 69 | /** The memory `shardCount` was handed: the least of the machine's, the cgroup's and the cell's. */ | |
| 70 | 70 | memoryBytes: number; |
| 71 | 71 | /** Room one worker is given (`WORKER_MEMORY_BYTES` in scripts/shards.ts). */ |
| 72 | 72 | workerBytes: number; |
scripts/shards.ts +27 −7
| @@ -10,8 +10,8 @@ | ||
| 10 | 10 | /** |
| 11 | 11 | * No more than this many `bun test` processes at once, whatever the host: |
| 12 | 12 | * a ceiling for a big shared machine, not the check cell's number. The cell |
| 13 | * (16 CPUs, 8 GiB) is bounded by its CPUs and its memory (#2038); the | |
| 14 | * old ceiling of 8 left half its CPUs idle. | |
| 13 | * (16 CPUs, 8 GiB) is bounded by its CPUs and its memory (#2038), its CPUs | |
| 14 | * through {@link workerCpus}; the old ceiling of 8 left half its CPUs idle. | |
| 15 | 15 | */ |
| 16 | 16 | export const SHARDS_MAX = 32; |
| 17 | 17 | |
| @@ -24,6 +24,26 @@ export const SHARDS_MAX = 32; | ||
| 24 | 24 | */ |
| 25 | 25 | export const WORKER_MEMORY_BYTES = 512 * 1024 * 1024; |
| 26 | 26 | |
| 27 | /** | |
| 28 | * The share of a check cell's declared CPUs the workers are sized to. The | |
| 29 | * cell's watchdog (`src/apps/contain.ts`) stops a check whose tree averages | |
| 30 | * more than `check.limits.cpus` cores over 30 s, and a worker is more than | |
| 31 | * one core at its peaks (Bun's own threads, git children, a booted forge): | |
| 32 | * one worker per declared CPU averaged 16.1 of 16 and stopped main (#2161). | |
| 33 | */ | |
| 34 | export const CELL_CPU_SHARE = 3 / 4; | |
| 35 | ||
| 36 | /** | |
| 37 | * The CPUs the gate sizes its workers to: the machine's, held under | |
| 38 | * {@link CELL_CPU_SHARE} of the cell's CPU limit (`FORGE_CELL_CPUS`) when | |
| 39 | * there is one, at least one. `availableParallelism()` sees the host, not | |
| 40 | * the cell: the cell is a watchdog, not a CPU quota the process can read. | |
| 41 | */ | |
| 42 | export function workerCpus(machine: number, declared: number | undefined): number { | |
| 43 | if (declared === undefined || !Number.isFinite(declared) || declared <= 0) return machine; | |
| 44 | return Math.max(1, Math.min(machine, Math.floor(declared * CELL_CPU_SHARE))); | |
| 45 | } | |
| 46 | ||
| 27 | 47 | /** |
| 28 | 48 | * How many shards: one per CPU the process may use, no more than its |
| 29 | 49 | * memory holds at {@link WORKER_MEMORY_BYTES} each, capped at |
| @@ -33,10 +53,10 @@ export const WORKER_MEMORY_BYTES = 512 * 1024 * 1024; | ||
| 33 | 53 | * `FORGE_CHECK_SHARDS` overrides it (still capped at the file count) so a |
| 34 | 54 | * small or busy host can turn it down; that is an explicit operator |
| 35 | 55 | * choice and wins over the cell cap below. `cellCpus` (`FORGE_CELL_CPUS`, the |
| 36 | * CPU limit of the cell the check runs in) caps the count at one less than | |
| 37 | * the limit, since the runner itself needs a core and the cell's guard | |
| 38 | * kills a check that uses more than its limit; unset or not a whole number | |
| 39 | * from 1 is no limit. | |
| 56 | * CPU limit of the cell the check runs in) holds the count to | |
| 57 | * {@link workerCpus} of it, since the runner and each worker's own threads | |
| 58 | * need room and the cell's guard kills a check that uses more than its | |
| 59 | * limit; unset or not a whole number from 1 is no limit. | |
| 40 | 60 | */ |
| 41 | 61 | export function shardCount( |
| 42 | 62 | cpus: number, |
| @@ -47,7 +67,7 @@ export function shardCount( | ||
| 47 | 67 | ): { ok: true; value: number } | { ok: false; error: string } { |
| 48 | 68 | let n = Math.min(SHARDS_MAX, Math.floor(cpus)); |
| 49 | 69 | if (cellCpus !== undefined && /^[1-9][0-9]{0,3}$/.test(cellCpus.trim())) |
| 50 | n = Math.min(n, Number(cellCpus.trim()) - 1); | |
| 70 | n = workerCpus(n, Number(cellCpus.trim())); | |
| 51 | 71 | if (memory !== undefined && Number.isFinite(memory) && memory > 0) |
| 52 | 72 | n = Math.min(n, Math.floor(memory / WORKER_MEMORY_BYTES)); |
| 53 | 73 | if (override !== undefined && override.trim() !== "") { |
scripts/test-shards.ts +7
| @@ -65,6 +65,7 @@ import { cgroupMemory, summaryLines } from "./gate-summary.ts"; | ||
| 65 | 65 | import { GATE_FILE, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, parseImpact } from "../src/checks/impact.ts"; |
| 66 | 66 | import { fastSet, importsOf, duration } from "../src/checks/tiers.ts"; |
| 67 | 67 | import { CHECK_CACHE } from "../src/checks/cache.ts"; |
| 68 | import { parseCheck } from "../src/apps/manifest.ts"; | |
| 68 | 69 | |
| 69 | 70 | const TIMES = new URL("./test-times.json", import.meta.url).pathname; |
| 70 | 71 | // The forge hands the map over in the checkout (`src/checks/impact-store.ts`): |
| @@ -339,11 +340,17 @@ const readLimit = (path: string): string | undefined => { | ||
| 339 | 340 | return undefined; |
| 340 | 341 | } |
| 341 | 342 | }; |
| 343 | // The cell's memory as forge.json declares it: its watchdog stops the | |
| 344 | // check over it, and nothing in /proc or /sys says it. Its CPUs come from | |
| 345 | // FORGE_CELL_CPUS (shardCountFromEnv). | |
| 346 | const cell = parseCheck(readLimit(new URL("../forge.json", import.meta.url).pathname), undefined); | |
| 347 | const declared = cell.ok ? cell.value?.limits : undefined; | |
| 342 | 348 | const cpus = availableParallelism(); |
| 343 | 349 | const shardsOverride = process.env["FORGE_CHECK_SHARDS"]; |
| 344 | 350 | const memory = memoryLimit(totalmem(), [ |
| 345 | 351 | readLimit("/sys/fs/cgroup/memory.max"), |
| 346 | 352 | readLimit("/sys/fs/cgroup/memory/memory.limit_in_bytes"), |
| 353 | declared?.memoryMiB === undefined ? undefined : String(declared.memoryMiB * 1024 * 1024), | |
| 347 | 354 | ]); |
| 348 | 355 | const count = shardCountFromEnv(process.env, cpus, files.length, memory); |
| 349 | 356 | if (!count.ok) { |
test/shards.test.ts +40 −1
| @@ -11,6 +11,7 @@ import { | ||
| 11 | 11 | SECONDS_PER_BYTE_DEFAULT, |
| 12 | 12 | SHARDS_MAX, |
| 13 | 13 | WORKER_MEMORY_BYTES, |
| 14 | CELL_CPU_SHARE, | |
| 14 | 15 | formatTimes, |
| 15 | 16 | heaviestFirst, |
| 16 | 17 | memoryLimit, |
| @@ -25,6 +26,7 @@ import { | ||
| 25 | 26 | splitFiles, |
| 26 | 27 | tally, |
| 27 | 28 | weigher, |
| 29 | workerCpus, | |
| 28 | 30 | type ShardResult, |
| 29 | 31 | } from "../scripts/shards.ts"; |
| 30 | 32 | import { parseCheck } from "../src/apps/manifest.ts"; |
| @@ -224,7 +226,8 @@ describe("shardCount", () => { | ||
| 224 | 226 | |
| 225 | 227 | test("the cell's CPU limit caps the shards below it, whatever the host reports (#2257)", () => { |
| 226 | 228 | expect(shardCount(80, 400, undefined, undefined, "4")).toEqual({ ok: true, value: 3 }); |
| 227 | expect(shardCount(80, 400, undefined, undefined, "16")).toEqual({ ok: true, value: 15 }); | |
| 229 | // workerCpus of the limit, not one less: 15 workers still peak past 16 cores (#2161). | |
| 230 | expect(shardCount(80, 400, undefined, undefined, "16")).toEqual({ ok: true, value: 12 }); | |
| 228 | 231 | expect(shardCount(80, 2, undefined, undefined, "16")).toEqual({ ok: true, value: 2 }); |
| 229 | 232 | expect(shardCount(80, 400, undefined, undefined, "1")).toEqual({ ok: true, value: 1 }); |
| 230 | 233 | expect(shardCount(2, 400, undefined, undefined, "16")).toEqual({ ok: true, value: 2 }); |
| @@ -253,6 +256,42 @@ describe("shardCount", () => { | ||
| 253 | 256 | expect(shardCount(16, 400, "20", 4 * GiB)).toEqual({ ok: true, value: 20 }); |
| 254 | 257 | }); |
| 255 | 258 | |
| 259 | test("the cell's declared CPUs hold the workers under the watchdog's limit (#2161)", () => { | |
| 260 | const GiB = 1024 * 1024 * 1024; | |
| 261 | // A 64-core host, the 16-CPU cell: 12 workers, not 16 (which averaged | |
| 262 | // 16.1 cores and was stopped) and not 32. | |
| 263 | expect(workerCpus(64, 16)).toBe(12); | |
| 264 | expect(shardCount(workerCpus(64, 16), 400, undefined, 8 * GiB)).toEqual({ ok: true, value: 12 }); | |
| 265 | // Never more than the machine has, at least one, and no cell is the machine. | |
| 266 | expect(workerCpus(4, 16)).toBe(4); | |
| 267 | expect(workerCpus(64, 1)).toBe(1); | |
| 268 | expect(workerCpus(64, undefined)).toBe(64); | |
| 269 | expect(workerCpus(64, 0)).toBe(64); | |
| 270 | expect(CELL_CPU_SHARE).toBeLessThan(1); | |
| 271 | }); | |
| 272 | ||
| 273 | test("FORGE_CELL_CPUS holds the count to workerCpus of it; the lowest bound wins (#2161)", () => { | |
| 274 | const GiB = 1024 * 1024 * 1024; | |
| 275 | const cell = { FORGE_CELL_CPUS: "16" }; | |
| 276 | // The 16-CPU cell on a 64-core host: 12, the cell's share, not 15 or 32. | |
| 277 | expect(shardCountFromEnv(cell, 64, 400, 8 * GiB)).toEqual({ ok: true, value: workerCpus(64, 16) }); | |
| 278 | expect(workerCpus(64, 16)).toBe(12); | |
| 279 | // A smaller machine, less memory or fewer files is lower still. | |
| 280 | expect(shardCountFromEnv(cell, 8, 400, 8 * GiB)).toEqual({ ok: true, value: 8 }); | |
| 281 | expect(shardCountFromEnv(cell, 64, 400, 4 * GiB)).toEqual({ ok: true, value: 8 }); | |
| 282 | expect(shardCountFromEnv(cell, 64, 5, 8 * GiB)).toEqual({ ok: true, value: 5 }); | |
| 283 | // The operator's override still wins over the cell. | |
| 284 | expect(shardCountFromEnv({ ...cell, FORGE_CHECK_SHARDS: "14" }, 64, 400, 8 * GiB)).toEqual({ ok: true, value: 14 }); | |
| 285 | }); | |
| 286 | ||
| 287 | test("forge.json's check cell runs fewer workers than it has CPUs", () => { | |
| 288 | const check = parseCheck(readFileSync(new URL("../forge.json", import.meta.url), "utf8"), undefined); | |
| 289 | if (!check.ok || check.value === undefined) throw new Error("forge.json has no check"); | |
| 290 | const cpus = check.value.limits.cpus; | |
| 291 | if (cpus === undefined) throw new Error("forge.json's check declares no cpus"); | |
| 292 | expect(workerCpus(1024, cpus)).toBeLessThan(cpus); | |
| 293 | }); | |
| 294 | ||
| 256 | 295 | test("memoryLimit: the least of the machine and each cgroup limit it can read", () => { |
| 257 | 296 | const GiB = 1024 * 1024 * 1024; |
| 258 | 297 | expect(memoryLimit(256 * GiB, [`${8 * GiB}\n`, undefined])).toBe(8 * GiB); |
test/standards.test.ts +5 −1
| @@ -89,9 +89,13 @@ describe("what the open web expects", () => { | ||
| 89 | 89 | expect(res.status).toBe(200); |
| 90 | 90 | expect(res.headers.get("content-type")).toContain("application/json"); |
| 91 | 91 | const doc = (await res.json()) as { releases: unknown[] }; |
| 92 | expect(doc.releases.length).toBe(releasesFile.releases.length); | |
| 92 | 93 | const read = parseReleases(doc, true); |
| 93 | // A child reads the newest MAX_RELEASES; the record itself keeps every release. | |
| 94 | // A child reads the newest MAX_RELEASES; the record itself keeps every release, | |
| 95 | // and every entry, past those too, is in the child's shape. | |
| 94 | 96 | expect(read!.length).toBe(Math.min(doc.releases.length, MAX_RELEASES)); |
| 97 | const rest = doc.releases.slice(MAX_RELEASES); | |
| 98 | expect(parseReleases({ releases: rest }, true)!.length).toBe(rest.length); | |
| 95 | 99 | expect(read![0]!.name).toBe(releasesFile.releases[0]!.version); |
| 96 | 100 | expect(read![0]!.url).toStartWith(`${base}/releases.json#`); |
| 97 | 101 | }); |
new above
offline