Public: anyone can read this; the crew can change it.
Sentinel: red:01m23vygyffx5hg2f5vm8r74sr is past its budget: trevato/forge: main is red at c9aa7d6 — stopped: used 4193 MiB of memory, over its 4096 MiB limit; 1 pull check wait fo… #3111
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3105-sentinel-red-01m23vygyffx5hg2f5vm8r74sr → main 1 commit fork
2 files
changed +21
−3
scripts/test-shards.ts +8 −3
| @@ -42,6 +42,7 @@ | ||
| 42 | 42 | import { availableParallelism, tmpdir, totalmem } from "node:os"; |
| 43 | 43 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; |
| 44 | 44 | import { join } from "node:path"; |
| 45 | import { resolveLimits } from "../src/apps/contain.ts"; | |
| 45 | 46 | import { |
| 46 | 47 | formatTimes, |
| 47 | 48 | heaviestFirst, |
| @@ -344,8 +345,10 @@ const readLimit = (path: string): string | undefined => { | ||
| 344 | 345 | } |
| 345 | 346 | }; |
| 346 | 347 | // The cell's memory as forge.json declares it: its watchdog stops the |
| 347 | // check over it, and nothing in /proc or /sys says it. Its CPUs come from | |
| 348 | // FORGE_CELL_CPUS (shardCountFromEnv). | |
| 348 | // check over it, and nothing in /proc or /sys says it. The machine grants | |
| 349 | // no more than its maximum (resolveLimits), so a declared 8 GiB on a 4 GiB | |
| 350 | // machine is 4 GiB: sizing shards by the declaration overran the cap (#3105). | |
| 351 | // Its CPUs come from FORGE_CELL_CPUS (shardCountFromEnv). | |
| 349 | 352 | const cell = parseCheck(readLimit(new URL("../forge.json", import.meta.url).pathname), undefined); |
| 350 | 353 | const declared = cell.ok ? cell.value?.limits : undefined; |
| 351 | 354 | const cpus = availableParallelism(); |
| @@ -353,7 +356,9 @@ const shardsOverride = process.env["FORGE_CHECK_SHARDS"]; | ||
| 353 | 356 | const memory = memoryLimit(totalmem(), [ |
| 354 | 357 | readLimit("/sys/fs/cgroup/memory.max"), |
| 355 | 358 | readLimit("/sys/fs/cgroup/memory/memory.limit_in_bytes"), |
| 356 | declared?.memoryMiB === undefined ? undefined : String(declared.memoryMiB * 1024 * 1024), | |
| 359 | declared?.memoryMiB === undefined | |
| 360 | ? undefined | |
| 361 | : String(resolveLimits("step", declared).memoryMiB * 1024 * 1024), | |
| 357 | 362 | ]); |
| 358 | 363 | const count = shardCountFromEnv(process.env, cpus, files.length, memory); |
| 359 | 364 | if (!count.ok) { |
test/shards.test.ts +13
| @@ -7,6 +7,7 @@ import { afterAll, describe, expect, test } from "bun:test"; | ||
| 7 | 7 | import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 8 | 8 | import { tmpdir } from "node:os"; |
| 9 | 9 | import { join } from "node:path"; |
| 10 | import { resolveLimits } from "../src/apps/contain.ts"; | |
| 10 | 11 | import { |
| 11 | 12 | SECONDS_PER_BYTE_DEFAULT, |
| 12 | 13 | SHARDS_MAX, |
| @@ -297,6 +298,18 @@ describe("shardCount", () => { | ||
| 297 | 298 | expect(shardCount(16, 400, "20", 4 * GiB)).toEqual({ ok: true, value: 20 }); |
| 298 | 299 | }); |
| 299 | 300 | |
| 301 | test("the declared memory is held to what the machine grants before shards are sized (#3105)", () => { | |
| 302 | const src = readFileSync(new URL("../scripts/test-shards.ts", import.meta.url), "utf8"); | |
| 303 | expect(src).toContain('resolveLimits("step", declared).memoryMiB'); | |
| 304 | const saved = process.env["FORGE_CELL_MEMORY_MAX_MIB"]; | |
| 305 | delete process.env["FORGE_CELL_MEMORY_MAX_MIB"]; | |
| 306 | try { | |
| 307 | expect(resolveLimits("step", { memoryMiB: 8192 }).memoryMiB).toBe(4096); | |
| 308 | } finally { | |
| 309 | if (saved !== undefined) process.env["FORGE_CELL_MEMORY_MAX_MIB"] = saved; | |
| 310 | } | |
| 311 | }); | |
| 312 | ||
| 300 | 313 | test("the cell's declared CPUs hold the workers under the watchdog's limit (#2161)", () => { |
| 301 | 314 | const GiB = 1024 * 1024 * 1024; |
| 302 | 315 | // A 64-core host, the 16-CPU cell: 12 workers, not 16 (which averaged |
new above
offline