Public: anyone can read this; the crew can change it.
The gate's measuring run starts no more coverage workers than the check cell's 4 GiB holds #3124
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/3118-the-gate-s-measuring-run-starts-no-more → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +52
−4
scripts/shards.ts +17 −3
| @@ -24,6 +24,16 @@ export const SHARDS_MAX = 32; | ||
| 24 | 24 | */ |
| 25 | 25 | export const WORKER_MEMORY_BYTES = 512 * 1024 * 1024; |
| 26 | 26 | |
| 27 | /** | |
| 28 | * Memory one worker of a measuring run (`--impact`: one file per `bun test` | |
| 29 | * under `--coverage`) is given room for. Coverage roughly doubles a worker | |
| 30 | * that boots forges and apps: six main checks of 4 such workers were stopped | |
| 31 | * at 4105–4668 MiB of tree RSS against the cell's 4096 MiB (#3118), so about | |
| 32 | * 1170 MiB each at the peak; 1.25 GiB holds that with a little room, and | |
| 33 | * gives 3 workers in a 4096 MiB cell. | |
| 34 | */ | |
| 35 | export const MEASURE_WORKER_MEMORY_BYTES = 1280 * 1024 * 1024; | |
| 36 | ||
| 27 | 37 | // Lives in src/ because the running forge sizes its checks with it, and the |
| 28 | 38 | // image ships src/ alone (#2652). |
| 29 | 39 | export { CELL_CPU_SHARE, workerCpus } from "../src/core/capacity.ts"; |
| @@ -34,7 +44,9 @@ import { workerCpus } from "../src/core/capacity.ts"; | ||
| 34 | 44 | * memory holds at {@link WORKER_MEMORY_BYTES} each, capped at |
| 35 | 45 | * {@link SHARDS_MAX} and at the number of files, at least one. One per CPU, |
| 36 | 46 | * not fewer: most of a forge test's time is spent waiting on git children |
| 37 | * and timers, not computing. `memory` absent is not a bound. | |
| 47 | * and timers, not computing. `memory` absent is not a bound; `workerBytes` | |
| 48 | * is what one worker needs of it ({@link MEASURE_WORKER_MEMORY_BYTES} for a | |
| 49 | * measuring run). | |
| 38 | 50 | * `FORGE_CHECK_SHARDS` overrides it (still capped at the file count) so a |
| 39 | 51 | * small or busy host can turn it down; that is an explicit operator |
| 40 | 52 | * choice and wins over the cell cap below. `cellCpus` (`FORGE_CELL_CPUS`, the |
| @@ -49,12 +61,13 @@ export function shardCount( | ||
| 49 | 61 | override?: string, |
| 50 | 62 | memory?: number, |
| 51 | 63 | cellCpus?: string, |
| 64 | workerBytes: number = WORKER_MEMORY_BYTES, | |
| 52 | 65 | ): { ok: true; value: number } | { ok: false; error: string } { |
| 53 | 66 | let n = Math.min(SHARDS_MAX, Math.floor(cpus)); |
| 54 | 67 | if (cellCpus !== undefined && /^[1-9][0-9]{0,3}$/.test(cellCpus.trim())) |
| 55 | 68 | n = workerCpus(n, Number(cellCpus.trim())); |
| 56 | 69 | if (memory !== undefined && Number.isFinite(memory) && memory > 0) |
| 57 | n = Math.min(n, Math.floor(memory / WORKER_MEMORY_BYTES)); | |
| 70 | n = Math.min(n, Math.floor(memory / workerBytes)); | |
| 58 | 71 | if (override !== undefined && override.trim() !== "") { |
| 59 | 72 | if (!/^[1-9][0-9]{0,2}$/.test(override.trim())) |
| 60 | 73 | return { |
| @@ -76,8 +89,9 @@ export function shardCountFromEnv( | ||
| 76 | 89 | cpus: number, |
| 77 | 90 | files: number, |
| 78 | 91 | memory?: number, |
| 92 | workerBytes?: number, | |
| 79 | 93 | ): { ok: true; value: number } | { ok: false; error: string } { |
| 80 | return shardCount(cpus, files, env["FORGE_CHECK_SHARDS"], memory, env["FORGE_CELL_CPUS"]); | |
| 94 | return shardCount(cpus, files, env["FORGE_CHECK_SHARDS"], memory, env["FORGE_CELL_CPUS"], workerBytes); | |
| 81 | 95 | } |
| 82 | 96 | |
| 83 | 97 | /** |
scripts/test-shards.ts +9 −1
| @@ -46,6 +46,7 @@ import { resolveLimits } from "../src/apps/contain.ts"; | ||
| 46 | 46 | import { |
| 47 | 47 | formatTimes, |
| 48 | 48 | heaviestFirst, |
| 49 | MEASURE_WORKER_MEMORY_BYTES, | |
| 49 | 50 | memoryLimit, |
| 50 | 51 | overlayTimes, |
| 51 | 52 | refreshTimes, |
| @@ -360,7 +361,14 @@ const memory = memoryLimit(totalmem(), [ | ||
| 360 | 361 | ? undefined |
| 361 | 362 | : String(resolveLimits("step", declared).memoryMiB * 1024 * 1024), |
| 362 | 363 | ]); |
| 363 | const count = shardCountFromEnv(process.env, cpus, files.length, memory); | |
| 364 | // A measuring run's workers are under coverage and need more room each (#3118). | |
| 365 | const count = shardCountFromEnv( | |
| 366 | process.env, | |
| 367 | cpus, | |
| 368 | files.length, | |
| 369 | memory, | |
| 370 | measure ? MEASURE_WORKER_MEMORY_BYTES : undefined, | |
| 371 | ); | |
| 364 | 372 | if (!count.ok) { |
| 365 | 373 | console.error(`test-shards: ${count.error}`); |
| 366 | 374 | process.exit(2); |
test/shards.test.ts +26
| @@ -24,6 +24,7 @@ import { | ||
| 24 | 24 | parseTimes, |
| 25 | 25 | refreshTimes, |
| 26 | 26 | parseShardSpec, |
| 27 | MEASURE_WORKER_MEMORY_BYTES, | |
| 27 | 28 | shardCount, |
| 28 | 29 | shardCountFromEnv, |
| 29 | 30 | sliceFiles, |
| @@ -298,6 +299,31 @@ describe("shardCount", () => { | ||
| 298 | 299 | expect(shardCount(16, 400, "20", 4 * GiB)).toEqual({ ok: true, value: 20 }); |
| 299 | 300 | }); |
| 300 | 301 | |
| 302 | test("a measuring run under coverage starts fewer workers than the cell's 4 GiB would hold plain (#3118)", () => { | |
| 303 | const GiB = 1024 * 1024 * 1024; | |
| 304 | // 16 CPUs held to 4 workers by the cell's CPU limit. | |
| 305 | expect(shardCount(16, 630, undefined, 4 * GiB, "6")).toEqual({ ok: true, value: 4 }); | |
| 306 | const measuring = shardCount(16, 630, undefined, 4 * GiB, "6", MEASURE_WORKER_MEMORY_BYTES); | |
| 307 | expect(measuring).toEqual({ ok: true, value: 3 }); | |
| 308 | // 3 coverage workers at their measured ~1170 MiB peak fit under 4096 MiB; 4 did not. | |
| 309 | expect(3 * 1170).toBeLessThan(4096); | |
| 310 | expect(4 * 1170).toBeGreaterThan(4096); | |
| 311 | expect(shardCountFromEnv({ FORGE_CELL_CPUS: "6" }, 16, 630, 4 * GiB)).toEqual({ ok: true, value: 4 }); | |
| 312 | expect( | |
| 313 | shardCountFromEnv({ FORGE_CELL_CPUS: "6" }, 16, 630, 4 * GiB, MEASURE_WORKER_MEMORY_BYTES), | |
| 314 | ).toEqual({ ok: true, value: 3 }); | |
| 315 | // The override still wins. | |
| 316 | expect( | |
| 317 | shardCountFromEnv({ FORGE_CELL_CPUS: "6", FORGE_CHECK_SHARDS: "4" }, 16, 630, 4 * GiB, MEASURE_WORKER_MEMORY_BYTES), | |
| 318 | ).toEqual({ ok: true, value: 4 }); | |
| 319 | }); | |
| 320 | ||
| 321 | test("the runner sizes a measuring run by the coverage figure (#3118)", () => { | |
| 322 | // The script runs at import, so its wiring is read from its source. | |
| 323 | const source = readFileSync(new URL("../scripts/test-shards.ts", import.meta.url), "utf8"); | |
| 324 | expect(source).toContain("measure ? MEASURE_WORKER_MEMORY_BYTES : undefined"); | |
This asserts on source text, so it passes even if the wiring is wrong and fails on a harmless reformat. Move the choice of worker bytes and the resolved memory into an exported function and test its output. |
||
| 325 | }); | |
| 326 | ||
| 301 | 327 | test("the declared memory is held to what the machine grants before shards are sized (#3105)", () => { |
| 302 | 328 | const src = readFileSync(new URL("../scripts/test-shards.ts", import.meta.url), "utf8"); |
| 303 | 329 | expect(src).toContain('resolveLimits("step", declared).memoryMiB'); |
new above
offline
These compare constants written in the test, so they cannot fail. Compute them from MEASURE_WORKER_MEMORY_BYTES or remove them.