Public: anyone can read this; the crew can change it.
Runner checks honour the shard setting: FORGE_CHECK_SHARDS reaches runner jobs, and the log says the count in force #14846
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14844-runner-checks-honour-the-shard-setting-f → main 2 commits fork 3 threads · 3 unresolved
6 files
changed +122
−11
src/checks/run_tier.ts +4 −3
| @@ -19,10 +19,9 @@ import { redDetail } from "./report.ts"; | ||
| 19 | 19 | import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts"; |
| 20 | 20 | import { failedFiles, onlyEnv, passedFiles } from "./verify.ts"; |
| 21 | 21 | import { exportTree } from "../apps/checkout.ts"; |
| 22 | import { workerCpus } from "../core/capacity.ts"; | |
| 23 | 22 | import { childEnv, killedStep, runStep } from "../apps/process.ts"; |
| 24 | 23 | import { noteReached } from "../apps/egress_report.ts"; |
| 25 | import { capacityEnv, cpuLimit, egressSetting } from "../core/capacity.ts"; | |
| 24 | import { capacityEnv, checkShardCpus, egressSetting, shardsInForce, shardsLine } from "../core/capacity.ts"; | |
| 26 | 25 | import { cellsAvailable, origin, limitWarnings, resolveLimits, STEP_HOSTS, stoppedForCpu, type CellSpec } from "../apps/contain.ts"; |
| 27 | 26 | import { summariseCheckFailure } from "./summary.ts"; |
| 28 | 27 | import { err, type Result } from "../kernel/result.ts"; |
| @@ -207,10 +206,12 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco | ||
| 207 | 206 | if (small !== null) return small; |
| 208 | 207 | for (const line of limitWarnings(manifest.limits)) write(line); |
| 209 | 208 | const cellCpus = resolveLimits("step", manifest.limits).cpus; |
| 209 | const shardCpus = checkShardCpus(cellCpus); | |
| 210 | write(shardsLine(shardsInForce(app.db, shardCpus))); | |
| 210 | 211 | const env = childEnv({ |
| 211 | 212 | // The shard count an admin set (`core/capacity.ts`), else the |
| 212 | 213 | // default sized to what this check's cell holds, never the pod's. |
| 213 | ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)), | |
| 214 | ...capacityEnv(app.db, shardCpus), | |
| 214 | 215 | CI: "1", |
| 215 | 216 | FORGE_URL: options.forgeUrl, |
| 216 | 217 | FORGE_CHECK: run.id, |
src/checks/tier_runner.ts +7 −2
| @@ -7,8 +7,8 @@ import { headDepth } from "./history.ts"; | ||
| 7 | 7 | import { clearedIn } from "./base_red.ts"; |
| 8 | 8 | import { recordCrashFlakes } from "./flake.ts"; |
| 9 | 9 | import { handoverEnv } from "./handover.ts"; |
| 10 | import { egressSetting } from "../core/capacity.ts"; | |
| 11 | import { STEP_HOSTS } from "../apps/contain.ts"; | |
| 10 | import { capacityEnv, checkShardCpus, egressSetting, shardsInForce, shardsLine } from "../core/capacity.ts"; | |
| 11 | import { resolveLimits, STEP_HOSTS } from "../apps/contain.ts"; | |
| 12 | 12 | import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts"; |
| 13 | 13 | import type { CheckOutcome } from "./runner.ts"; |
| 14 | 14 | import type { LaneState } from "./lane_state.ts"; |
| @@ -45,6 +45,10 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck): | ||
| 45 | 45 | // or an answer that is not the check's own verdict, and it runs here as it always has. |
| 46 | 46 | if (lane.placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) { |
| 47 | 47 | const lines: string[] = []; |
| 48 | // The shard count an admin set reaches a runner's check as it does one run here (#14844); the | |
| 49 | // default is sized as one run here is (`checkShardCpus`), never from the runner machine's CPUs. | |
| 50 | const shardCpus = checkShardCpus(resolveLimits("step", manifest.limits).cpus); | |
outdated
The sized default here uses the raw cellCpus. run_tier.ts uses workerCpus(cpuLimit(), cellCpus). Share one computation so both places give the same count. |
||
| 51 | write(shardsLine(shardsInForce(app.db, shardCpus))); | |
| 48 | 52 | const sent = await runRunnerStep(app, { |
| 49 | 53 | runId: x.runId, |
| 50 | 54 | repoId: repo.id, |
| @@ -61,6 +65,7 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck): | ||
| 61 | 65 | ], |
| 62 | 66 | // Only the check's own variables cross, never the forge's; the runner adds its cell's. |
| 63 | 67 | env: { |
| 68 | ...capacityEnv(app.db, shardCpus), | |
| 64 | 69 | CI: "1", |
| 65 | 70 | FORGE_URL: options.forgeUrl, |
| 66 | 71 | FORGE_CHECK: run.id, |
src/core/capacity.ts +30 −5
| @@ -698,11 +698,36 @@ export function edgeSigninOn(db: Database): boolean { | ||
| 698 | 698 | * small cell is never told the pod's count. |
| 699 | 699 | */ |
| 700 | 700 | export function capacityEnv(db: Database, cpus?: number): Record<string, string> { |
| 701 | const shards = | |
| 702 | storedCapacity(db, "checks.shards") ?? | |
| 703 | explicitDefault("checks.shards") ?? | |
| 704 | String(shardsForCpus(cpus ?? cpuLimit())); | |
| 705 | return { FORGE_CHECK_SHARDS: shards }; | |
| 701 | return { FORGE_CHECK_SHARDS: String(shardsInForce(db, cpus).count) }; | |
| 702 | } | |
| 703 | ||
| 704 | /** Where the shard count in force came from: an admin's setting, the environment, or sized. */ | |
| 705 | export type ShardsSource = "setting" | "environment" | "sized"; | |
| 706 | ||
| 707 | /** | |
| 708 | * The shard count {@link capacityEnv} hands a check, and where it came from: an admin's value, | |
| 709 | * else `FORGE_CHECK_SHARDS` in the forge's environment, else sized from `cpus`. | |
| 710 | */ | |
| 711 | export function shardsInForce(db: Database, cpus?: number): { count: number; source: ShardsSource } { | |
| 712 | const stored = storedCapacity(db, "checks.shards"); | |
| 713 | if (stored !== undefined) return { count: Number(stored), source: "setting" }; | |
| 714 | const env = explicitDefault("checks.shards"); | |
| 715 | if (env !== undefined) return { count: Number(env), source: "environment" }; | |
| 716 | return { count: shardsForCpus(cpus ?? cpuLimit()), source: "sized" }; | |
| 717 | } | |
| 718 | ||
| 719 | /** | |
| 720 | * The CPUs a check's default shard count is sized from, wherever it runs: its step cell's share, | |
| 721 | * held to this pod's (`workerCpus`). A runner job is sized the same way as a check run here, so an | |
| 722 | * unset setting gives one count whichever machine takes the check (#14844). | |
| 723 | */ | |
| 724 | export function checkShardCpus(cellCpus: number | undefined): number { | |
| 725 | return workerCpus(cpuLimit(), cellCpus); | |
| 726 | } | |
| 727 | ||
| 728 | /** The check log's line for {@link shardsInForce}: `3 shards (setting)`. */ | |
| 729 | export function shardsLine(shards: { count: number; source: ShardsSource }): string { | |
| 730 | return `${shards.count} shard${shards.count === 1 ? "" : "s"} (${shards.source})`; | |
| 706 | 731 | } |
| 707 | 732 | |
| 708 | 733 | export interface CapacityChange { |
test/capacity.test.ts +10 −1
| @@ -31,6 +31,8 @@ import { | ||
| 31 | 31 | capacityEnv, |
| 32 | 32 | capacityValue, |
| 33 | 33 | egressSetting, |
| 34 | shardsInForce, | |
| 35 | shardsLine, | |
| 34 | 36 | } from "../src/core/capacity.ts"; |
| 35 | 37 | import type { Repo } from "../src/core/repos.ts"; |
| 36 | 38 | import { listRuns, type Run } from "../src/core/runs.ts"; |
| @@ -288,6 +290,9 @@ describe("capacity: only a forge admin reads or changes it", () => { | ||
| 288 | 290 | }); |
| 289 | 291 | expect(posted.status).toBe(303); |
| 290 | 292 | expect(capacityEnv(app.db)).toEqual({ FORGE_CHECK_SHARDS: "3" }); |
| 293 | // The check log names the count and its source (#14844): the setting wins over the | |
| 294 | // environment. | |
| 295 | expect(shardsLine(shardsInForce(app.db, 80))).toBe("3 shards (setting)"); | |
| 291 | 296 | |
| 292 | 297 | const put = await api( |
| 293 | 298 | aliceAdmin, |
| @@ -346,8 +351,12 @@ describe("capacity: only a forge admin reads or changes it", () => { | ||
| 346 | 351 | try { |
| 347 | 352 | expect(capacityEnv(app.db, 80)).toEqual({ FORGE_CHECK_SHARDS: "4" }); |
| 348 | 353 | expect(capacityEnv(app.db, 2)).toEqual({ FORGE_CHECK_SHARDS: "2" }); |
| 354 | expect(shardsLine(shardsInForce(app.db, 1))).toBe("1 shard (sized)"); | |
| 355 | process.env.FORGE_CHECK_SHARDS = "5"; | |
| 356 | expect(shardsInForce(app.db, 2)).toEqual({ count: 5, source: "environment" }); | |
Restore the env var in the finally block. As written, a failed expect() leaves FORGE_CHECK_SHARDS=5 set for the tests that follow. |
||
| 349 | 357 | } finally { |
| 350 | if (fromEnv !== undefined) process.env.FORGE_CHECK_SHARDS = fromEnv; | |
| 358 | if (fromEnv === undefined) delete process.env.FORGE_CHECK_SHARDS; | |
| 359 | else process.env.FORGE_CHECK_SHARDS = fromEnv; | |
| 351 | 360 | } |
| 352 | 361 | const shardsDefault = capacityDefault("checks.shards"); |
| 353 | 362 |
test/checks-placement.test.ts +14
| @@ -354,5 +354,19 @@ describe("checks placed", () => { | ||
| 354 | 354 | expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain("here"); |
| 355 | 355 | expect(calls).toHaveLength(0); |
| 356 | 356 | }); |
| 357 | ||
| 358 | test("a check run here logs the shard count in force and where it came from (#14844)", async () => { | |
| 359 | setCapacity(app.db, "checks.shards", "3", alice.id); | |
| 360 | try { | |
| 361 | const run = await settled(await push("echo sharded-here")); | |
| 362 | expect(run.state).toBe("ok"); | |
| 363 | expect(run.placement).toBe("process"); | |
| 364 | const log = readRunLog(dataDir, run.id).lines.join("\n"); | |
| 365 | expect(log).toContain("3 shards (setting)"); | |
| 366 | expect(log).toContain("sharded-here"); | |
| 367 | } finally { | |
| 368 | setCapacity(app.db, "checks.shards", null, alice.id); | |
| 369 | } | |
| 370 | }); | |
| 357 | 371 | }); |
| 358 | 372 | }); |
test/checks-runner-placement.test.ts +57
| @@ -26,6 +26,8 @@ import { | ||
| 26 | 26 | } from "../src/checks/runner_jobs.ts"; |
| 27 | 27 | import { readSteps, runnerTakes, runnerVerdict } from "../src/checks/runner_step.ts"; |
| 28 | 28 | import { startChecks } from "../src/checks/service.ts"; |
| 29 | import { checkShardCpus, setCapacity, shardsForCpus } from "../src/core/capacity.ts"; | |
| 30 | import { resolveLimits } from "../src/apps/contain.ts"; | |
| 29 | 31 | import { createToken } from "../src/core/tokens.ts"; |
| 30 | 32 | import { ok, err } from "../src/kernel/result.ts"; |
| 31 | 33 | import { readRunLog } from "../src/runs/log.ts"; |
| @@ -488,6 +490,61 @@ describe("a pull's check with FORGE_CHECKS_PLACEMENT=runner", () => { | ||
| 488 | 490 | expect(log).toContain("ran-the-check"); |
| 489 | 491 | }); |
| 490 | 492 | |
| 493 | test("a runner's job carries the admin's shard count, and its log says the count in force and where it came from (#14844)", async () => { | |
| 494 | setCapacity(f.app.db, "checks.shards", "3", f.alice.id); | |
| 495 | try { | |
| 496 | const sharded = enrolRunner(f.app.db, "sharded", f.alice.id); | |
| 497 | const asked = api("/api/v1/runner/claim", sharded.token, { wait: 20, cells: true }); | |
| 498 | await f.until(() => deskOf(f.app).claimers.size > 0, "the runner waiting in its claim"); | |
| 499 | const oid = await pull("sharded", "sh.txt"); | |
| 500 | const claim = (await (await asked).json()) as { job: RunnerJob; token: string }; | |
| 501 | expect(claim.job.env["FORGE_CHECK_SHARDS"]).toBe("3"); | |
| 502 | const door = `/api/v1/runner/jobs/${claim.job.id}`; | |
| 503 | expect((await api(door, claim.token, { step: "check", lines: ["ran-on-sharded"] })).status).toBe(200); | |
| 504 | expect( | |
| 505 | (await api(door, claim.token, { done: { steps: [{ name: "check", ok: true, ms: 1 }], contained: true, note: "" } })) | |
| 506 | .status, | |
| 507 | ).toBe(200); | |
| 508 | const run = await f.settled((r) => r.commitOid === oid, 30_000); | |
| 509 | expect(run.state).toBe("ok"); | |
| 510 | expect(run.placement).toBe("runner"); | |
| 511 | const log = logOf(run.id); | |
| 512 | expect(log).toContain("3 shards (setting)"); | |
| 513 | expect(log).toContain("ran-on-sharded"); | |
| 514 | expect(log).not.toContain("running here"); | |
| 515 | } finally { | |
| 516 | setCapacity(f.app.db, "checks.shards", null, f.alice.id); | |
| 517 | } | |
| 518 | }); | |
| 519 | ||
| 520 | test("with no setting a runner's job gets the default sized as a check run here is, and its log says sized (#14844)", async () => { | |
| 521 | const fromEnv = process.env.FORGE_CHECK_SHARDS; | |
| 522 | delete process.env.FORGE_CHECK_SHARDS; | |
| 523 | try { | |
| 524 | // What run_tier.ts would hand the same check: the step cell's share, held to this pod. | |
| 525 | const sized = shardsForCpus(checkShardCpus(resolveLimits("step", undefined).cpus)); | |
| 526 | const sizer = enrolRunner(f.app.db, "sizer", f.alice.id); | |
| 527 | const asked = api("/api/v1/runner/claim", sizer.token, { wait: 20, cells: true }); | |
| 528 | await f.until(() => deskOf(f.app).claimers.size > 0, "the runner waiting in its claim"); | |
| 529 | const oid = await pull("sizer", "sz.txt"); | |
| 530 | const claim = (await (await asked).json()) as { job: RunnerJob; token: string }; | |
| 531 | expect(claim.job.env["FORGE_CHECK_SHARDS"]).toBe(String(sized)); | |
| 532 | const door = `/api/v1/runner/jobs/${claim.job.id}`; | |
| 533 | expect((await api(door, claim.token, { step: "check", lines: ["ran-on-sizer"] })).status).toBe(200); | |
| 534 | expect( | |
| 535 | (await api(door, claim.token, { done: { steps: [{ name: "check", ok: true, ms: 1 }], contained: true, note: "" } })) | |
| 536 | .status, | |
| 537 | ).toBe(200); | |
| 538 | const run = await f.settled((r) => r.commitOid === oid, 30_000); | |
| 539 | expect(run.state).toBe("ok"); | |
| 540 | expect(run.placement).toBe("runner"); | |
| 541 | expect(logOf(run.id)).toContain(`${sized} shard${sized === 1 ? "" : "s"} (sized)`); | |
| 542 | } finally { | |
| 543 | if (fromEnv === undefined) delete process.env.FORGE_CHECK_SHARDS; | |
| 544 | else process.env.FORGE_CHECK_SHARDS = fromEnv; | |
| 545 | } | |
| 546 | }); | |
| 547 | ||
| 491 | 548 | describe("with a runner process connected", () => { |
| 492 | 549 | let proc: ReturnType<typeof Bun.spawn> | undefined; |
| 493 | 550 | let runnerId = ""; |
new above
offline
No test covers this log line. Add one for a locally-run check.