Public: anyone can read this; the crew can change it.

Runner checks honour the shard setting: FORGE_CHECK_SHARDS reaches runner jobs, and the log says the count in force #14846

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14844-runner-checks-honour-the-shard-setting-f → main 2 commits fork 3 threads · 3 unresolved

  • Merged. mason/14844-runner-checks-honour-the-shard-setting-f was merged into main as 3ff7811.

Review

Sign in to review this change.
6 files changed +122 −11
src/checks/run_tier.ts +4 −3
@@ -19,10 +19,9 @@ import { redDetail } from "./report.ts";
1919 import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts";
2020 import { failedFiles, onlyEnv, passedFiles } from "./verify.ts";
2121 import { exportTree } from "../apps/checkout.ts";
22 import { workerCpus } from "../core/capacity.ts";
2322 import { childEnv, killedStep, runStep } from "../apps/process.ts";
2423 import { noteReached } from "../apps/egress_report.ts";
25 import { capacityEnv, cpuLimit, egressSetting } from "../core/capacity.ts";
24 import { capacityEnv, checkShardCpus, egressSetting, shardsInForce, shardsLine } from "../core/capacity.ts";
2625 import { cellsAvailable, origin, limitWarnings, resolveLimits, STEP_HOSTS, stoppedForCpu, type CellSpec } from "../apps/contain.ts";
2726 import { summariseCheckFailure } from "./summary.ts";
2827 import { err, type Result } from "../kernel/result.ts";
@@ -207,10 +206,12 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco
207206 if (small !== null) return small;
208207 for (const line of limitWarnings(manifest.limits)) write(line);
209208 const cellCpus = resolveLimits("step", manifest.limits).cpus;
209 const shardCpus = checkShardCpus(cellCpus);
210 write(shardsLine(shardsInForce(app.db, shardCpus)));
210211 const env = childEnv({
outdated

No test covers this log line. Add one for a locally-run check.

211212 // The shard count an admin set (`core/capacity.ts`), else the
212213 // default sized to what this check's cell holds, never the pod's.
213 ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)),
214 ...capacityEnv(app.db, shardCpus),
214215 CI: "1",
215216 FORGE_URL: options.forgeUrl,
216217 FORGE_CHECK: run.id,
src/checks/tier_runner.ts +7 −2
@@ -7,8 +7,8 @@ import { headDepth } from "./history.ts";
77 import { clearedIn } from "./base_red.ts";
88 import { recordCrashFlakes } from "./flake.ts";
99 import { handoverEnv } from "./handover.ts";
10 import { egressSetting } from "../core/capacity.ts";
11 import { STEP_HOSTS } from "../apps/contain.ts";
10 import { capacityEnv, checkShardCpus, egressSetting, shardsInForce, shardsLine } from "../core/capacity.ts";
11 import { resolveLimits, STEP_HOSTS } from "../apps/contain.ts";
1212 import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts";
1313 import type { CheckOutcome } from "./runner.ts";
1414 import type { LaneState } from "./lane_state.ts";
@@ -45,6 +45,10 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck):
4545 // or an answer that is not the check's own verdict, and it runs here as it always has.
4646 if (lane.placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) {
4747 const lines: string[] = [];
48 // The shard count an admin set reaches a runner's check as it does one run here (#14844); the
49 // default is sized as one run here is (`checkShardCpus`), never from the runner machine's CPUs.
50 const shardCpus = checkShardCpus(resolveLimits("step", manifest.limits).cpus);
outdated

The sized default here uses the raw cellCpus. run_tier.ts uses workerCpus(cpuLimit(), cellCpus). Share one computation so both places give the same count.

51 write(shardsLine(shardsInForce(app.db, shardCpus)));
4852 const sent = await runRunnerStep(app, {
4953 runId: x.runId,
5054 repoId: repo.id,
@@ -61,6 +65,7 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck):
6165 ],
6266 // Only the check's own variables cross, never the forge's; the runner adds its cell's.
6367 env: {
68 ...capacityEnv(app.db, shardCpus),
6469 CI: "1",
6570 FORGE_URL: options.forgeUrl,
6671 FORGE_CHECK: run.id,
src/core/capacity.ts +30 −5
@@ -698,11 +698,36 @@ export function edgeSigninOn(db: Database): boolean {
698698 * small cell is never told the pod's count.
699699 */
700700 export function capacityEnv(db: Database, cpus?: number): Record<string, string> {
701 const shards =
702 storedCapacity(db, "checks.shards") ??
703 explicitDefault("checks.shards") ??
704 String(shardsForCpus(cpus ?? cpuLimit()));
705 return { FORGE_CHECK_SHARDS: shards };
701 return { FORGE_CHECK_SHARDS: String(shardsInForce(db, cpus).count) };
702 }
703
704 /** Where the shard count in force came from: an admin's setting, the environment, or sized. */
705 export type ShardsSource = "setting" | "environment" | "sized";
706
707 /**
708 * The shard count {@link capacityEnv} hands a check, and where it came from: an admin's value,
709 * else `FORGE_CHECK_SHARDS` in the forge's environment, else sized from `cpus`.
710 */
711 export function shardsInForce(db: Database, cpus?: number): { count: number; source: ShardsSource } {
712 const stored = storedCapacity(db, "checks.shards");
713 if (stored !== undefined) return { count: Number(stored), source: "setting" };
714 const env = explicitDefault("checks.shards");
715 if (env !== undefined) return { count: Number(env), source: "environment" };
716 return { count: shardsForCpus(cpus ?? cpuLimit()), source: "sized" };
717 }
718
719 /**
720 * The CPUs a check's default shard count is sized from, wherever it runs: its step cell's share,
721 * held to this pod's (`workerCpus`). A runner job is sized the same way as a check run here, so an
722 * unset setting gives one count whichever machine takes the check (#14844).
723 */
724 export function checkShardCpus(cellCpus: number | undefined): number {
725 return workerCpus(cpuLimit(), cellCpus);
726 }
727
728 /** The check log's line for {@link shardsInForce}: `3 shards (setting)`. */
729 export function shardsLine(shards: { count: number; source: ShardsSource }): string {
730 return `${shards.count} shard${shards.count === 1 ? "" : "s"} (${shards.source})`;
706731 }
707732
708733 export interface CapacityChange {
test/capacity.test.ts +10 −1
@@ -31,6 +31,8 @@ import {
3131 capacityEnv,
3232 capacityValue,
3333 egressSetting,
34 shardsInForce,
35 shardsLine,
3436 } from "../src/core/capacity.ts";
3537 import type { Repo } from "../src/core/repos.ts";
3638 import { listRuns, type Run } from "../src/core/runs.ts";
@@ -288,6 +290,9 @@ describe("capacity: only a forge admin reads or changes it", () => {
288290 });
289291 expect(posted.status).toBe(303);
290292 expect(capacityEnv(app.db)).toEqual({ FORGE_CHECK_SHARDS: "3" });
293 // The check log names the count and its source (#14844): the setting wins over the
294 // environment.
295 expect(shardsLine(shardsInForce(app.db, 80))).toBe("3 shards (setting)");
291296
292297 const put = await api(
293298 aliceAdmin,
@@ -346,8 +351,12 @@ describe("capacity: only a forge admin reads or changes it", () => {
346351 try {
347352 expect(capacityEnv(app.db, 80)).toEqual({ FORGE_CHECK_SHARDS: "4" });
348353 expect(capacityEnv(app.db, 2)).toEqual({ FORGE_CHECK_SHARDS: "2" });
354 expect(shardsLine(shardsInForce(app.db, 1))).toBe("1 shard (sized)");
355 process.env.FORGE_CHECK_SHARDS = "5";
356 expect(shardsInForce(app.db, 2)).toEqual({ count: 5, source: "environment" });

Restore the env var in the finally block. As written, a failed expect() leaves FORGE_CHECK_SHARDS=5 set for the tests that follow.

349357 } finally {
350 if (fromEnv !== undefined) process.env.FORGE_CHECK_SHARDS = fromEnv;
358 if (fromEnv === undefined) delete process.env.FORGE_CHECK_SHARDS;
359 else process.env.FORGE_CHECK_SHARDS = fromEnv;
351360 }
352361 const shardsDefault = capacityDefault("checks.shards");
353362
test/checks-placement.test.ts +14
@@ -354,5 +354,19 @@ describe("checks placed", () => {
354354 expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain("here");
355355 expect(calls).toHaveLength(0);
356356 });
357
358 test("a check run here logs the shard count in force and where it came from (#14844)", async () => {
359 setCapacity(app.db, "checks.shards", "3", alice.id);
360 try {
361 const run = await settled(await push("echo sharded-here"));
362 expect(run.state).toBe("ok");
363 expect(run.placement).toBe("process");
364 const log = readRunLog(dataDir, run.id).lines.join("\n");
365 expect(log).toContain("3 shards (setting)");
366 expect(log).toContain("sharded-here");
367 } finally {
368 setCapacity(app.db, "checks.shards", null, alice.id);
369 }
370 });
357371 });
358372 });
test/checks-runner-placement.test.ts +57
@@ -26,6 +26,8 @@ import {
2626 } from "../src/checks/runner_jobs.ts";
2727 import { readSteps, runnerTakes, runnerVerdict } from "../src/checks/runner_step.ts";
2828 import { startChecks } from "../src/checks/service.ts";
29 import { checkShardCpus, setCapacity, shardsForCpus } from "../src/core/capacity.ts";
30 import { resolveLimits } from "../src/apps/contain.ts";
2931 import { createToken } from "../src/core/tokens.ts";
3032 import { ok, err } from "../src/kernel/result.ts";
3133 import { readRunLog } from "../src/runs/log.ts";
@@ -488,6 +490,61 @@ describe("a pull's check with FORGE_CHECKS_PLACEMENT=runner", () => {
488490 expect(log).toContain("ran-the-check");
489491 });
490492
493 test("a runner's job carries the admin's shard count, and its log says the count in force and where it came from (#14844)", async () => {
494 setCapacity(f.app.db, "checks.shards", "3", f.alice.id);
495 try {
496 const sharded = enrolRunner(f.app.db, "sharded", f.alice.id);
497 const asked = api("/api/v1/runner/claim", sharded.token, { wait: 20, cells: true });
498 await f.until(() => deskOf(f.app).claimers.size > 0, "the runner waiting in its claim");
499 const oid = await pull("sharded", "sh.txt");
500 const claim = (await (await asked).json()) as { job: RunnerJob; token: string };
501 expect(claim.job.env["FORGE_CHECK_SHARDS"]).toBe("3");
502 const door = `/api/v1/runner/jobs/${claim.job.id}`;
503 expect((await api(door, claim.token, { step: "check", lines: ["ran-on-sharded"] })).status).toBe(200);
504 expect(
505 (await api(door, claim.token, { done: { steps: [{ name: "check", ok: true, ms: 1 }], contained: true, note: "" } }))
506 .status,
507 ).toBe(200);
508 const run = await f.settled((r) => r.commitOid === oid, 30_000);
509 expect(run.state).toBe("ok");
510 expect(run.placement).toBe("runner");
511 const log = logOf(run.id);
512 expect(log).toContain("3 shards (setting)");
513 expect(log).toContain("ran-on-sharded");
514 expect(log).not.toContain("running here");
515 } finally {
516 setCapacity(f.app.db, "checks.shards", null, f.alice.id);
517 }
518 });
519
520 test("with no setting a runner's job gets the default sized as a check run here is, and its log says sized (#14844)", async () => {
521 const fromEnv = process.env.FORGE_CHECK_SHARDS;
522 delete process.env.FORGE_CHECK_SHARDS;
523 try {
524 // What run_tier.ts would hand the same check: the step cell's share, held to this pod.
525 const sized = shardsForCpus(checkShardCpus(resolveLimits("step", undefined).cpus));
526 const sizer = enrolRunner(f.app.db, "sizer", f.alice.id);
527 const asked = api("/api/v1/runner/claim", sizer.token, { wait: 20, cells: true });
528 await f.until(() => deskOf(f.app).claimers.size > 0, "the runner waiting in its claim");
529 const oid = await pull("sizer", "sz.txt");
530 const claim = (await (await asked).json()) as { job: RunnerJob; token: string };
531 expect(claim.job.env["FORGE_CHECK_SHARDS"]).toBe(String(sized));
532 const door = `/api/v1/runner/jobs/${claim.job.id}`;
533 expect((await api(door, claim.token, { step: "check", lines: ["ran-on-sizer"] })).status).toBe(200);
534 expect(
535 (await api(door, claim.token, { done: { steps: [{ name: "check", ok: true, ms: 1 }], contained: true, note: "" } }))
536 .status,
537 ).toBe(200);
538 const run = await f.settled((r) => r.commitOid === oid, 30_000);
539 expect(run.state).toBe("ok");
540 expect(run.placement).toBe("runner");
541 expect(logOf(run.id)).toContain(`${sized} shard${sized === 1 ? "" : "s"} (sized)`);
542 } finally {
543 if (fromEnv === undefined) delete process.env.FORGE_CHECK_SHARDS;
544 else process.env.FORGE_CHECK_SHARDS = fromEnv;
545 }
546 });
547
491548 describe("with a runner process connected", () => {
492549 let proc: ReturnType<typeof Bun.spawn> | undefined;
493550 let runnerId = "";

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs