Public: anyone can read this; the crew can change it.

checks.concurrency counts a runner's capacity, so adding a runner raises how many checks run at once #15268

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/15253-checks-concurrency-counts-a-runner-s-cap → main 5 commits fork

  • Merged. mason/15253-checks-concurrency-counts-a-runner-s-cap was merged into main as 913eacc.

Review

Sign in to review this change.
11 files changed +158 −12
docs/features/runners.md +8 −2
@@ -34,8 +34,14 @@ started by hand are in addition: an admin removes them with
3434 `DELETE /api/v1/admin/runners/<id>`. Every `local-*` row created by `forge` is
3535 deleted at each boot, so never name a hand-made runner `local-N`.
3636
37 Not yet: a page for them, `checks.concurrency` counting runner capacity, a job
38 that survives a restart, caches by hash (#14201's follow-ups).
37 `checks.concurrency` counts the forge's own cells; under the runner placement
38 each runner seen in the last minute adds the jobs it runs at once
39 (`FORGE_RUNNER_JOBS`, which every claim tells the forge), up to the setting's
40 ceiling of 16. One that stops asking stops counting a minute later, with no
41 restart (#15253).
42
43 Not yet: a page for them, a job that survives a restart, caches by hash
44 (#14201's follow-ups).
3945
4046 ## Start one, in two lines
4147
src/checks/lane_state.ts +14 −4
@@ -7,7 +7,8 @@ import type { Run } from "../core/runs.ts";
77 import { Gate } from "../git/env.ts";
88 import { runStep } from "../apps/process.ts";
99 import type { DeepsecDeps } from "./deepsec_step.ts";
10 import { capacityCount } from "../core/capacity.ts";
10 import { CAPACITY, capacityCount } from "../core/capacity.ts";
11 import { liveRunnerJobs } from "./runner_jobs.ts";
1112 import type { Placement } from "./placement.ts";
1213 import type { PodRoom } from "./pod_room.ts";
1314 import type { GateStage } from "./gate_reason.ts";
@@ -213,9 +214,18 @@ export interface LaneState {
213214 export const slotted = (lane: LaneState): number =>
214215 lane.inFlight.size - (lane.verifying === undefined ? 0 : 1);
215216
217 /** The most checks at once however many runners are live: the setting's own ceiling. */
218 export const CHECKS_CEILING = ((kind) => (kind.type === "count" ? kind.max : 16))(CAPACITY["checks.concurrency"].kind);
219
216220 /**
217221 * How many checks may run at once, read on every pick (`core/capacity.ts`): an admin's change is
218 * honoured by the next check, not the next boot. `concurrency` is the boot default.
222 * honoured by the next check, not the next boot. `concurrency` is the boot default. Under the
223 * runner placement the jobs of every live runner add to it (`liveRunnerJobs`), so one that
224 * connects raises it at the next tick and one gone quiet drops out once `SEEN_FRESH_MS` passes;
225 * never above `CHECKS_CEILING`, so a crowd of runners cannot overload this machine's lane.
219226 */
220 export const capacity = (lane: LaneState): number =>
221 capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1);
227 export function capacity(lane: LaneState, now: number = Date.now()): number {
228 const setting = capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1);
229 const runners = lane.placement.mode === "runner" ? liveRunnerJobs(lane.app.db, now) : 0;
230 return Math.min(CHECKS_CEILING, setting + runners);
231 }
src/checks/runner_jobs.ts +24
@@ -210,6 +210,30 @@ export function runnersAround(db: Database, now: number): boolean {
210210 );
211211 }
212212
213 /** The most jobs one runner may say it runs at once, as the runner itself clamps it. */
214 export const MAX_RUNNER_JOBS = 64;
215
216 /**
217 * Keep what a runner's claim says it runs at once (`FORGE_RUNNER_JOBS`); anything but a whole
218 * number from 1 to `MAX_RUNNER_JOBS` is ignored, and an unchanged count writes nothing.
219 */
220 export function noteRunnerJobs(db: Database, runnerId: string, said: unknown): void {
221 if (typeof said !== "number" || !Number.isInteger(said) || said < 1 || said > MAX_RUNNER_JOBS) return;
222 db.query("UPDATE check_runner SET jobs = ?2 WHERE id = ?1 AND jobs != ?2").run(runnerId, said);
223 }
224
225 /**
226 * The jobs the live runners run at once: the sum of each one's count over those seen within
227 * `SEEN_FRESH_MS`. One not seen since stops counting on the next read, with no restart.
228 */
229 export function liveRunnerJobs(db: Database, now: number): number {
230 return (
231 db
232 .query<{ n: number | null }, [number]>("SELECT SUM(jobs) AS n FROM check_runner WHERE seen_at > ?1")
233 .get(now - SEEN_FRESH_MS)?.n ?? 0
234 );
235 }
236
213237 /**
214238 * Put a check's steps on the desk. `dir` is the bare repository the commit is in; it stays here.
215239 */
src/contract/openapi/routes/postApiV1RunnerClaim.ts +2 −2
@@ -6,10 +6,10 @@ export const route: Route = {
66 summary:
77 "A runner asks for a check to run. Takes a runner token as the bearer",
88 detail:
9 "A runner asks for a check to run. Takes a runner token as the bearer (from POST /api/v1/admin/runners), never an account's. Held open up to wait seconds (25 at most); answers { job, token, lease } with the job's own token, or 204 when nothing was offered",
9 "A runner asks for a check to run. Takes a runner token as the bearer (from POST /api/v1/admin/runners), never an account's. Held open up to wait seconds (25 at most); jobs (1 to 64) is how many it runs at once, counted into checks.concurrency while it is seen; answers { job, token, lease } with the job's own token, or 204 when nothing was offered",
1010 auth: false,
1111 role: "anonymous",
12 body: { wait: "integer", cells: "boolean" },
12 body: { wait: "integer", cells: "boolean", jobs: "integer" },
1313 alsoStatus: [204],
1414 refusals: { unauthorized: "401: not a runner token this forge enrolled" },
1515 errorStatus: { 401: "`unauthorized`: not a runner token this forge enrolled" },
src/core/capacity.ts +1 −1
@@ -72,7 +72,7 @@ export const CAPACITY: Record<CapacityKey, Definition> = {
7272 fallback: "1",
7373 sized: (cpus) => checksForCpus(cpus),
7474 reason: "A check runs a whole test suite; more at once starve the web process of CPU.",
75 help: "How many repository checks run at once. Lowering it lets running ones finish; the next one waits. Empty: the pod's CPUs over three per shard (shards × 3), from 1 to 16 — 80 CPUs give 6.",
75 help: "How many repository checks run at once on this forge's own cells; each runner seen in the last minute adds the jobs it runs at once, up to 16 in all. Lowering it lets running ones finish; the next one waits. Empty: the pod's CPUs over three per shard (shards × 3), from 1 to 16 — 80 CPUs give 6.",
7676 },
7777 "checks.shards": {
7878 label: "Test shards per check",
src/core/migration-pins/20261121190000-check-runner-jobs.pin added +1
@@ -0,0 +1,1 @@
1 13fe381633fd481421dc119336082a048c03ca6b3f3acfc388c68a40d33d7c88
src/core/migrations/20261121190000-check-runner-jobs.ts added +6
@@ -0,0 +1,6 @@
1 // How many jobs a runner runs at once (#15253): what its claim says (`FORGE_RUNNER_JOBS`), so
2 // `checks.concurrency` can count a live runner's capacity. One until a runner says otherwise.
3 // Additive: one column.
4 export default `
5 ALTER TABLE check_runner ADD COLUMN jobs INTEGER NOT NULL DEFAULT 1;
6 `;
src/http/api/runners.ts +2
@@ -18,6 +18,7 @@ import {
1818 holdsClaim,
1919 jobByToken,
2020 nextOffer,
21 noteRunnerJobs,
2122 renewJob,
2223 runnerByToken,
2324 settleJob,
@@ -117,6 +118,7 @@ export function runnerRoutes(router: Router<Handler>): void {
117118 const input = await boundedBody(ctx.request, STRANGER_MAX);
118119 if (input === "too_large")
119120 return apiError("too_large", 413, `at most ${STRANGER_MAX} bytes`);
121 noteRunnerJobs(ctx.app.db, runner.id, input.jobs);
120122 const wait =
121123 Math.min(Math.max(Number(input.wait) || 0, 0), CLAIM_WAIT_MAX_S) * 1000;
122124 const until = Date.now() + wait;
src/runner/main.ts +4 −3
@@ -98,6 +98,8 @@ export async function runRunner(options: RunnerOptions): Promise<void> {
9898 const forge = options.forgeUrl.replace(/\/+$/, "");
9999 const stopping = options.signal ?? new AbortController().signal;
100100 const jobsRoot = join(options.root, "jobs");
101 // How many jobs run at once: one claim loop each, and what each claim tells the forge.
102 const slots = Math.min(Math.max(Math.trunc(options.jobs ?? 1), 1), 64);
101103 // What a runner that died mid-job left: no job is using it now.
102104 await rm(jobsRoot, { recursive: true, force: true }).catch(() => {});
103105 mkdirSync(jobsRoot, { recursive: true });
@@ -397,7 +399,7 @@ export async function runRunner(options: RunnerOptions): Promise<void> {
397399 res = await post(
398400 "/api/v1/runner/claim",
399401 options.token,
400 { wait: options.once === true ? 0 : CLAIM_WAIT_S, cells: cells.ok },
402 { wait: options.once === true ? 0 : CLAIM_WAIT_S, cells: cells.ok, jobs: slots },
401403 stopping,
402404 );
403405 } catch {
@@ -419,8 +421,7 @@ export async function runRunner(options: RunnerOptions): Promise<void> {
419421 }
420422 }
421423
422 const n = Math.min(Math.max(Math.trunc(options.jobs ?? 1), 1), 64);
423 await Promise.all(Array.from({ length: n }, () => loop()));
424 await Promise.all(Array.from({ length: slots }, () => loop()));
424425 }
425426
426427 if (import.meta.main) {
test/checks-runner-capacity.test.ts added +83
@@ -0,0 +1,83 @@
1 /**
2 * #15253: `checks.concurrency` counts a live runner's capacity. Under the runner placement the
3 * checks allowed at once are the setting plus the jobs of every runner seen within
4 * `SEEN_FRESH_MS`, never above the setting's own ceiling; a runner gone quiet stops counting
5 * on the next read, with no restart.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { CHECKS_CEILING, capacity, type LaneState } from "../src/checks/lane_state.ts";
13 import type { Placement } from "../src/checks/placement.ts";
14 import { enrolRunner, liveRunnerJobs, noteRunnerJobs, runnerByToken, SEEN_FRESH_MS } from "../src/checks/runner_jobs.ts";
15 import { CAPACITY, setCapacity } from "../src/core/capacity.ts";
16
17 let dataDir: string;
18 let app: App;
19 beforeAll(async () => {
20 dataDir = await mkdtemp(join(tmpdir(), "forge-runner-capacity-"));
21 app = createApp({ dataDir, port: 0 });
22 });
23 afterAll(async () => {
24 app.close();
25 await rm(dataDir, { recursive: true, force: true });
26 });
27
28 /** Only what `capacity` reads of a lane. */
29 const lane = (placement: Placement, concurrency = 2): LaneState =>
30 ({ app, options: { forgeUrl: "http://forge.test", concurrency }, placement }) as unknown as LaneState;
31
32 const T = 10_000_000;
33
34 test("the setting plus the jobs of live runners, under the runner placement only", () => {
35 const runner = lane({ mode: "runner", waitMs: 300 });
36 expect(capacity(runner, T)).toBe(2);
37 const one = enrolRunner(app.db, "one", "admin");
38 const two = enrolRunner(app.db, "two", "admin");
39 runnerByToken(app.db, one.token, T);
40 expect(capacity(runner, T)).toBe(3);
41 runnerByToken(app.db, two.token, T);
42 noteRunnerJobs(app.db, two.runner.id, 4);
43 expect(liveRunnerJobs(app.db, T)).toBe(5);
44 expect(capacity(runner, T)).toBe(7);
45 // An admin's setting is read on every pick, and the runners add to it.
46 setCapacity(app.db, "checks.concurrency", "3", "admin");
47 expect(capacity(runner, T)).toBe(8);
48 setCapacity(app.db, "checks.concurrency", null, "admin");
49 // Runners never take a check under the process placement: they add nothing there.
50 expect(capacity(lane({ mode: "process" }), T)).toBe(2);
51 });
52
53 test("a runner not seen within SEEN_FRESH_MS stops counting, with no restart", () => {
54 const runner = lane({ mode: "runner", waitMs: 300 });
55 expect(capacity(runner, T + SEEN_FRESH_MS - 1)).toBe(7);
56 expect(capacity(runner, T + SEEN_FRESH_MS)).toBe(2);
57 });
58
59 test("never above the setting's own ceiling", () => {
60 const big = enrolRunner(app.db, "big", "admin");
61 runnerByToken(app.db, big.token, T);
62 noteRunnerJobs(app.db, big.runner.id, 64);
63 const kind = CAPACITY["checks.concurrency"].kind;
64 expect(CHECKS_CEILING).toBe(kind.type === "count" ? kind.max : -1);
65 expect(capacity(lane({ mode: "runner", waitMs: 300 }), T)).toBe(CHECKS_CEILING);
66 });
67
68 test("a count a claim says is kept only when it is a whole number from 1 to 64", () => {
69 const r = enrolRunner(app.db, "said", "admin");
70 const jobs = (): number =>
71 app.db.query<{ jobs: number }, [string]>("SELECT jobs FROM check_runner WHERE id = ?1").get(r.runner.id)!.jobs;
72 expect(jobs()).toBe(1);
73 for (const bad of [0, 65, 2.5, "3", null, undefined, -1]) noteRunnerJobs(app.db, r.runner.id, bad);
74 expect(jobs()).toBe(1);
75 noteRunnerJobs(app.db, r.runner.id, 6);
76 expect(jobs()).toBe(6);
77 });
78
79 test("the setting's line says what it counts", () => {
80 expect(CAPACITY["checks.concurrency"].help).toStartWith(
81 "How many repository checks run at once on this forge's own cells; each runner seen in the last minute adds the jobs it runs at once, up to 16 in all.",
82 );
83 });
test/runner-claim.test.ts +13
@@ -58,3 +58,16 @@ test("seen_at is rewritten at most once per 5 s, and a clock set back is correct
5858 runnerByToken(f.app.db, token, 1_001_000);
5959 expect(seen()).toBe(1_001_000);
6060 });
61
62 test("a claim says how many jobs its runner runs at once, and the forge keeps it (#15253)", async () => {
63 const { token, runner } = enrolRunner(f.app.db, "four", f.alice.id);
64 const r = await fetch(`${f.servers.url}/api/v1/runner/claim`, {
65 method: "POST",
66 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
67 body: JSON.stringify({ wait: 0, jobs: 4 }),
68 });
69 await r.body?.cancel();
70 expect(
71 f.app.db.query<{ jobs: number }, [string]>("SELECT jobs FROM check_runner WHERE id = ?1").get(runner.id)!.jobs,
72 ).toBe(4);
73 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs