Public: anyone can read this; the crew can change it.
checks.concurrency counts a runner's capacity, so adding a runner raises how many checks run at once #15268
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/15253-checks-concurrency-counts-a-runner-s-cap → main 5 commits fork
11 files
changed +158
−12
docs/features/runners.md +8 −2
| @@ -34,8 +34,14 @@ started by hand are in addition: an admin removes them with | ||
| 34 | 34 | `DELETE /api/v1/admin/runners/<id>`. Every `local-*` row created by `forge` is |
| 35 | 35 | deleted at each boot, so never name a hand-made runner `local-N`. |
| 36 | 36 | |
| 37 | Not yet: a page for them, `checks.concurrency` counting runner capacity, a job | |
| 38 | that survives a restart, caches by hash (#14201's follow-ups). | |
| 37 | `checks.concurrency` counts the forge's own cells; under the runner placement | |
| 38 | each runner seen in the last minute adds the jobs it runs at once | |
| 39 | (`FORGE_RUNNER_JOBS`, which every claim tells the forge), up to the setting's | |
| 40 | ceiling of 16. One that stops asking stops counting a minute later, with no | |
| 41 | restart (#15253). | |
| 42 | ||
| 43 | Not yet: a page for them, a job that survives a restart, caches by hash | |
| 44 | (#14201's follow-ups). | |
| 39 | 45 | |
| 40 | 46 | ## Start one, in two lines |
| 41 | 47 |
src/checks/lane_state.ts +14 −4
| @@ -7,7 +7,8 @@ import type { Run } from "../core/runs.ts"; | ||
| 7 | 7 | import { Gate } from "../git/env.ts"; |
| 8 | 8 | import { runStep } from "../apps/process.ts"; |
| 9 | 9 | import type { DeepsecDeps } from "./deepsec_step.ts"; |
| 10 | import { capacityCount } from "../core/capacity.ts"; | |
| 10 | import { CAPACITY, capacityCount } from "../core/capacity.ts"; | |
| 11 | import { liveRunnerJobs } from "./runner_jobs.ts"; | |
| 11 | 12 | import type { Placement } from "./placement.ts"; |
| 12 | 13 | import type { PodRoom } from "./pod_room.ts"; |
| 13 | 14 | import type { GateStage } from "./gate_reason.ts"; |
| @@ -213,9 +214,18 @@ export interface LaneState { | ||
| 213 | 214 | export const slotted = (lane: LaneState): number => |
| 214 | 215 | lane.inFlight.size - (lane.verifying === undefined ? 0 : 1); |
| 215 | 216 | |
| 217 | /** The most checks at once however many runners are live: the setting's own ceiling. */ | |
| 218 | export const CHECKS_CEILING = ((kind) => (kind.type === "count" ? kind.max : 16))(CAPACITY["checks.concurrency"].kind); | |
| 219 | ||
| 216 | 220 | /** |
| 217 | 221 | * How many checks may run at once, read on every pick (`core/capacity.ts`): an admin's change is |
| 218 | * honoured by the next check, not the next boot. `concurrency` is the boot default. | |
| 222 | * honoured by the next check, not the next boot. `concurrency` is the boot default. Under the | |
| 223 | * runner placement the jobs of every live runner add to it (`liveRunnerJobs`), so one that | |
| 224 | * connects raises it at the next tick and one gone quiet drops out once `SEEN_FRESH_MS` passes; | |
| 225 | * never above `CHECKS_CEILING`, so a crowd of runners cannot overload this machine's lane. | |
| 219 | 226 | */ |
| 220 | export const capacity = (lane: LaneState): number => | |
| 221 | capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1); | |
| 227 | export function capacity(lane: LaneState, now: number = Date.now()): number { | |
| 228 | const setting = capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1); | |
| 229 | const runners = lane.placement.mode === "runner" ? liveRunnerJobs(lane.app.db, now) : 0; | |
| 230 | return Math.min(CHECKS_CEILING, setting + runners); | |
| 231 | } |
src/checks/runner_jobs.ts +24
| @@ -210,6 +210,30 @@ export function runnersAround(db: Database, now: number): boolean { | ||
| 210 | 210 | ); |
| 211 | 211 | } |
| 212 | 212 | |
| 213 | /** The most jobs one runner may say it runs at once, as the runner itself clamps it. */ | |
| 214 | export const MAX_RUNNER_JOBS = 64; | |
| 215 | ||
| 216 | /** | |
| 217 | * Keep what a runner's claim says it runs at once (`FORGE_RUNNER_JOBS`); anything but a whole | |
| 218 | * number from 1 to `MAX_RUNNER_JOBS` is ignored, and an unchanged count writes nothing. | |
| 219 | */ | |
| 220 | export function noteRunnerJobs(db: Database, runnerId: string, said: unknown): void { | |
| 221 | if (typeof said !== "number" || !Number.isInteger(said) || said < 1 || said > MAX_RUNNER_JOBS) return; | |
| 222 | db.query("UPDATE check_runner SET jobs = ?2 WHERE id = ?1 AND jobs != ?2").run(runnerId, said); | |
| 223 | } | |
| 224 | ||
| 225 | /** | |
| 226 | * The jobs the live runners run at once: the sum of each one's count over those seen within | |
| 227 | * `SEEN_FRESH_MS`. One not seen since stops counting on the next read, with no restart. | |
| 228 | */ | |
| 229 | export function liveRunnerJobs(db: Database, now: number): number { | |
| 230 | return ( | |
| 231 | db | |
| 232 | .query<{ n: number | null }, [number]>("SELECT SUM(jobs) AS n FROM check_runner WHERE seen_at > ?1") | |
| 233 | .get(now - SEEN_FRESH_MS)?.n ?? 0 | |
| 234 | ); | |
| 235 | } | |
| 236 | ||
| 213 | 237 | /** |
| 214 | 238 | * Put a check's steps on the desk. `dir` is the bare repository the commit is in; it stays here. |
| 215 | 239 | */ |
src/contract/openapi/routes/postApiV1RunnerClaim.ts +2 −2
| @@ -6,10 +6,10 @@ export const route: Route = { | ||
| 6 | 6 | summary: |
| 7 | 7 | "A runner asks for a check to run. Takes a runner token as the bearer", |
| 8 | 8 | detail: |
| 9 | "A runner asks for a check to run. Takes a runner token as the bearer (from POST /api/v1/admin/runners), never an account's. Held open up to wait seconds (25 at most); answers { job, token, lease } with the job's own token, or 204 when nothing was offered", | |
| 9 | "A runner asks for a check to run. Takes a runner token as the bearer (from POST /api/v1/admin/runners), never an account's. Held open up to wait seconds (25 at most); jobs (1 to 64) is how many it runs at once, counted into checks.concurrency while it is seen; answers { job, token, lease } with the job's own token, or 204 when nothing was offered", | |
| 10 | 10 | auth: false, |
| 11 | 11 | role: "anonymous", |
| 12 | body: { wait: "integer", cells: "boolean" }, | |
| 12 | body: { wait: "integer", cells: "boolean", jobs: "integer" }, | |
| 13 | 13 | alsoStatus: [204], |
| 14 | 14 | refusals: { unauthorized: "401: not a runner token this forge enrolled" }, |
| 15 | 15 | errorStatus: { 401: "`unauthorized`: not a runner token this forge enrolled" }, |
src/core/capacity.ts +1 −1
| @@ -72,7 +72,7 @@ export const CAPACITY: Record<CapacityKey, Definition> = { | ||
| 72 | 72 | fallback: "1", |
| 73 | 73 | sized: (cpus) => checksForCpus(cpus), |
| 74 | 74 | reason: "A check runs a whole test suite; more at once starve the web process of CPU.", |
| 75 | help: "How many repository checks run at once. Lowering it lets running ones finish; the next one waits. Empty: the pod's CPUs over three per shard (shards × 3), from 1 to 16 — 80 CPUs give 6.", | |
| 75 | help: "How many repository checks run at once on this forge's own cells; each runner seen in the last minute adds the jobs it runs at once, up to 16 in all. Lowering it lets running ones finish; the next one waits. Empty: the pod's CPUs over three per shard (shards × 3), from 1 to 16 — 80 CPUs give 6.", | |
| 76 | 76 | }, |
| 77 | 77 | "checks.shards": { |
| 78 | 78 | label: "Test shards per check", |
src/core/migration-pins/20261121190000-check-runner-jobs.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 13fe381633fd481421dc119336082a048c03ca6b3f3acfc388c68a40d33d7c88 |
src/core/migrations/20261121190000-check-runner-jobs.ts added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | // How many jobs a runner runs at once (#15253): what its claim says (`FORGE_RUNNER_JOBS`), so | |
| 2 | // `checks.concurrency` can count a live runner's capacity. One until a runner says otherwise. | |
| 3 | // Additive: one column. | |
| 4 | export default ` | |
| 5 | ALTER TABLE check_runner ADD COLUMN jobs INTEGER NOT NULL DEFAULT 1; | |
| 6 | `; |
src/http/api/runners.ts +2
| @@ -18,6 +18,7 @@ import { | ||
| 18 | 18 | holdsClaim, |
| 19 | 19 | jobByToken, |
| 20 | 20 | nextOffer, |
| 21 | noteRunnerJobs, | |
| 21 | 22 | renewJob, |
| 22 | 23 | runnerByToken, |
| 23 | 24 | settleJob, |
| @@ -117,6 +118,7 @@ export function runnerRoutes(router: Router<Handler>): void { | ||
| 117 | 118 | const input = await boundedBody(ctx.request, STRANGER_MAX); |
| 118 | 119 | if (input === "too_large") |
| 119 | 120 | return apiError("too_large", 413, `at most ${STRANGER_MAX} bytes`); |
| 121 | noteRunnerJobs(ctx.app.db, runner.id, input.jobs); | |
| 120 | 122 | const wait = |
| 121 | 123 | Math.min(Math.max(Number(input.wait) || 0, 0), CLAIM_WAIT_MAX_S) * 1000; |
| 122 | 124 | const until = Date.now() + wait; |
src/runner/main.ts +4 −3
| @@ -98,6 +98,8 @@ export async function runRunner(options: RunnerOptions): Promise<void> { | ||
| 98 | 98 | const forge = options.forgeUrl.replace(/\/+$/, ""); |
| 99 | 99 | const stopping = options.signal ?? new AbortController().signal; |
| 100 | 100 | const jobsRoot = join(options.root, "jobs"); |
| 101 | // How many jobs run at once: one claim loop each, and what each claim tells the forge. | |
| 102 | const slots = Math.min(Math.max(Math.trunc(options.jobs ?? 1), 1), 64); | |
| 101 | 103 | // What a runner that died mid-job left: no job is using it now. |
| 102 | 104 | await rm(jobsRoot, { recursive: true, force: true }).catch(() => {}); |
| 103 | 105 | mkdirSync(jobsRoot, { recursive: true }); |
| @@ -397,7 +399,7 @@ export async function runRunner(options: RunnerOptions): Promise<void> { | ||
| 397 | 399 | res = await post( |
| 398 | 400 | "/api/v1/runner/claim", |
| 399 | 401 | options.token, |
| 400 | { wait: options.once === true ? 0 : CLAIM_WAIT_S, cells: cells.ok }, | |
| 402 | { wait: options.once === true ? 0 : CLAIM_WAIT_S, cells: cells.ok, jobs: slots }, | |
| 401 | 403 | stopping, |
| 402 | 404 | ); |
| 403 | 405 | } catch { |
| @@ -419,8 +421,7 @@ export async function runRunner(options: RunnerOptions): Promise<void> { | ||
| 419 | 421 | } |
| 420 | 422 | } |
| 421 | 423 | |
| 422 | const n = Math.min(Math.max(Math.trunc(options.jobs ?? 1), 1), 64); | |
| 423 | await Promise.all(Array.from({ length: n }, () => loop())); | |
| 424 | await Promise.all(Array.from({ length: slots }, () => loop())); | |
| 424 | 425 | } |
| 425 | 426 | |
| 426 | 427 | if (import.meta.main) { |
test/checks-runner-capacity.test.ts added +83
| @@ -0,0 +1,83 @@ | ||
| 1 | /** | |
| 2 | * #15253: `checks.concurrency` counts a live runner's capacity. Under the runner placement the | |
| 3 | * checks allowed at once are the setting plus the jobs of every runner seen within | |
| 4 | * `SEEN_FRESH_MS`, never above the setting's own ceiling; a runner gone quiet stops counting | |
| 5 | * on the next read, with no restart. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { CHECKS_CEILING, capacity, type LaneState } from "../src/checks/lane_state.ts"; | |
| 13 | import type { Placement } from "../src/checks/placement.ts"; | |
| 14 | import { enrolRunner, liveRunnerJobs, noteRunnerJobs, runnerByToken, SEEN_FRESH_MS } from "../src/checks/runner_jobs.ts"; | |
| 15 | import { CAPACITY, setCapacity } from "../src/core/capacity.ts"; | |
| 16 | ||
| 17 | let dataDir: string; | |
| 18 | let app: App; | |
| 19 | beforeAll(async () => { | |
| 20 | dataDir = await mkdtemp(join(tmpdir(), "forge-runner-capacity-")); | |
| 21 | app = createApp({ dataDir, port: 0 }); | |
| 22 | }); | |
| 23 | afterAll(async () => { | |
| 24 | app.close(); | |
| 25 | await rm(dataDir, { recursive: true, force: true }); | |
| 26 | }); | |
| 27 | ||
| 28 | /** Only what `capacity` reads of a lane. */ | |
| 29 | const lane = (placement: Placement, concurrency = 2): LaneState => | |
| 30 | ({ app, options: { forgeUrl: "http://forge.test", concurrency }, placement }) as unknown as LaneState; | |
| 31 | ||
| 32 | const T = 10_000_000; | |
| 33 | ||
| 34 | test("the setting plus the jobs of live runners, under the runner placement only", () => { | |
| 35 | const runner = lane({ mode: "runner", waitMs: 300 }); | |
| 36 | expect(capacity(runner, T)).toBe(2); | |
| 37 | const one = enrolRunner(app.db, "one", "admin"); | |
| 38 | const two = enrolRunner(app.db, "two", "admin"); | |
| 39 | runnerByToken(app.db, one.token, T); | |
| 40 | expect(capacity(runner, T)).toBe(3); | |
| 41 | runnerByToken(app.db, two.token, T); | |
| 42 | noteRunnerJobs(app.db, two.runner.id, 4); | |
| 43 | expect(liveRunnerJobs(app.db, T)).toBe(5); | |
| 44 | expect(capacity(runner, T)).toBe(7); | |
| 45 | // An admin's setting is read on every pick, and the runners add to it. | |
| 46 | setCapacity(app.db, "checks.concurrency", "3", "admin"); | |
| 47 | expect(capacity(runner, T)).toBe(8); | |
| 48 | setCapacity(app.db, "checks.concurrency", null, "admin"); | |
| 49 | // Runners never take a check under the process placement: they add nothing there. | |
| 50 | expect(capacity(lane({ mode: "process" }), T)).toBe(2); | |
| 51 | }); | |
| 52 | ||
| 53 | test("a runner not seen within SEEN_FRESH_MS stops counting, with no restart", () => { | |
| 54 | const runner = lane({ mode: "runner", waitMs: 300 }); | |
| 55 | expect(capacity(runner, T + SEEN_FRESH_MS - 1)).toBe(7); | |
| 56 | expect(capacity(runner, T + SEEN_FRESH_MS)).toBe(2); | |
| 57 | }); | |
| 58 | ||
| 59 | test("never above the setting's own ceiling", () => { | |
| 60 | const big = enrolRunner(app.db, "big", "admin"); | |
| 61 | runnerByToken(app.db, big.token, T); | |
| 62 | noteRunnerJobs(app.db, big.runner.id, 64); | |
| 63 | const kind = CAPACITY["checks.concurrency"].kind; | |
| 64 | expect(CHECKS_CEILING).toBe(kind.type === "count" ? kind.max : -1); | |
| 65 | expect(capacity(lane({ mode: "runner", waitMs: 300 }), T)).toBe(CHECKS_CEILING); | |
| 66 | }); | |
| 67 | ||
| 68 | test("a count a claim says is kept only when it is a whole number from 1 to 64", () => { | |
| 69 | const r = enrolRunner(app.db, "said", "admin"); | |
| 70 | const jobs = (): number => | |
| 71 | app.db.query<{ jobs: number }, [string]>("SELECT jobs FROM check_runner WHERE id = ?1").get(r.runner.id)!.jobs; | |
| 72 | expect(jobs()).toBe(1); | |
| 73 | for (const bad of [0, 65, 2.5, "3", null, undefined, -1]) noteRunnerJobs(app.db, r.runner.id, bad); | |
| 74 | expect(jobs()).toBe(1); | |
| 75 | noteRunnerJobs(app.db, r.runner.id, 6); | |
| 76 | expect(jobs()).toBe(6); | |
| 77 | }); | |
| 78 | ||
| 79 | test("the setting's line says what it counts", () => { | |
| 80 | expect(CAPACITY["checks.concurrency"].help).toStartWith( | |
| 81 | "How many repository checks run at once on this forge's own cells; each runner seen in the last minute adds the jobs it runs at once, up to 16 in all.", | |
| 82 | ); | |
| 83 | }); |
test/runner-claim.test.ts +13
| @@ -58,3 +58,16 @@ test("seen_at is rewritten at most once per 5 s, and a clock set back is correct | ||
| 58 | 58 | runnerByToken(f.app.db, token, 1_001_000); |
| 59 | 59 | expect(seen()).toBe(1_001_000); |
| 60 | 60 | }); |
| 61 | ||
| 62 | test("a claim says how many jobs its runner runs at once, and the forge keeps it (#15253)", async () => { | |
| 63 | const { token, runner } = enrolRunner(f.app.db, "four", f.alice.id); | |
| 64 | const r = await fetch(`${f.servers.url}/api/v1/runner/claim`, { | |
| 65 | method: "POST", | |
| 66 | headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, | |
| 67 | body: JSON.stringify({ wait: 0, jobs: 4 }), | |
| 68 | }); | |
| 69 | await r.body?.cancel(); | |
| 70 | expect( | |
| 71 | f.app.db.query<{ jobs: number }, [string]>("SELECT jobs FROM check_runner WHERE id = ?1").get(runner.id)!.jobs, | |
| 72 | ).toBe(4); | |
| 73 | }); |
new above
offline