Public: anyone can read this; the crew can change it.
A runner's claim never blocks the forge: claims are cheap to wait on and their work is done when a check is queued #14887
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14883-a-runner-s-claim-never-blocks-the-forge → main 4 commits fork 1 thread · 1 unresolved
2 files
changed +74
−4
src/checks/runner_jobs.ts +14 −4
| @@ -30,6 +30,8 @@ export const JOB_TOKEN_PREFIX = "forge_job_"; | ||
| 30 | 30 | export const LEASE_MS = 30_000; |
| 31 | 31 | /** A runner that asked for work this recently is connected. */ |
| 32 | 32 | export const SEEN_FRESH_MS = 60_000; |
| 33 | /** A runner's `seen_at` is rewritten no more often than this. */ | |
| 34 | const SEEN_WRITE_MS = 5_000; | |
| 33 | 35 | |
| 34 | 36 | export interface RunnerStepSpec { |
| 35 | 37 | readonly name: "install" | "check"; |
| @@ -186,6 +188,9 @@ export function runnerByToken( | ||
| 186 | 188 | >("SELECT id, name, created_at, seen_at FROM check_runner WHERE token_hash = ?1") |
| 187 | 189 | .get(sha256(presented)); |
| 188 | 190 | if (row === null) return undefined; |
| 191 | // One write is a commit: a runner polling every few seconds is noted at most once per | |
| 192 | // `SEEN_WRITE_MS`, which is well inside `SEEN_FRESH_MS`. | |
| 193 | if (now - row.seen_at < SEEN_WRITE_MS && row.seen_at <= now) return toEnrolled(row); | |
| 189 | 194 | db.query("UPDATE check_runner SET seen_at = ?2 WHERE id = ?1").run( |
| 190 | 195 | row.id, |
| 191 | 196 | now, |
| @@ -248,11 +253,12 @@ export function claimJob( | ||
| 248 | 253 | now: number, |
| 249 | 254 | leaseMs: number = LEASE_MS, |
| 250 | 255 | ): Claim | undefined { |
| 256 | // Ids only: the spec (a job's whole build) is read for the one row this runner takes. | |
| 251 | 257 | const queued = db |
| 252 | 258 | .query< |
| 253 | { id: string; spec: string }, | |
| 259 | { id: string }, | |
| 254 | 260 | [number] |
| 255 | >("SELECT id, spec FROM check_job WHERE state = 'queued' AND (needs_cell = 0 OR ?1 = 1) ORDER BY id LIMIT 8") | |
| 261 | >("SELECT id FROM check_job WHERE state = 'queued' AND (needs_cell = 0 OR ?1 = 1) ORDER BY id LIMIT 8") | |
| 256 | 262 | .all(runner.cells ? 1 : 0); |
| 257 | 263 | for (const row of queued) { |
| 258 | 264 | const token = `${JOB_TOKEN_PREFIX}${secret()}`; |
| @@ -263,8 +269,12 @@ export function claimJob( | ||
| 263 | 269 | claims = claims + 1 WHERE id = ?1 AND state = 'queued'`, |
| 264 | 270 | ) |
| 265 | 271 | .run(row.id, runner.id, sha256(token), now + leaseMs).changes > 0; |
| 266 | if (taken) | |
| 267 | return { job: JSON.parse(row.spec) as RunnerJob, token, leaseMs }; | |
| 272 | if (taken) { | |
| 273 | const spec = db | |
| 274 | .query<{ spec: string }, [string]>("SELECT spec FROM check_job WHERE id = ?1") | |
| 275 | .get(row.id)!.spec; | |
| 276 | return { job: JSON.parse(spec) as RunnerJob, token, leaseMs }; | |
| 277 | } | |
| 268 | 278 | } |
| 269 | 279 | return undefined; |
| 270 | 280 | } |
test/runner-claim.test.ts added +60
| @@ -0,0 +1,60 @@ | ||
| 1 | /** | |
| 2 | * #14883: a runner's claim hands each queued job to exactly one of 20 runners, and a runner's | |
| 3 | * `seen_at` is rewritten at most once per 5 s. | |
| 4 | */ | |
| 5 | import { expect, test } from "bun:test"; | |
| 6 | import { enrolRunner, offerJob, runnerByToken, wakeClaimers, type RunnerJob } from "../src/checks/runner_jobs.ts"; | |
| 7 | import { checksForge } from "./helpers/checks-forge.ts"; | |
| 8 | import { quietLogs } from "./helpers/globals.ts"; | |
| 9 | ||
| 10 | quietLogs(); | |
| 11 | const f = checksForge("forge-claim-load-"); | |
| 12 | ||
| 13 | const job = (n: number): Omit<RunnerJob, "id"> => ({ | |
| 14 | run: `run${n}`, | |
| 15 | repo: "alice/lib", | |
| 16 | commit: "a".repeat(40), | |
| 17 | depth: 1, | |
| 18 | steps: [{ name: "check", command: "true" }], | |
| 19 | env: { CI: "1", PAD: "x".repeat(2_000) }, | |
| 20 | limits: {}, | |
| 21 | timeoutS: 60, | |
| 22 | egress: [], | |
| 23 | egressMode: "enforce", | |
| 24 | cell: "preferred", | |
| 25 | cache: { repo: "r", tree: "", base: "" }, | |
| 26 | }); | |
| 27 | ||
| 28 | test("20 runners claiming 50 jobs: each job is claimed exactly once", async () => { | |
| 29 | const tokens = Array.from({ length: 20 }, (_, i) => enrolRunner(f.app.db, `r${i}`, f.alice.id).token); | |
| 30 | const claim = (token: string): Promise<Response> => | |
| 31 | fetch(`${f.servers.url}/api/v1/runner/claim`, { | |
| 32 | method: "POST", | |
| 33 | headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, | |
| 34 | body: JSON.stringify({ wait: 0 }), | |
| 35 | }); | |
| 36 | for (let i = 0; i < 50; i++) offerJob(f.app.db, { runId: `run${i}`, repoId: "r", dir: "/nowhere", job: job(i) }); | |
| 37 | wakeClaimers(f.app); | |
| 38 | const got: string[] = []; | |
| 39 | for (let round = 0; round < 3; round++) { | |
| 40 | const rs = await Promise.all(tokens.map(claim)); | |
| 41 | for (const r of rs) if (r.status === 200) got.push(((await r.json()) as { job: { id: string } }).job.id); | |
| 42 | } | |
| 43 | expect(got.length).toBe(50); | |
| 44 | expect(new Set(got).size).toBe(50); | |
| 45 | }, 30_000); | |
| 46 | ||
| 47 | test("seen_at is rewritten at most once per 5 s, and a clock set back is corrected", () => { | |
| 48 | const { token, runner } = enrolRunner(f.app.db, "seen", f.alice.id); | |
| 49 | const seen = (): number => | |
| 50 | f.app.db.query<{ seen_at: number }, [string]>("SELECT seen_at FROM check_runner WHERE id = ?1").get(runner.id)!.seen_at; | |
| 51 | runnerByToken(f.app.db, token, 1_000_000); | |
| 52 | expect(seen()).toBe(1_000_000); | |
| 53 | runnerByToken(f.app.db, token, 1_004_999); | |
| 54 | expect(seen()).toBe(1_000_000); | |
| 55 | runnerByToken(f.app.db, token, 1_005_000); | |
| 56 | expect(seen()).toBe(1_005_000); | |
| 57 | // seen_at in the future of now (clock skew): written, not held back. | |
| 58 | runnerByToken(f.app.db, token, 1_001_000); | |
| 59 | expect(seen()).toBe(1_001_000); | |
| 60 | }); |
new above
offline
The issue number here (#14883) differs from this pull (#14887). Make it match the issue it closes.