Public: anyone can read this; the crew can change it.

A runner's claim never blocks the forge: claims are cheap to wait on and their work is done when a check is queued #14887

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14883-a-runner-s-claim-never-blocks-the-forge → main 4 commits fork 1 thread · 1 unresolved

  • Merged. mason/14883-a-runner-s-claim-never-blocks-the-forge was merged into main as 72d41e9.

Review

Sign in to review this change.
2 files changed +74 −4
src/checks/runner_jobs.ts +14 −4
@@ -30,6 +30,8 @@ export const JOB_TOKEN_PREFIX = "forge_job_";
3030 export const LEASE_MS = 30_000;
3131 /** A runner that asked for work this recently is connected. */
3232 export const SEEN_FRESH_MS = 60_000;
33 /** A runner's `seen_at` is rewritten no more often than this. */
34 const SEEN_WRITE_MS = 5_000;
3335
3436 export interface RunnerStepSpec {
3537 readonly name: "install" | "check";
@@ -186,6 +188,9 @@ export function runnerByToken(
186188 >("SELECT id, name, created_at, seen_at FROM check_runner WHERE token_hash = ?1")
187189 .get(sha256(presented));
188190 if (row === null) return undefined;
191 // One write is a commit: a runner polling every few seconds is noted at most once per
192 // `SEEN_WRITE_MS`, which is well inside `SEEN_FRESH_MS`.
193 if (now - row.seen_at < SEEN_WRITE_MS && row.seen_at <= now) return toEnrolled(row);
189194 db.query("UPDATE check_runner SET seen_at = ?2 WHERE id = ?1").run(
190195 row.id,
191196 now,
@@ -248,11 +253,12 @@ export function claimJob(
248253 now: number,
249254 leaseMs: number = LEASE_MS,
250255 ): Claim | undefined {
256 // Ids only: the spec (a job's whole build) is read for the one row this runner takes.
251257 const queued = db
252258 .query<
253 { id: string; spec: string },
259 { id: string },
254260 [number]
255 >("SELECT id, spec FROM check_job WHERE state = 'queued' AND (needs_cell = 0 OR ?1 = 1) ORDER BY id LIMIT 8")
261 >("SELECT id FROM check_job WHERE state = 'queued' AND (needs_cell = 0 OR ?1 = 1) ORDER BY id LIMIT 8")
256262 .all(runner.cells ? 1 : 0);
257263 for (const row of queued) {
258264 const token = `${JOB_TOKEN_PREFIX}${secret()}`;
@@ -263,8 +269,12 @@ export function claimJob(
263269 claims = claims + 1 WHERE id = ?1 AND state = 'queued'`,
264270 )
265271 .run(row.id, runner.id, sha256(token), now + leaseMs).changes > 0;
266 if (taken)
267 return { job: JSON.parse(row.spec) as RunnerJob, token, leaseMs };
272 if (taken) {
273 const spec = db
274 .query<{ spec: string }, [string]>("SELECT spec FROM check_job WHERE id = ?1")
275 .get(row.id)!.spec;
276 return { job: JSON.parse(spec) as RunnerJob, token, leaseMs };
277 }
268278 }
269279 return undefined;
270280 }
test/runner-claim.test.ts added +60
@@ -0,0 +1,60 @@
1 /**
2 * #14883: a runner's claim hands each queued job to exactly one of 20 runners, and a runner's

The issue number here (#14883) differs from this pull (#14887). Make it match the issue it closes.

3 * `seen_at` is rewritten at most once per 5 s.
4 */
5 import { expect, test } from "bun:test";
6 import { enrolRunner, offerJob, runnerByToken, wakeClaimers, type RunnerJob } from "../src/checks/runner_jobs.ts";
7 import { checksForge } from "./helpers/checks-forge.ts";
8 import { quietLogs } from "./helpers/globals.ts";
9
10 quietLogs();
11 const f = checksForge("forge-claim-load-");
12
13 const job = (n: number): Omit<RunnerJob, "id"> => ({
14 run: `run${n}`,
15 repo: "alice/lib",
16 commit: "a".repeat(40),
17 depth: 1,
18 steps: [{ name: "check", command: "true" }],
19 env: { CI: "1", PAD: "x".repeat(2_000) },
20 limits: {},
21 timeoutS: 60,
22 egress: [],
23 egressMode: "enforce",
24 cell: "preferred",
25 cache: { repo: "r", tree: "", base: "" },
26 });
27
28 test("20 runners claiming 50 jobs: each job is claimed exactly once", async () => {
29 const tokens = Array.from({ length: 20 }, (_, i) => enrolRunner(f.app.db, `r${i}`, f.alice.id).token);
30 const claim = (token: string): Promise<Response> =>
31 fetch(`${f.servers.url}/api/v1/runner/claim`, {
32 method: "POST",
33 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
34 body: JSON.stringify({ wait: 0 }),
35 });
36 for (let i = 0; i < 50; i++) offerJob(f.app.db, { runId: `run${i}`, repoId: "r", dir: "/nowhere", job: job(i) });
37 wakeClaimers(f.app);
38 const got: string[] = [];
39 for (let round = 0; round < 3; round++) {
40 const rs = await Promise.all(tokens.map(claim));
41 for (const r of rs) if (r.status === 200) got.push(((await r.json()) as { job: { id: string } }).job.id);
42 }
43 expect(got.length).toBe(50);
44 expect(new Set(got).size).toBe(50);
45 }, 30_000);
46
47 test("seen_at is rewritten at most once per 5 s, and a clock set back is corrected", () => {
48 const { token, runner } = enrolRunner(f.app.db, "seen", f.alice.id);
49 const seen = (): number =>
50 f.app.db.query<{ seen_at: number }, [string]>("SELECT seen_at FROM check_runner WHERE id = ?1").get(runner.id)!.seen_at;
51 runnerByToken(f.app.db, token, 1_000_000);
52 expect(seen()).toBe(1_000_000);
53 runnerByToken(f.app.db, token, 1_004_999);
54 expect(seen()).toBe(1_000_000);
55 runnerByToken(f.app.db, token, 1_005_000);
56 expect(seen()).toBe(1_005_000);
57 // seen_at in the future of now (clock skew): written, not held back.
58 runnerByToken(f.app.db, token, 1_001_000);
59 expect(seen()).toBe(1_001_000);
60 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs