Public: anyone can read this; the crew can change it.

Compute leaves the forge: runners that join and pull work, pools that scale on the queue, SLAs that can be measured #13672

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13652-compute-leaves-the-forge-runners-that-jo → main 4 commits fork

  • Merged. mason/13652-compute-leaves-the-forge-runners-that-jo was merged into main as 4e8e801.

Review

Sign in to review this change.
6 files changed +378 −86
AGENTS.md +2

Diff not shown — a line 2,989 characters wide (limit 2,000). +2

View file
src/checks/runner.ts added +119
@@ -0,0 +1,119 @@
1 /**
2 * The seam between the check queue and whatever runs a check (#13652, R0).
3 *
4 * The queue picks a run, claims it and keeps its lease; it then hands "a runner" a `CheckJob`
5 * and hears an outcome. A job is plain data — ids, a commit, a tier, flags — with no database
6 * handle, path or callback in it, so it survives `JSON.stringify` and could cross a wire. What a
7 * runner needs besides (where the log goes, when to stop) is `RunnerIo`. Today the one runner is
8 * this process (`service.ts` builds it); nothing here changes what a check does.
9 */
10
11 import type { CheckDetail, Run } from "../core/runs.ts";
12 import type { GateStage } from "./gate_reason.ts";
13
14 /**
15 * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after
16 * landing.
17 */
18 export type CheckTier = "run" | "fast" | "full" | "verify";
19
20 export interface CheckJob {
21 readonly runId: string;
22 readonly repoId: string;
23 readonly commitOid: string;
24 /** The pull's issue the check is for; null for a branch's own check. */
25 readonly issueId: string | null;
26 readonly tier: CheckTier;
27 /** A bisection step: only these test files run. */
28 readonly only?: readonly string[];
29 /** A bisection step: the commit whose `check.verify` it runs. */
30 readonly commandsFrom?: string;
31 /** The default branch's own push: its cache is kept. */
32 readonly keep: boolean;
33 /** A green run keeps the impact map it measured. */
34 readonly keepImpact: boolean;
35 /** The default branch's own run: only its log may open issues. */
36 readonly fileOrderDependent: boolean;
37 /** The default branch's own run: a failure of one or two tests is re-run alone first. */
38 readonly isMain: boolean;
39 }
40
41 /** What a runner has: a place to say what it is doing, and the word to stop. */
42 export interface RunnerIo {
43 readonly write: (line: string) => void;
44 readonly signal: AbortSignal;
45 }
46
47 export type CheckOutcome =
48 | {
49 ok: true;
50 proven?: string;
51 passed?: readonly string[];
52 /** A red it cleared as main's (#10304): `base_red.ts`'s words, kept in the summary. */
53 cleared?: string;
54 }
55 | {
56 ok: false;
57 error: string;
58 detail?: CheckDetail | undefined;
59 /** A whole verification: the files its log ran to the end green (#9524). */
60 passed?: readonly string[];
61 infrastructure?: boolean;
62 /** The check step's error when the cell's watch stopped it for CPU (#11617). */
63 cpuStop?: string;
64 /** Where it stopped red (#5276); none is before any step ran. */
65 stage?: GateStage;
66 /** The check step was ended by a signal from outside: no verdict (#6722). */
67 killed?: boolean;
68 };
69
70 export interface Runner {
71 /** Who ran it, as a result records it. */
72 readonly name: string;
73 run(job: CheckJob, io: RunnerIo): Promise<CheckOutcome>;
74 }
75
76 /** The runner that is the forge's own process, always there. */
77 export const LOCAL_RUNNER = "local";
78
79 /**
80 * What a claimed check run asks of a runner. `origin` is the red verification a bisection step
81 * searches (`culprit.ts`); without it such a step has no job, and the words say why.
82 */
83 export function describeJob(
84 run: Pick<Run, "id" | "repoId" | "commitOid" | "issueId" | "env">,
85 facts: {
86 readonly defaultBranch: string;
87 readonly verifyEnv: string;
88 readonly inVerifyLane: boolean;
89 readonly pulled: boolean;
90 readonly origin?: { readonly files: readonly string[]; readonly commitOid: string } | undefined;
91 },
92 ): CheckJob | { error: string } {
93 const base = { runId: run.id, repoId: run.repoId, commitOid: run.commitOid, issueId: run.issueId };
94 if (facts.inVerifyLane) {
95 const whole = run.env === facts.verifyEnv;
96 if (!whole && facts.origin === undefined)
97 return { error: "the red verification this step bisects is gone" };
98 return {
99 ...base,
100 tier: "verify",
101 ...(whole || facts.origin === undefined
102 ? {}
103 : { only: facts.origin.files, commandsFrom: facts.origin.commitOid }),
104 keep: false,
105 keepImpact: whole,
106 fileOrderDependent: whole,
107 isMain: true,
108 };
109 }
110 const main = run.issueId === null && run.env === facts.defaultBranch;
111 return {
112 ...base,
113 tier: facts.pulled ? "fast" : "run",
114 keep: main,
115 keepImpact: !facts.pulled && run.env === facts.defaultBranch,
116 fileOrderDependent: main,
117 isMain: main,
118 };
119 }
src/checks/service.ts +91 −86
@@ -150,6 +150,7 @@ import {
150150 import { evictSoon, lend } from "./install_evict.ts";
151151 import { checkStage, type GateStage } from "./gate_reason.ts";
152152 import { rankOnce } from "./ranked_first.ts";
153 import { describeJob, LOCAL_RUNNER, type CheckOutcome, type Runner } from "./runner.ts";
153154 import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts";
154155 import type { JobOptions } from "../cluster/jobs.ts";
155156 import type { CheckDetail } from "../core/runs.ts";
@@ -218,6 +219,8 @@ export interface ChecksOptions {
218219 * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`.
219220 */
220221 readonly verifyEveryMs?: number | undefined;
222 /** Test seam: stands in for the tier the local runner runs, to see what reaches it. */
223 readonly runTier?: ((x: Execution) => Promise<CheckOutcome>) | undefined;
221224 /** Test seam: how often the queue is looked at. @default TICK_MS */
222225 readonly tickMs?: number | undefined;
223226 /** The forge's CPU count; a 1-CPU forge holds pull checks for a crew run. @default forgeCpus */
@@ -229,6 +232,73 @@ export interface ChecksOptions {
229232 readonly holdNow?: (() => number) | undefined;
230233 }
231234
235 /** What one tier run is given (`runTier`). */
236 export interface Execution {
237 readonly bare: string;
238 readonly repo: Repo;
239 /** Names the checkout. */
240 readonly id: string;
241 readonly runId?: string;
242 readonly commitOid: string;
243 readonly pull?: Pull | undefined;
244 /**
245 * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after
246 * landing.
247 */
248 readonly tier: "run" | "fast" | "full" | "verify";
249 /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */
250 readonly only?: readonly string[];
251 /** A bisection step: the commit whose `check.verify` it runs. */
252 readonly commandsFrom?: string;
253 /** A check of the default branch's own push: its cache is kept. */
254 readonly keep?: boolean;
255 /** A green run keeps the impact map it measured; `full` always does. */
256 readonly keepImpact?: boolean;
257 /** The default branch's own run: only its log may open issues. */
258 readonly fileOrderDependent?: boolean;
259 /**
260 * The default branch's own run: a failure of one or two tests is re-run alone before it counts
261 * as red.
262 */
263 readonly isMain?: boolean;
264 readonly write: (line: string) => void;
265 readonly signal: AbortSignal;
266 }
267
268 /** The runner every check goes to: this process (`runner.ts`); `tier` runs one described check. */
269 export function makeLocalRunner(app: App, tier: (x: Execution) => Promise<CheckOutcome>): Runner {
270 return {
271 name: LOCAL_RUNNER,
272 async run(job, io) {
273 const repo = getRepoById(app.db, job.repoId);
274 if (repo === undefined) return { ok: false, error: "the repository is gone" };
275 const bare = await runDir(app.db, app.reposRoot, repo, job.issueId);
276 // Only a pull's own check is judged against the pull, resolved here and nowhere else; the
277 // verification lane never is.
278 const issue =
279 job.tier === "fast" && job.issueId !== null ? getIssueById(app.db, job.issueId) : undefined;
280 const pull = issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined;
281 return tier({
282 bare,
283 repo,
284 id: job.runId,
285 runId: job.runId,
286 commitOid: job.commitOid,
287 pull,
288 tier: job.tier,
289 ...(job.only === undefined ? {} : { only: job.only }),
290 ...(job.commandsFrom === undefined ? {} : { commandsFrom: job.commandsFrom }),
291 keep: job.keep,
292 keepImpact: job.keepImpact,
293 fileOrderDependent: job.fileOrderDependent,
294 isMain: job.isMain,
295 write: io.write,
296 signal: io.signal,
297 });
298 },
299 };
300 }
301
232302 /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */
233303 export interface GateDone {
234304 ok: boolean;
@@ -1085,98 +1155,33 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
10851155 }
10861156 }
10871157
1158 const localRunner = makeLocalRunner(app, (x) => (options.runTier ?? runTier)(x));
1159
1160 /** Describe a claimed check as data and hand it to the runner. */
10881161 async function check(
10891162 run: Run,
10901163 repo: Repo,
10911164 write: (line: string) => void,
10921165 signal: AbortSignal,
1093 ): Promise<
1094 | {
1095 ok: true;
1096 proven?: string;
1097 passed?: readonly string[];
1098 /** A red it cleared as main's (#10304): `base_red.ts`'s words, kept in the summary. */
1099 cleared?: string;
1100 }
1101 | { ok: false; error: string; detail?: CheckDetail | undefined; infrastructure?: boolean; cpuStop?: string; passed?: readonly string[] }
1102 > {
1103 const bare = await runDir(app.db, app.reposRoot, repo, run.issueId);
1104 // A pull's check is the fast tier, judged against where the pull left
1105 // the base; anything else is the whole check (`apps/manifest.ts`).
1106 const issue =
1107 run.issueId === null ? undefined : getIssueById(app.db, run.issueId);
1108 const pull =
1109 issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined;
1110 if (inVerifyLane(run.env)) {
1111 // A bisection step runs only the red's failed files, with the red's
1112 // own command: an older commit may not say how to verify (#7786).
1113 const origin = run.env === VERIFY_ENV ? undefined : getRun(app.db, bisectOrigin(run.reason) ?? "");
1114 if (run.env !== VERIFY_ENV && origin === undefined)
1115 return { ok: false, error: "the red verification this step bisects is gone" };
1116 return runTier({
1117 bare,
1118 repo,
1119 id: run.id,
1120 runId: run.id,
1121 commitOid: run.commitOid,
1122 tier: "verify",
1123 ...(origin === undefined
1124 ? { keepImpact: true }
1125 : { only: origin.checkDetail?.files ?? [], commandsFrom: origin.commitOid }),
1126 fileOrderDependent: origin === undefined,
1127 isMain: true,
1128 write,
1129 signal,
1130 });
1131 }
1132 const outcome = await runTier({
1133 bare,
1134 repo,
1135 id: run.id,
1136 runId: run.id,
1137 commitOid: run.commitOid,
1138 pull,
1139 tier: pull === undefined ? "run" : "fast",
1140 keep: run.issueId === null && run.env === repo.defaultBranch,
1141 keepImpact: pull === undefined && run.env === repo.defaultBranch,
1142 fileOrderDependent: run.issueId === null && run.env === repo.defaultBranch,
1143 isMain: run.issueId === null && run.env === repo.defaultBranch,
1144 write,
1145 signal,
1166 ): Promise<CheckOutcome> {
1167 const issue = run.issueId === null ? undefined : getIssueById(app.db, run.issueId);
1168 const lane = inVerifyLane(run.env);
1169 // A bisection step runs only the red's failed files, with the red's own command: an older
1170 // commit may not say how to verify (#7786).
1171 const origin =
1172 lane && run.env !== VERIFY_ENV ? getRun(app.db, bisectOrigin(run.reason) ?? "") : undefined;
1173 const job = describeJob(run, {
1174 defaultBranch: repo.defaultBranch,
1175 verifyEnv: VERIFY_ENV,
1176 inVerifyLane: lane,
1177 pulled: issue?.kind === "pull",
1178 origin:
1179 origin === undefined
1180 ? undefined
1181 : { files: origin.checkDetail?.files ?? [], commitOid: origin.commitOid },
11461182 });
1147 return outcome;
1148 }
1149
1150 interface Execution {
1151 readonly bare: string;
1152 readonly repo: Repo;
1153 /** Names the checkout. */
1154 readonly id: string;
1155 readonly runId?: string;
1156 readonly commitOid: string;
1157 readonly pull?: Pull | undefined;
1158 /**
1159 * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after
1160 * landing.
1161 */
1162 readonly tier: "run" | "fast" | "full" | "verify";
1163 /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */
1164 readonly only?: readonly string[];
1165 /** A bisection step: the commit whose `check.verify` it runs. */
1166 readonly commandsFrom?: string;
1167 /** A check of the default branch's own push: its cache is kept. */
1168 readonly keep?: boolean;
1169 /** A green run keeps the impact map it measured; `full` always does. */
1170 readonly keepImpact?: boolean;
1171 /** The default branch's own run: only its log may open issues. */
1172 readonly fileOrderDependent?: boolean;
1173 /**
1174 * The default branch's own run: a failure of one or two tests is re-run alone before it counts
1175 * as red.
1176 */
1177 readonly isMain?: boolean;
1178 readonly write: (line: string) => void;
1179 readonly signal: AbortSignal;
1183 if ("error" in job) return { ok: false, error: job.error };
1184 return localRunner.run(job, { write, signal });
11801185 }
11811186
11821187 async function runTier(
test/checks-runner-flags.test.ts added +123
@@ -0,0 +1,123 @@
1 /**
2 * What reaches the tier from a described check (#13652, R0): the flags for main's push, a pull's
3 * push and a bisection step, and the path of a repository that is gone.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
10 import { openPull } from "../src/actions/pulls.ts";
11 import { createApp, type App } from "../src/app.ts";
12 import { BISECT_ENV, BISECT_PREFIX, VERIFY_ENV } from "../src/checks/verify.ts";
13 import { makeLocalRunner, startChecks, type Execution } from "../src/checks/service.ts";
14 import { getRun, listRuns } from "../src/core/runs.ts";
15 import { createUser, getUserByName } from "../src/core/users.ts";
16 import { quietLogs } from "./helpers/globals.ts";
17 import { pollUntil } from "./helpers/poll.ts";
18
19 quietLogs();
20
21 let dir: string;
22 let app: App;
23 beforeAll(async () => {
24 dir = await mkdtemp(join(tmpdir(), "rf-"));
25 app = createApp({ dataDir: dir, port: 0, host: "forge.test" });
26 const u = await createUser(app.db, {
27 origin: "seed",
28 trusted: true,
29 username: "alice",
30 email: "[email protected]",
31 password: "a-good-passphrase",
32 adminIfFirst: true,
33 });
34 if (!u.ok) throw new Error(u.error);
35 });
36 afterAll(async () => {
37 await rm(dir, { recursive: true, force: true });
38 });
39
40 const manifest = JSON.stringify({ check: { run: "true", install: "", verify: "true" } });
41
42 test("each kind of check reaches the tier with the flags its job names", async () => {
43 process.env["FORGE_CELLS"] = "off";
44 const seen = new Map<string, Execution>();
45 const checks = startChecks(app, {
46 forgeUrl: "http://forge.test",
47 concurrency: 1,
48 runTier: async (x) => {
49 seen.set(x.id, x);
50 return { ok: true };
51 },
52 });
53 try {
54 const u = getUserByName(app.db, "alice")!;
55 const made = await createRepo(app, u, { name: "flags", visibility: "public" });
56 if (!made.ok) throw new Error(String(made.error));
57 const repo = made.value;
58 const main = await commitToBranch(app, u, repo, {
59 message: "main",
60 files: [{ path: "forge.json", content: manifest }],
61 });
62 if (!main.ok) throw new Error(JSON.stringify(main.error));
63 const branch = await commitToBranch(app, u, repo, {
64 branch: "feature",
65 message: "feature",
66 files: [{ path: "b.txt", content: "b\n" }],
67 });
68 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
69 const pull = await openPull(app, u, repo, { title: "P", baseBranch: "main", headBranch: "feature" });
70 if (!pull.ok) throw new Error(String(pull.error));
71 const verify = await checks.request(repo, VERIFY_ENV, main.value.commitOid, "verify");
72 if (verify === undefined) throw new Error("no verify run");
73 const step = await checks.request(
74 repo,
75 BISECT_ENV,
76 main.value.commitOid,
77 `${BISECT_PREFIX}${verify.id}`,
78 );
79 if (step === undefined) throw new Error("no bisection run");
80 const settled = (id: string) => getRun(app.db, id)?.state === "ok";
81 await pollUntil(() => seen.size > 0, "a check reached the tier");
82 await pollUntil(() => settled(verify.id) && settled(step.id), "both runs ended ok");
83
84 const runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
85 const byId = (id: string) => seen.get(id);
86 const pushRun = runs.find((r) => r.issueId === null && r.env === "main");
87 const pullRun = runs.find((r) => r.issueId !== null);
88 expect(pushRun && byId(pushRun.id)).toMatchObject({
89 tier: "run", keep: true, keepImpact: true, fileOrderDependent: true, isMain: true, pull: undefined,
90 });
91 expect(pullRun && byId(pullRun.id)).toMatchObject({
92 tier: "fast", keep: false, keepImpact: false, fileOrderDependent: false, isMain: false,
93 });
94 expect(byId(pullRun!.id)?.pull?.id).toBe(pullRun!.issueId!);
95 expect(byId(verify.id)).toMatchObject({
96 tier: "verify", keepImpact: true, fileOrderDependent: true, isMain: true, pull: undefined,
97 });
98 expect(byId(verify.id)?.keep).toBeFalsy();
99 const bisected = byId(step.id);
100 expect(bisected).toMatchObject({
101 tier: "verify", only: [], commandsFrom: main.value.commitOid, isMain: true, pull: undefined,
102 });
103 expect(bisected?.keepImpact).toBeFalsy();
104 expect(bisected?.fileOrderDependent).toBeFalsy();
105 } finally {
106 await checks.stop();
107 }
108 });
109
110 test("a job for a repository that is gone ends red, without reaching the tier", async () => {
111 let reached = 0;
112 const runner = makeLocalRunner(app, async () => {
113 reached++;
114 return { ok: true };
115 });
116 const job = {
117 runId: "r", repoId: "no-such-repo", commitOid: "abc", issueId: null, tier: "run" as const,
118 keep: false, keepImpact: false, fileOrderDependent: false, isMain: false,
119 };
120 const out = await runner.run(job, { write: () => {}, signal: new AbortController().signal });
121 expect(out).toEqual({ ok: false, error: "the repository is gone" });
122 expect(reached).toBe(0);
123 });
test/checks-runner-job.test.ts added +35
@@ -0,0 +1,35 @@
1 /**
2 * The check queue's seam (#13652, R0): a job is plain data, fully describing what a runner runs.
3 */
4 import { describe, expect, test } from "bun:test";
5 import { describeJob } from "../src/checks/runner.ts";
6
7 const run = { id: "r1", repoId: "p1", commitOid: "abc", issueId: null, env: "main" };
8 const facts = { defaultBranch: "main", verifyEnv: "verify", inVerifyLane: false, pulled: false };
9
10 describe("describeJob", () => {
11 test("main's own push keeps cache and impact and may file issues", () => {
12 expect(describeJob(run, facts)).toEqual({
13 runId: "r1", repoId: "p1", commitOid: "abc", issueId: null, tier: "run",
14 keep: true, keepImpact: true, fileOrderDependent: true, isMain: true,
15 });
16 });
17
18 test("a pull's check is the fast tier and keeps nothing", () => {
19 const job = describeJob({ ...run, issueId: "i1", env: "pull/1" }, { ...facts, pulled: true });
20 expect(job).toMatchObject({ tier: "fast", keep: false, keepImpact: false, isMain: false, issueId: "i1" });
21 });
22
23 test("a whole verification keeps its impact map; a bisection step runs the red's files", () => {
24 const lane = { ...facts, inVerifyLane: true };
25 expect(describeJob({ ...run, env: "verify" }, lane)).toMatchObject({ tier: "verify", keepImpact: true, fileOrderDependent: true });
26 const step = describeJob({ ...run, env: "bisect" }, { ...lane, origin: { files: ["a.test.ts"], commitOid: "old" } });
27 expect(step).toMatchObject({ tier: "verify", only: ["a.test.ts"], commandsFrom: "old", keepImpact: false });
28 expect(describeJob({ ...run, env: "bisect" }, lane)).toEqual({ error: "the red verification this step bisects is gone" });
29 });
30
31 test("a job survives the wire unchanged", () => {
32 const job = describeJob(run, facts);
33 expect(JSON.parse(JSON.stringify(job))).toEqual(job);
34 });
35 });
test/helpers/poll.ts added +8
@@ -0,0 +1,8 @@
1 /** Waits until `done` holds, polling; for work that runs on its own pace in the same process. */
2 export async function pollUntil(done: () => boolean, what: string, timeoutMs = 30_000): Promise<void> {
3 const deadline = Date.now() + timeoutMs;
4 while (!done()) {
5 if (Date.now() > deadline) throw new Error(`never came: ${what}`);
6 await Bun.sleep(50);
7 }
8 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs