Public: anyone can read this; the crew can change it.

src/checks/service.ts is split into small modules named for one concept, with no behaviour change #14019

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14018-src-checks-service-ts-is-split-into-smal → main 1 commit fork 1 thread · 1 unresolved

  • Merged. mason/14018-src-checks-service-ts-is-split-into-smal was merged into main as 49655b1.

Review

Sign in to review this change.
24 files changed +2543 −2128
AGENTS.md +3 −1

Diff not shown — a line 2,989 characters wide (limit 2,000). +3 −1

View file
src/checks/abandoned.ts added +32
@@ -0,0 +1,32 @@
1
2 import type { App } from "../app.ts";
3 import { announceRun } from "../actions/runs.ts";
4 import { REQUEUED_SUMMARY, sweepAbandonedChecks } from "../core/runs.ts";
5 import { appendRunLog } from "../runs/log.ts";
6 import { sweepSoon } from "./sweep.ts";
7
8 /**
9 * Settle the checks a dead forge left running, and say so on the bus: a
10 * check whose commit still heads an open pull is queued again for whichever
11 * checks service is up to run, so the change keeps reading *working*.
12 * Called on boot and on the way by whichever forge is alive. A cancelled
13 * check is not announced, as a superseded one is not: `run.finished` reads
14 * as a verdict on the change, and the forge dying is none.
15 */
16 export function requeueAbandonedChecks(app: App, staleMs: number): number {
17 const swept = sweepAbandonedChecks(app.db, staleMs);
18 for (const { cancelled, again } of swept.requeued) {
19 appendRunLog(app.config.dataDir, cancelled.id, REQUEUED_SUMMARY);
20 announceRun(app, again, again.actorId ?? undefined);
21 }
22 for (const run of swept.cancelled)
23 appendRunLog(app.config.dataDir, run.id, run.summary);
24 for (const run of swept.failed) {
25 appendRunLog(app.config.dataDir, run.id, run.summary);
26 announceRun(app, run);
27 }
28 // On the same timer: what a dead or lapsed run left on the disk. Removing
29 // a checkout is thousands of unlinks: off the loop, never in front of /healthz (#7688).
30 sweepSoon(app);
31 return swept.requeued.length + swept.failed.length + swept.cancelled.length;
32 }
src/checks/event_handler.ts added +189
@@ -0,0 +1,189 @@
1

Blank first line: add a header comment naming the one concept (what starts a check). Same in every new module.

2 import { pullDir } from "../core/pulls.ts";
3 import type { App } from "../app.ts";
4 import { announceRun } from "../actions/runs.ts";
5 import type { ForgeEvent } from "../core/events.ts";
6 import { getPull, listPulls, type Pull } from "../core/pulls.ts";
7 import { getRepoById, type Repo } from "../core/repos.ts";
8 import { getRun, cancelRunsForPull, checksForHead } from "../core/runs.ts";
9 import { resolveOid } from "../git/merge.ts";
10 import { checkOnApproval } from "./approval.ts";
11 import { settleLandedRuns } from "../crew/interrupted.ts";
12 import type { LaneState } from "./lane_state.ts";
13 import { recheckBase } from "./recheck_base.ts";
14
15 export async function onEvent(lane: LaneState, event: ForgeEvent): Promise<void> {
16 const { app } = lane;
17 if (lane.stopped || event.repoId === null) return;
18 const repo = getRepoById(app.db, event.repoId);
19 if (!repo) return;
20 const actor = event.actorId ?? undefined;
21 switch (event.kind) {
22 case "push": {
23 // A fork that silently followed holds its parent's commit, byte for
24 // byte: checking the copy proves nothing the parent's own check did
25 // not, once per fork, on every landed pull. A merge of the parent
26 // into the fork's own work is a new commit, and is checked.
27 if (
28 typeof event.payload["followed"] === "string" &&
29 event.payload["merged"] !== true
30 )
31 return;
32 const updates = event.payload["updates"];
33 if (!Array.isArray(updates)) return;
34 for (const update of updates as { ref?: unknown; newOid?: unknown }[]) {
35 if (
36 typeof update.ref !== "string" ||
37 typeof update.newOid !== "string"
38 )
39 continue;
40 if (
41 !update.ref.startsWith("refs/heads/") ||
42 /^0+$/.test(update.newOid)
43 )
44 continue;
45 const branch = update.ref.slice("refs/heads/".length);
46 if (branch === repo.defaultBranch)
47 await lane.request(
48 repo,
49 branch,
50 update.newOid,
51 "push",
52 undefined,
53 actor,
54 );
55 await recheckBase(lane, repo, branch, update.newOid);
56 for (const pull of listPulls(app.db, repo.id, {
57 sealed: true,
58 state: "open",
59 limit: 200,
60 })) {
61 if (pull.crossRepo || pull.headBranch !== branch) continue;
62 await lane.request(
63 repo,
64 `pull-${pull.number}`,
65 update.newOid,
66 "push",
67 pull.id,
68 actor,
69 );
70 }
71 }
72 return;
73 }
74 case "pull.opened":
75 case "pull.reopened": {
76 const number = event.payload["number"];
77 if (typeof number !== "number") return;
78 const pull = getPull(app.db, repo.id, number);
79 if (!pull) return;
80 const head = await headOf(app, repo, pull);
81 if (head !== undefined)
82 await lane.request(
83 repo,
84 `pull-${pull.number}`,
85 head,
86 "pull",
87 pull.id,
88 actor,
89 );
90 return;
91 }
92 case "pull.pushed": {
93 const number = event.payload["number"];
94 const headOid = event.payload["headOid"];
95 if (typeof number !== "number" || typeof headOid !== "string") return;
96 const pull = getPull(app.db, repo.id, number);
97 if (!pull) return;
98 await lane.request(
99 repo,
100 `pull-${pull.number}`,
101 headOid,
102 "push",
103 pull.id,
104 actor,
105 );
106 return;
107 }
108 // An approval is what lets a reader's pull run: check its head now.
109 case "pull.run_granted":
110 case "pull.reviewed": {
111 const number = event.payload["number"];
112 if (typeof number !== "number") return;
113 if (
114 event.kind === "pull.reviewed" &&
115 event.payload["state"] !== "approved"
116 )
117 return;
118 const pull = getPull(app.db, repo.id, number);
119 if (!pull || pull.state !== "open") return;
120 const head = await headOf(app, repo, pull);
121 if (head === undefined) return;
122 // A head already green is not checked again for nothing (#4391).
123 const env = `pull-${pull.number}`;
124 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
125 const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
126 // The merge queue gates only an autonomous repository's lands (`crew/autonomy.ts` admit).
127 const landGated = repo.autonomy === "autonomous" && repo.autonomyBy !== null;
128 if (!checkOnApproval(checksForHead(app.db, pull.id, head), env, tip, landGated))
129 return;
130 await lane.request(repo, env, head, "approved", pull.id, actor);
131 return;
132 }
133 // Landed or closed: nothing left to check, so nothing waits in the lane.
134 case "pull.merged":
135 case "pull.closed": {
136 const number = event.payload["number"];
137 if (typeof number !== "number") return;
138 const pull = getPull(app.db, repo.id, number);
139 if (!pull) return;
140 const cancelled = cancelRunsForPull(
141 app.db,
142 pull.id,
143 event.kind === "pull.merged" ? "pull landed" : "pull closed",
144 );
145 for (const run of cancelled) {
146 lane.inFlight.get(run.id)?.abort.abort();
147 announceRun(app, run);
148 }
149 // A builder run still in flight when its pull lands reads as done now (#12049).
150 if (event.kind === "pull.merged")
151 for (const id of settleLandedRuns(app.db, { issueId: pull.id, headBranch: pull.headBranch, repoId: repo.id, number })) {
152 const done = getRun(app.db, id);
153 if (done) announceRun(app, done);
154 }
155 // A land moves the default branch without a push event: check its
156 // new head, or a red main stays red until someone pushes to it. The
157 // check settles first: a candidate's next step supersedes it.
158 const mergeOid = event.payload["mergeOid"];
159 if (
160 event.kind === "pull.merged" &&
161 pull.baseBranch === repo.defaultBranch &&
162 typeof mergeOid === "string" &&
163 /^[0-9a-f]{40}$/.test(mergeOid)
164 )
165 await lane.request(
166 repo,
167 repo.defaultBranch,
168 mergeOid,
169 "push",
170 undefined,
171 actor,
172 lane.settle > 0 ? new Date(Date.now() + lane.settle).toISOString() : undefined,
173 );
174 if (event.kind === "pull.merged" && typeof mergeOid === "string" && /^[0-9a-f]{40}$/.test(mergeOid))
175 await recheckBase(lane, repo, pull.baseBranch, mergeOid);
176 return;
177 }
178 default:
179 return;
180 }
181 }
182
183 async function headOf(app: App, repo: Repo, pull: Pull): Promise<string | undefined> {
184 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
185 const ref = pull.crossRepo
186 ? `refs/pull/${pull.number}/head`
187 : `refs/heads/${pull.headBranch}`;
188 return (await resolveOid(dir, ref)) ?? (pull.headOid || undefined);
189 }
src/checks/execute.ts added +250
@@ -0,0 +1,250 @@
1
2 import { releaseOnGreen } from "../actions/holds.ts";
3 import { announceRun } from "../actions/runs.ts";
4 import { announceShard } from "./shard_event.ts";
5 import { issueMayRun } from "../core/pull_trust.ts";
6 import type { Repo } from "../core/repos.ts";
7 import { cancelRun, getRun, claimRun, finishRun, overranBefore, vanishedBefore, VANISHED_SUMMARY, infraBefore, INFRA_SUMMARY, CPU_STOP_SUMMARY, DISK_USED_SUMMARY, requeueCheck, chainCutOffs, CHECK_RELEASE_LIMIT, RELEASE_SUMMARY, type Run } from "../core/runs.ts";
8 import { withDisk } from "./disk_red.ts";
9 import { proposeRevert } from "./revert_red.ts";
10 import { afterVerify, closeOnGreen } from "./culprit.ts";
11 import { noteLookPages } from "./look_pages.ts";
12 import { reportGreen, reportRed } from "./report.ts";
13 import { bisectOrigin, inVerifyLane, VERIFY_ENV } from "./verify.ts";
14 import { getIssueById } from "../core/issues.ts";
15 import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts";
16 import { createLog } from "../kernel/log.ts";
17 import { holdLease } from "../runs/lease.ts";
18 import { appendRunLog } from "../runs/log.ts";
19 import { cancelledLine } from "../runs/cancelled.ts";
20 import { requeueCpuStop } from "./cpu_stop.ts";
21 import { isUsedRefusal } from "../apps/disk_guard.ts";
22 import { JOB_VANISHED } from "./job_step.ts";
23 import { describeJob, type CheckOutcome } from "./runner.ts";
24 import type { LaneState } from "./lane_state.ts";
25
26 const log = createLog("checks");
27
28 export async function execute(
29 lane: LaneState,
30 run: Run,
31 repo: Repo,
32 signal: AbortSignal,
33 slot = true,
34 ): Promise<void> {
35 const { app, options } = lane;
36 // The verification lane takes no slot of the gate (`startVerify`).
37 const release = slot ? await lane.gate.acquire() : () => {};
38 try {
39 // A slot that opens while this forge is stopping is not taken: the run
40 // stays queued for the forge that carries on.
41 if (lane.stopped) return;
42 // Asked again as it starts: a check queued before a rule changed, or
43 // handed back by a forge that stopped, runs only what may run now.
44 if (!issueMayRun(app.db, repo, run.issueId, run.commitOid)) {
45 cancelRun(app.db, run.id, "waiting for a writer to approve this head");
46 return;
47 }
48 const claimed = claimRun(app.db, run.id);
49 if (!claimed) return;
50 const lease = holdLease(app.db, run.id);
51 announceRun(app, claimed);
52 const write = (line: string): void => {
53 appendRunLog(app.config.dataDir, run.id, line);
54 announceShard(app, run, line);
55 };
56 if (options.label !== undefined)
57 write(`forge ${options.label} — checked in its parent's lane`);
58 const started = Date.now();
59 // The budget: past it the check is aborted, whatever it is doing.
60 const limit = checkLimitMs(run.env, {
61 gate: options.budgetMs ?? BUDGETS.gate,
62 verify: options.verifyBudgetMs ?? BUDGETS.verify,
63 review: BUDGETS.review,
64 });
65 const budget = new AbortController();
66 let overran = false;
67 const forward = (): void => budget.abort();
68 signal.addEventListener("abort", forward, { once: true });
69 const clock = setTimeout(() => {
70 overran = true;
71 budget.abort();
72 }, limit);
73 const outcome = await check(lane, claimed, repo, write, budget.signal).finally(
74 () => {
75 clearTimeout(clock);
76 signal.removeEventListener("abort", forward);
77 lease();
78 },
79 );
80 // Where a check cut off here is asked again. A verification is of the
81 // default branch as it stands: it goes back at the branch's newest
82 // checked commit, the one `askVerifications` would ask of. Left at the
83 // commit it began on, each restart and each overrun began it there
84 // again, no other was asked while it was active, and main went
85 // unverified from 01:48Z to 03:37Z on 2026-11-14 (#12178).
86 const againAt = (): string | undefined =>
87 run.env === VERIFY_ENV
88 ? app.db
89 .query<{ commit_oid: string }, [string, string]>(
90 `SELECT commit_oid FROM run WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL
91 AND env = ?2 ORDER BY id DESC LIMIT 1`,
92 )
93 .get(repo.id, repo.defaultBranch)?.commit_oid || undefined
94 : undefined;
95 if (overran) {
96 const summary = budgetSummary(
97 "a full gate",
98 Date.now() - started,
99 limit,
100 );
101 write(summary);
102 if (!overranBefore(app.db, claimed)) {
103 const moved = requeueCheck(app.db, run.id, undefined, summary, againAt());
104 if (moved) {
105 announceRun(app, moved.again, run.actorId ?? undefined);
106 return;
107 }
108 }
109 // The second overrun: a failure, and the card the vitals carry.
110 const failed = finishRun(app.db, run.id, {
111 state: "failed",
112 summary,
113 });
114 if (failed) announceRun(app, failed);
115 return;
116 }
117 // Ended from outside while it ran: its pull landed or closed, and
118 // the row says so. That is not this check failing.
119 // A person's cancel ends the log with who cancelled it (#11229).
120 const row = getRun(app.db, run.id);
121 if (!outcome.ok && row?.state === "cancelled") {
122 write(cancelledLine(row) ?? `superseded: ${row.summary}`);
123 return;
124 }
125 if (!outcome.ok && lane.stopped && signal.aborted) {
126 // Cut off by this forge stopping, not by the check: the same check
127 // goes back on the queue, and the forge that carries on runs it.
128 // Past the release ceiling the shutdown is what this check hangs or
129 // slows: it fails, saying so, instead of riding every release.
130 if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) {
131 const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`;
132 write(summary);
133 const failed = finishRun(app.db, run.id, { state: "failed", summary });
134 if (failed) announceRun(app, failed);
135 return;
136 }
137 const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt());
138 if (!moved) return;
139 write(RELEASE_SUMMARY);
140 announceRun(app, moved.again, run.actorId ?? undefined);
141 return;
142 }
143 // The cluster lost the Job (evicted, node gone): interrupted, not
144 // failed, and queued once more as after a restart; the second loss fails.
145 if (!outcome.ok && outcome.error === JOB_VANISHED && !vanishedBefore(app.db, claimed)) {
146 const moved = requeueCheck(app.db, run.id, undefined, VANISHED_SUMMARY);
147 if (moved) {
148 write(VANISHED_SUMMARY);
149 announceRun(app, moved.again, run.actorId ?? undefined);
150 return;
151 }
152 }
153 // The install failed twice on what the machine did, not the code: the
154 // check is queued once more and nothing reads it as red (#5093).
155 if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) {
156 const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY);
157 if (moved) {
158 write(INFRA_SUMMARY);
159 announceRun(app, moved.again, run.actorId ?? undefined);
160 return;
161 }
162 }
163 // The disk was 80% used when the step would have started (#11799):
164 // the machine's wait. Queued again, not red; the pick holds it until there is room.
165 if (!outcome.ok && isUsedRefusal(outcome.error)) {
166 const moved = requeueCheck(app.db, run.id, undefined, DISK_USED_SUMMARY);
167 if (moved) {
168 write(DISK_USED_SUMMARY);
169 announceRun(app, moved.again, run.actorId ?? undefined);
170 return;
171 }
172 }
173 // Stopped for CPU by the cell's watch: the machine's, not the pull's.
174 // Queued once more on its own, and no builder hears check_failed for
175 // it; a second stop in a row is reported as any red is (#11617).
176 if (!outcome.ok && outcome.cpuStop !== undefined) {
177 const moved = requeueCpuStop(app.db, claimed, outcome.cpuStop);
178 if (moved) {
179 write(CPU_STOP_SUMMARY);
180 announceRun(app, moved.again, run.actorId ?? undefined);
181 return;
182 }
183 }
184 const seconds = Math.round((Date.now() - started) / 1000);
185 const done = finishRun(app.db, run.id, {
186 state: outcome.ok ? "ok" : "failed",
187 summary: outcome.ok
188 ? outcome.proven === undefined
189 ? `check passed in ${seconds}s${outcome.cleared === undefined ? "" : `; ${outcome.cleared}`}`
190 : `already proven green by ${outcome.proven}`
191 : `check failed: ${outcome.error}`,
192 ...(outcome.ok
193 ? {}
194 : { checkDetail: withDisk(outcome.detail, app.config.dataDir) }),
195 });
196 write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`);
197 // The pages the gate chose to picture are said on the pull (#13373), worked out by the
198 // forge from the pull's commits, never read from the log the branch's code writes.
199 if (done) void noteLookPages(app, repo, done).catch((error) => log.error("look pages note failed", { error: String(error) }));
200 if (done) announceRun(app, done);
201 if (done && outcome.ok) releaseOnGreen(app, done);
202 if (!outcome.ok && run.issueId === null && run.env === repo.defaultBranch)
203 proposeRevert(app, repo.id).catch((error) =>
204 log.error("revert proposal failed", { error: String(error) }),
205 );
206 // A red verification after landing names its culprit (#7786).
207 if (done && run.env === VERIFY_ENV) {
208 reportRed(app, repo, done).forEach(write);
209 // A file the run printed passing closes its open report issue (#9524).
210 reportGreen(app, repo, done, outcome.passed ?? []).forEach(write);
211 // A green that ran every file a culprit's issue names closes it (#11395).
212 closeOnGreen(app, repo, done, outcome.passed ?? []).forEach(write);
213 }
214 if (done && inVerifyLane(run.env))
215 await afterVerify(app, repo, done, (r, env, oid, reason) => lane.request(r, env, oid, reason))
216 .then((lines) => lines.forEach(write))
217 .catch((error) => write(`culprit not named: ${String(error)}`));
218 } finally {
219 release();
220 }
221 }
222
223 /** Describe a claimed check as data and hand it to the runner. */
224 async function check(
225 lane: LaneState,
226 run: Run,
227 repo: Repo,
228 write: (line: string) => void,
229 signal: AbortSignal,
230 ): Promise<CheckOutcome> {
231 const { app } = lane;
232 const issue = run.issueId === null ? undefined : getIssueById(app.db, run.issueId);
233 const inLane = inVerifyLane(run.env);
234 // A bisection step runs only the red's failed files, with the red's own command: an older
235 // commit may not say how to verify (#7786).
236 const origin =
237 inLane && run.env !== VERIFY_ENV ? getRun(app.db, bisectOrigin(run.reason) ?? "") : undefined;
238 const job = describeJob(run, {
239 defaultBranch: repo.defaultBranch,
240 verifyEnv: VERIFY_ENV,
241 inVerifyLane: inLane,
242 pulled: issue?.kind === "pull",
243 origin:
244 origin === undefined
245 ? undefined
246 : { files: origin.checkDetail?.files ?? [], commitOid: origin.commitOid },
247 });
248 if ("error" in job) return { ok: false, error: job.error };
249 return lane.runner.run(job, { write, signal });
250 }
src/checks/gate_run.ts added +71
@@ -0,0 +1,71 @@
1
2 import { pullDir } from "../core/pulls.ts";
3 import type { Pull } from "../core/pulls.ts";
4 import type { Repo } from "../core/repos.ts";
5 import type { GateDone } from "./lane_state.ts";
6 import { beginGate, endGate } from "./gate_record.ts";
7 import { JOB_VANISHED } from "./job_step.ts";
8 import { gateRunId } from "./queue.ts";
9 import type { LaneState } from "./lane_state.ts";
10 import { runTier } from "./run_tier.ts";
11
12 /** A gate this forge would not start, for it is stopping. */
13 const INTERRUPTED = "this forge is stopping";
14 async function gateOnce(
15 lane: LaneState,
16 repo: Repo,
17 pull: Pull,
18 commitOid: string,
19 controller: AbortController,
20 signal: AbortSignal | undefined,
21 ): Promise<GateDone> {
22 const { app } = lane;
23 const lines: string[] = [];
24 const abandon = (): void => controller.abort();
25 signal?.addEventListener("abort", abandon, { once: true });
26 // The slot minutes it spends are the flow's gate minutes (#6426).
27 const traced = beginGate(app.db, repo.id, pull.id, commitOid);
28 try {
29 const bare = await pullDir(app.db, app.reposRoot, repo, pull);
30 const done = await runTier(lane, {
31 bare,
32 repo,
33 id: gateRunId(commitOid),
34 commitOid,
35 pull,
36 tier: "full",
37 write: (line) => lines.push(line),
38 signal: controller.signal,
39 });
40 if (done.ok) return { ok: true, error: "", lines };
41 const stage = done.stage ?? "setup";
42 // Cut off by this forge stopping, the Job lost under it, the check
43 // killed by a signal from outside (the pod going, the OOM killer), or
44 // an install the machine failed twice: the code was not judged, so
45 // this is no red. A gate the drain lets finish is judged (#6611, #6722).
46 if (
47 controller.signal.aborted ||
48 done.error === JOB_VANISHED ||
49 done.infrastructure === true ||
50 done.killed === true
51 )
52 return { ok: false, error: done.error, lines, interrupted: true, stage };
53 return { ok: false, error: done.error, lines, stage };
54 } finally {
55 endGate(app.db, traced);
56 signal?.removeEventListener("abort", abandon);
57 lane.gates.delete(controller);
58 }
59 }
60
61 /**
62 * A gate asked of a forge already stopping is not started: the forge that carries on asks it
63 * again.
64 */
65 export function gateRun(lane: LaneState, repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone> {
66 if (lane.stopped) return Promise.resolve({ ok: false, error: INTERRUPTED, lines: [], interrupted: true });
67 const controller = new AbortController();
68 const done = gateOnce(lane, repo, pull, commitOid, controller, signal);
69 lane.gates.set(controller, { commitOid, done });
70 return done;
71 }
src/checks/lane_state.ts added +218
@@ -0,0 +1,218 @@
1
2 import type { App } from "../app.ts";
3 import { holdClock } from "../core/one_core.ts";
4 import type { Pull } from "../core/pulls.ts";
5 import type { Repo } from "../core/repos.ts";
6 import type { Run } from "../core/runs.ts";
7 import { Gate } from "../git/env.ts";
8 import { runStep } from "../apps/process.ts";
9 import { capacityCount } from "../core/capacity.ts";
10 import type { Placement } from "./placement.ts";
11 import type { PodRoom } from "./pod_room.ts";
12 import type { GateStage } from "./gate_reason.ts";
13 import type { CheckOutcome, Runner } from "./runner.ts";
14 import type { JobOptions } from "../cluster/jobs.ts";
15 import type { Execution } from "./local_runner.ts";
16
17 export interface ChecksOptions {
18 /** How many checks may run at once. @default 1 */
19 readonly concurrency?: number | undefined;
20 /** How full this forge's memory is, 0 to 1: its cgroup's unless a test says otherwise. */
21 readonly pressure?: (() => number | undefined) | undefined;
22 /** The forge's public URL, told to the check as `FORGE_URL`. */
23 readonly forgeUrl: string;
24 /** The parent's gate, for a hosted child: its checks take their turn in the parent's lane. */
25 readonly gate?: Gate | undefined;
26 /** Whose checks these are, written first in each run's log (a hosted child's name). */
27 readonly label?: string | undefined;
28 /** How long a check may run before it is killed. @default BUDGETS.gate */
29 readonly budgetMs?: number | undefined;
30 /** How long a verification of the default branch may run. @default BUDGETS.verify */
31 readonly verifyBudgetMs?: number | undefined;
32 /**
33 * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0,
34 * nothing held; the forge says `RED_HOLD_MS`.
35 */
36 readonly redHoldMs?: number | undefined;
37 /**
38 * How long main's check after a land waits before it may start, so the
39 * steps of one candidate supersede each other while queued. @default 0,
40 * none; the forge says `LAND_SETTLE_MS`.
41 */
42 readonly landSettleMs?: number | undefined;
43 /**
44 * How long `stop` lets checks already running finish before it cuts them
45 * off and queues them again (#1796). Nothing new starts meanwhile.
46 * @default 0
47 */
48 readonly drainMs?: number | undefined;
49 /**
50 * Where a check's commands run (`checks/placement.ts`). @default from the
51 * environment: `process`.
52 */
53 readonly placement?: Placement | undefined;
54 /** The kube client a `job` placement speaks through; tests stand one in. */
55 readonly job?: JobOptions | undefined;
56 /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */
57 readonly installRetryMs?: number | undefined;
58 /**
59 * Runs the check step itself; tests stand one in for a stop the cell's watch makes
60 * (#11617). @default runStep
61 */
62 readonly checkStep?: typeof runStep | undefined;
63 /**
64 * Where the pod's memory.max is read and the forge's size
65 * (`checks/pod_room.ts`); tests stand in a small pod. @default the real ones
66 */
67 readonly podRoom?: Omit<PodRoom, "env"> | undefined;
68 /**
69 * How often the default branch is verified after landing at most
70 * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds
71 * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`.
72 */
73 readonly verifyEveryMs?: number | undefined;
74 /** Test seam: stands in for the tier the local runner runs, to see what reaches it. */
75 readonly runTier?: ((x: Execution) => Promise<CheckOutcome>) | undefined;
76 /** Test seam: how often the queue is looked at. @default TICK_MS */
77 readonly tickMs?: number | undefined;
78 /** The forge's CPU count; a 1-CPU forge holds pull checks for a crew run. @default forgeCpus */
79 readonly cpus?: (() => number) | undefined;
80 /** Crew runs working now. @default the running agent runs */
81 readonly activeCrewRuns?: (() => number) | undefined;
82 /** Test seams: how long a held pull check waits at most, and the clock it is judged by. */
83 readonly holdMaxMs?: number | undefined;
84 readonly holdNow?: (() => number) | undefined;
85 }
86 /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */
87 export interface GateDone {
88 ok: boolean;
89 error: string;
90 lines: string[];
91 interrupted?: boolean;
92 stage?: GateStage;
93 }
94
95 export interface ChecksHandle {
96 /**
97 * Ask for a check of one commit; the answer is the run, or nothing when the commit declares no
98 * check.
99 */
100 request(
101 repo: Repo,
102 env: string,
103 commitOid: string,
104 reason: string,
105 issueId?: string,
106 actorId?: string,
107 ): Promise<Run | undefined>;
108 /**
109 * The full tier on a merge, before it is made (#1198): `commitOid` is the
110 * merge of the pull with its base, run in a clone of its own. Nothing is
111 * queued and no run is written; the answer is the verdict. `interrupted`
112 * is a gate that did not finish for a reason that is not the code — this
113 * forge stopping, or the cluster losing its Job — and is no verdict (#3155).
114 * `signal` aborting (the queue's deadline, #3430) cuts it off the same way.
115 */
116 gate(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone>;
117 /**
118 * The gate of `commitOid` this forge is running now, to wait on rather
119 * than start again (#6611); undefined when none is.
120 */
121 adopt?(commitOid: string): Promise<GateDone> | undefined;
122 /**
123 * Stop checking. Checks under way get `drainMs` to finish unless `drain`
124 * is false (a person's Ctrl-C): then they are cut off and queued again now.
125 */
126 stop(options?: { drain?: boolean }): Promise<void>;
127 /**
128 * Check slots free now for a merge-queue gate beyond a line's first
129 * (#4053): what neither a running check nor such a gate holds, less
130 * `PULL_RESERVE` kept for pull checks.
131 */
132 spareSlots(): number;
133 /** The gate checks take their turn at; a hosted child's checks share it. */
134 readonly lane: Gate;
135 /**
136 * Ask for every verification after landing that is due now
137 * (`checks/verify.ts`); the answer is the runs asked for. The forge's
138 * own timer calls it when `verifyEveryMs` is set.
139 */
140 verifyDue(now?: number): Promise<Run[]>;
141 }
142
143 /**
144 * Check slots the merge queue's extra gates never take (#4053): pull checks
145 * always have at least these, however long the queue.
146 */
147 export const PULL_RESERVE = 2;
148
149 /**
150 * How long main's check after a land waits to start (#4061). The merge queue
151 * lands a candidate's steps seconds apart, and only its tip was gated: a
152 * step's check that started at once ran the full suite on a commit the next
153 * step replaced, up to seven of them filling the lane for nine minutes. A
154 * step's check that waits is superseded by the next while still queued, and
155 * the tip's is proven green by the gate.
156 */
157 export const LAND_SETTLE_MS = 30_000;
158
159 /**
160 * What one checks service holds, in one plain record the lane's functions are handed
161 * (`startChecks` makes it): its queue's local state, and the two calls that reach back across
162 * modules (a request schedules, a schedule's start asks again).
163 */
164 export interface LaneState {
165 readonly app: App;
166 readonly options: ChecksOptions;
167 /** The gate checks take their turn at. */
168 readonly gate: Gate;
169 readonly placement: Placement;
170 /**
171 * Full gates on land in flight: not slots in the queue, but stopped with it — after the drain,
172 * as a check is. By commit, so a merge queue that carries on adopts one rather than gate the
173 * same tip again (#6611).
174 */
175 readonly gates: Map<AbortController, { commitOid: string; done: Promise<GateDone> }>;
176 readonly inFlight: Map<string, { promise: Promise<void>; abort: AbortController }>;
177 /** Checks already told why they wait, so the log says it once. */
178 readonly waiting: Set<string>;
179 /**
180 * Queued checks a full disk holds that have said so once (#11799); apart from `waiting`, so
181 * neither line hides the other.
182 */
183 readonly diskWaiting: Set<string>;
184 readonly memoryWaiting: Set<string>;
185 /** The verification-lane run in flight, which holds no check slot. */
186 verifying: string | undefined;
187 /** How long main's check after a land waits before it may start. */
188 readonly settle: number;
189 stopped: boolean;
190 /** While above 0, a batch of requests is being queued and `schedule` waits for its end. */
191 holding: number;
192 readonly pullHold: ReturnType<typeof holdClock>;
193 /** The runner every check goes to. */
194 readonly runner: Runner;
195 /** Fill every free slot with the queued check that deserves it. */
196 schedule(): void;
197 /** Ask for a check of one commit. */
198 request(
199 repo: Repo,
200 env: string,
201 commitOid: string,
202 reason: string,
203 issueId?: string,
204 actorId?: string,
205 notBefore?: string,
206 ): Promise<Run | undefined>;
207 }
208
209 /** Check slots in use: what runs, less the verification lane's, which holds none. */
210 export const slotted = (lane: LaneState): number =>
211 lane.inFlight.size - (lane.verifying === undefined ? 0 : 1);
212
213 /**
214 * How many checks may run at once, read on every pick (`core/capacity.ts`): an admin's change is
215 * honoured by the next check, not the next boot. `concurrency` is the boot default.
216 */
217 export const capacity = (lane: LaneState): number =>
218 capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1);
src/checks/local_runner.ts added +75
@@ -0,0 +1,75 @@
1
2 import { runDir } from "../core/pulls.ts";
3 import type { App } from "../app.ts";
4 import { getPull, type Pull } from "../core/pulls.ts";
5 import { getRepoById, type Repo } from "../core/repos.ts";
6 import { getIssueById } from "../core/issues.ts";
7 import { LOCAL_RUNNER, type CheckOutcome, type Runner } from "./runner.ts";
8
9
10 /** What one tier run is given (`runTier`). */
11 export interface Execution {
12 readonly bare: string;
13 readonly repo: Repo;
14 /** Names the checkout. */
15 readonly id: string;
16 readonly runId?: string;
17 readonly commitOid: string;
18 readonly pull?: Pull | undefined;
19 /**
20 * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after
21 * landing.
22 */
23 readonly tier: "run" | "fast" | "full" | "verify";
24 /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */
25 readonly only?: readonly string[];
26 /** A bisection step: the commit whose `check.verify` it runs. */
27 readonly commandsFrom?: string;
28 /** A check of the default branch's own push: its cache is kept. */
29 readonly keep?: boolean;
30 /** A green run keeps the impact map it measured; `full` always does. */
31 readonly keepImpact?: boolean;
32 /** The default branch's own run: only its log may open issues. */
33 readonly fileOrderDependent?: boolean;
34 /**
35 * The default branch's own run: a failure of one or two tests is re-run alone before it counts
36 * as red.
37 */
38 readonly isMain?: boolean;
39 readonly write: (line: string) => void;
40 readonly signal: AbortSignal;
41 }
42
43 /** The runner every check goes to: this process (`runner.ts`); `tier` runs one described check. */
44 export function makeLocalRunner(app: App, tier: (x: Execution) => Promise<CheckOutcome>): Runner {
45 return {
46 name: LOCAL_RUNNER,
47 async run(job, io) {
48 const repo = getRepoById(app.db, job.repoId);
49 if (repo === undefined) return { ok: false, error: "the repository is gone" };
50 const bare = await runDir(app.db, app.reposRoot, repo, job.issueId);
51 // Only a pull's own check is judged against the pull, resolved here and nowhere else; the
52 // verification lane never is.
53 const issue =
54 job.tier === "fast" && job.issueId !== null ? getIssueById(app.db, job.issueId) : undefined;
55 const pull = issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined;
56 return tier({
57 bare,
58 repo,
59 id: job.runId,
60 runId: job.runId,
61 commitOid: job.commitOid,
62 pull,
63 tier: job.tier,
64 ...(job.only === undefined ? {} : { only: job.only }),
65 ...(job.commandsFrom === undefined ? {} : { commandsFrom: job.commandsFrom }),
66 keep: job.keep,
67 keepImpact: job.keepImpact,
68 fileOrderDependent: job.fileOrderDependent,
69 isMain: job.isMain,
70 write: io.write,
71 signal: io.signal,
72 });
73 },
74 };
75 }
src/checks/pick.ts added +55
@@ -0,0 +1,55 @@
1
2 import type { Run } from "../core/runs.ts";
3 import { ownersFirst } from "./outside_lane.ts";
4 import { STARVATION_MS } from "../kernel/budgets.ts";
5
6
7 /**
8 * Whether a pull's check must pass over the free slots left (`free`): while
9 * a merge-queue gate is about to start (`gateQueued`), the last one is the
10 * gate's.
11 */
12 export function holdsLastSlot(free: number, gateWaits: boolean): boolean {
13 return gateWaits && free <= 1;
14 }
15
16 /**
17 * Which queued check takes a slot when one frees: the one requested most
18 * recently, so a push's own check is never stuck behind pulls that queued
19 * hours earlier — unless something has already waited `STARVATION_MS` or
20 * more, in which case the longest-waiting check goes first regardless of
21 * how fresh the rest are. Checks of approved pulls (`approved`) go before
22 * the rest, newest first among themselves (#6396): a check that is the last
23 * thing between a pull and landing is not held up by a new pull's. Ties
24 * (down to the same instant) fall back to id order, which is itself
25 * creation order (`kernel/id.ts`). Among the rest, and among the approved,
26 * the pull whose issue stands highest on the owner's ranked list (`rank`,
27 * 1 at the top) goes first, the unranked last, newest first only as the
28 * tie-break (#12230).
29 */
30 export function pickNext(
31 queued: readonly Run[],
32 now: number = Date.now(),
33 approved: (run: Run) => boolean = () => false,
34 rank: (run: Run) => number | undefined = () => undefined,
35 outside: (run: Run) => boolean = () => false,
36 ): Run | undefined {
37 // An unapproved outside contributor's check goes after everyone else's (`outside_lane.ts`).
38 queued = ownersFirst(queued, outside);
39 if (queued.length === 0) return undefined;
40 const newer = (a: Run, b: Run): boolean =>
41 a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id);
42 let oldest = queued[0]!;
43 const place = (run: Run): number => rank(run) ?? Number.MAX_SAFE_INTEGER;
44 const better = (a: Run, b: Run): boolean => place(a) < place(b) || (place(a) === place(b) && newer(a, b));
45 let newest: Run | undefined;
46 let newestApproved: Run | undefined;
47 for (const run of queued) {
48 if (newer(oldest, run)) oldest = run;
49 if (newest === undefined || better(run, newest)) newest = run;
50 if (approved(run) && (newestApproved === undefined || better(run, newestApproved)))
51 newestApproved = run;
52 }
53 if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest;
54 return newestApproved ?? newest;
55 }
src/checks/recheck_base.ts added +77
@@ -0,0 +1,77 @@
1
2 import { pullDir } from "../core/pulls.ts";
3 import type { App } from "../app.ts";
4 import { getPull, listOpenPulls, type Pull } from "../core/pulls.ts";
5 import type { Repo } from "../core/repos.ts";
6 import { getRun, checksForHead, type Run } from "../core/runs.ts";
7 import { resolveOid } from "../git/merge.ts";
8 import { BASE_MOVED, dueGreen, newestBaseFirst } from "./base_moved.ts";
9 import { createLog } from "../kernel/log.ts";
10 import type { LaneState } from "./lane_state.ts";
11
12 const log = createLog("checks");
13
14 /**
15 * The base moved (#12533): every open pull on it whose green was of an
16 * older tip is checked again, once per move (`base_moved.ts`), newest base
17 * first (`newestBaseFirst`). A queued check merges the newest tip when it
18 * starts, so a later move supersedes nothing and queues nothing. Skipped
19 * where the merge queue gates lands: its gate merges with the tip itself.
20 */
21 export async function recheckBase(lane: LaneState, repo: Repo, branch: string, tip: string): Promise<void> {
22 const { app } = lane;
23 if (landGated(repo)) return;
24 const due: { number: number; green: Run; pull: Pull }[] = [];
25 for (const pull of listOpenPulls(app.db, repo.id)) {
26 if (pull.baseBranch !== branch) continue;
27 const green = dueOf(app, pull, tip);
28 if (green !== undefined) due.push({ number: pull.number, green, pull });
29 }
30 await askInOrder(lane, repo, newestBaseFirst(due).map((item) => item.pull));
31 }
32
33 export const landGated = (repo: Repo): boolean => repo.autonomy === "autonomous" && repo.autonomyBy !== null;
34
35 function dueOf(app: App, pull: Pull, tip: string): Run | undefined {
36 if (pull.state !== "open" || pull.headOid === "" || pull.mergedAt !== null) return undefined;
37 return dueGreen(checksForHead(app.db, pull.id, pull.headOid), `pull-${pull.number}`, tip, false);
38 }
39
40 /**
41 * Asks a re-check for each pull, the first to take a slot first. The queue
42 * starts the newest request first (`pickNext`), so they are asked last-first
43 * with scheduling held until every one is queued: a free slot does not go
44 * to whichever happened to be asked before the rest.
45 */
46 async function askInOrder(lane: LaneState, repo: Repo, pulls: readonly Pull[]): Promise<void> {
47 if (pulls.length === 0) return;
48 lane.holding++;
49 try {
50 for (const pull of [...pulls].reverse()) {
51 try {
52 await lane.request(repo, `pull-${pull.number}`, pull.headOid, BASE_MOVED, pull.id);
53 } catch (error) {
54 log.error("check could not be asked again after the base moved", { pull: pull.number, error: String(error) });
55 }
56 }
57 } finally {
58 lane.holding--;
59 lane.schedule();
60 }
61 }
62
63 /** A check that finished green while its base moved is asked again, once, on the tip now. */
64 export async function recheckFinished(lane: LaneState, runId: string, repo: Repo): Promise<void> {
65 const { app } = lane;
66 // The common path, a red or a gated repository, reads no ref and no pull.
67 if (landGated(repo)) return;
68 const run = getRun(app.db, runId);
69 const number = /^pull-(\d+)$/.exec(run?.env ?? "")?.[1];
70 if (run === undefined || number === undefined || run.state !== "ok" || run.baseOid === "") return;
71 const pull = getPull(app.db, repo.id, Number(number));
72 if (!pull || pull.id !== run.issueId || pull.headOid !== run.commitOid) return;
73 // A branch's tip is a file read (`resolveOid`), no process.
74 const tip = await resolveOid(await pullDir(app.db, app.reposRoot, repo, pull), `refs/heads/${pull.baseBranch}`);
75 if (tip === undefined || tip === run.baseOid) return;
76 if (dueOf(app, pull, tip) !== undefined) await askInOrder(lane, repo, [pull]);
77 }
src/checks/request.ts added +99
@@ -0,0 +1,99 @@
1
2 import { runDir } from "../core/pulls.ts";
3 import { announceRun } from "../actions/runs.ts";
4 import { isContributionFork } from "../core/contribute.ts";
5 import { issueMayRun } from "../core/pull_trust.ts";
6 import type { Repo } from "../core/repos.ts";
7 import { cancelRun, createRun, listActive, type Run } from "../core/runs.ts";
8 import { ASKED } from "./red_main.ts";
9 import { VERIFY_ENV } from "./verify.ts";
10 import { transact } from "../kernel/db.ts";
11 import { createLog } from "../kernel/log.ts";
12 import { readFileAt } from "../apps/checkout.ts";
13 import { parseCheck } from "../apps/manifest.ts";
14 import { accountActive } from "../core/accounts.ts";
15 import type { LaneState } from "./lane_state.ts";
16
17 const log = createLog("checks");
18
19 export async function request(
20 lane: LaneState,
21 repo: Repo,
22 env: string,
23 commitOid: string,
24 reason: string,
25 issueId?: string,
26 actorId?: string,
27 notBefore?: string,
28 ): Promise<Run | undefined> {
29 const { app } = lane;
30 // A suspended or removed owner's code is not run, not even to check it.
31 if (lane.stopped || !accountActive(app.db, repo.ownerId)) return undefined;
32 // A guest's fork runs nothing: its code runs only as a pull a person approved.
33 if (isContributionFork(app.db, repo.id)) return undefined;
34 // A pull from someone who cannot write runs nothing until a writer
35 // approves this exact head (`core/pull_trust.ts`).
36 if (!issueMayRun(app.db, repo, issueId, commitOid)) {
37 log.info("check waits for a writer's approval", {
38 repo: `${repo.ownerName}/${repo.name}`,
39 env,
40 commit: commitOid.slice(0, 7),
41 });
42 return undefined;
43 }
44 // A sealed pull's commit is where its work is, never in the base (`pullDir`).
45 const bare = await runDir(app.db, app.reposRoot, repo, issueId);
46 const [forgeJson, packageJson] = await Promise.all([
47 readFileAt(bare, commitOid, "forge.json"),
48 readFileAt(bare, commitOid, "package.json"),
49 ]);
50 const declared = parseCheck(forgeJson, packageJson);
51 if (declared.ok && declared.value === undefined) return undefined;
52 // Main is verified after landing only where it says how (#7786).
53 if (env === VERIFY_ENV && (!declared.ok || declared.value?.verify == null)) return undefined;
54 // One check per environment at a time: a newer commit supersedes what
55 // is still waiting, and the same commit is not checked twice. Looked for
56 // and written in one write transaction, so a second forge process over
57 // this database (a roll) cannot queue the same check beside it.
58 const superseded: Run[] = [];
59 const { run, fresh } = transact(app.db, () => {
60 for (const active of listActive(app.db, "check")) {
61 if (active.repoId !== repo.id || active.env !== env) continue;
62 // A person asking for a check a red main holds queues it as theirs,
63 // which runs (`checks/red_main.ts`); any other repeat is the same check.
64 if (
65 active.commitOid === commitOid &&
66 !(reason === ASKED && active.state === "queued" && active.reason !== ASKED)
67 )
68 return { run: active, fresh: false };
69 // A pull's running check is for a head nobody will land either:
70 // it is stopped too, below, once the row says so.
71 if (active.state === "queued" || issueId !== undefined) {
72 const gone = cancelRun(
73 app.db,
74 active.id,
75 `superseded by ${commitOid.slice(0, 7)}`,
76 );
77 if (gone && active.state === "running") superseded.push(gone);
78 }
79 }
80 const created = createRun(app.db, {
81 repoId: repo.id,
82 kind: "check",
83 env,
84 commitOid,
85 reason,
86 ...(issueId === undefined ? {} : { issueId }),
87 ...(actorId === undefined ? {} : { actorId }),
88 ...(notBefore === undefined ? {} : { notBefore }),
89 });
90 return { run: created, fresh: true };
91 });
92 // The row is cancelled, so the check's own end finishes nothing and
93 // announces nothing; the abort is what frees its slot and its checkout.
94 for (const old of superseded) lane.inFlight.get(old.id)?.abort.abort();
95 if (!fresh) return run;
96 announceRun(app, run, actorId);
97 lane.schedule();
98 return run;
99 }
src/checks/run_tier.ts added +376
@@ -0,0 +1,376 @@
1
2 import { cellHides } from "../spawn/hosted-mark.ts";
3 import { existsSync, mkdirSync } from "node:fs";
4 import { rm } from "node:fs/promises";
5 import { join } from "node:path";
6 import { noteRunProgress } from "../core/runs.ts";
7 import { runGit } from "../git/env.ts";
8 import { checkKey, provenBy, recordProven } from "./proven.ts";
9 import { giveHistory } from "./history.ts";
10 import { readLearned } from "./learned.ts";
11 import { keepCache, restoreCache } from "./cache.ts";
12 import { clearedIn } from "./base_red.ts";
13 import { scratchEnv } from "./disk_full.ts";
14 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
15 import { KEPT_FILE, keepKept, seedKept } from "./kept-store.ts";
16 import { redDetail } from "./report.ts";
17 import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts";
18 import { failedFiles, onlyEnv, passedFiles } from "./verify.ts";
19 import { exportTree } from "../apps/checkout.ts";
20 import { workerCpus } from "../core/capacity.ts";
21 import { childEnv, killedStep, runStep } from "../apps/process.ts";
22 import { noteReached } from "../apps/egress_report.ts";
23 import { capacityEnv, cpuLimit, egressSetting } from "../core/capacity.ts";
24 import { cellsAvailable, origin, limitWarnings, resolveLimits, STEP_HOSTS, stoppedForCpu, type CellSpec } from "../apps/contain.ts";
25 import { summariseCheckFailure } from "./summary.ts";
26 import { err, type Result } from "../kernel/result.ts";
27 import { JOB_GATE_NOTE, jobGatePolicy, openJobGate } from "./job_gate.ts";
28 import type { Placement } from "./placement.ts";
29 import { refuseSmallPod } from "./pod_room.ts";
30 import { JOB_VANISHED, runJobStep } from "./job_step.ts";
31 import { checkStage } from "./gate_reason.ts";
32 import type { CheckOutcome } from "./runner.ts";
33 import type { LaneState } from "./lane_state.ts";
34 import type { Execution } from "./local_runner.ts";
35 import { resolveSource } from "./tier_source.ts";
36 import { resolveManifest } from "./tier_manifest.ts";
37 import { tryRunner } from "./tier_runner.ts";
38 import { runInstall } from "./tier_install.ts";
39 import { settleFlakes } from "./tier_flake.ts";
40 import { checksRoot, reclaimIfFull, removeRunDir } from "./sweep.ts";
41
42 /**
43 * Run one tier of one check: resolve the commit and the manifest it runs by, give it to a runner
44 * or run it here in a cell, and answer the verdict. The steps it is made of are `tier_*.ts`.
45 */
46 export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutcome> {
47 const { app, options } = lane;
48 const { bare, repo, write, signal } = x;
49 const sourced = await resolveSource(lane, x);
50 if (!sourced.ok) return sourced;
51 const { source, tip, builtOnBase, mergeLine } = sourced;
52 const run = { id: x.id, commitOid: source };
53 const resolved = await resolveManifest(lane, x, source);
54 if (!resolved.ok) return resolved;
55 const { manifest, outside } = resolved;
56 const command =
57 x.tier === "full"
58 ? manifest.full
59 : x.tier === "fast"
60 ? (manifest.fast ?? manifest.run)
61 : x.tier === "verify"
62 ? manifest.verify!
63 : manifest.run;
64 // The fast tier diffs what the pull changes against the commit it builds
65 // on, which the clone is given as a repository of its own: `origin/main`
66 // is not a ref there.
67 const base = manifest.fast !== null ? builtOnBase : undefined;
68 // The tree this runs on under this gate has been green before: that is
69 // the verdict, and nothing is checked out (#1795).
70 const learned =
71 base === undefined ? "" : readLearned(app.config.dataDir, x.pull!.id).join(",");
72 const tree = await runGit(["rev-parse", `${run.commitOid}^{tree}`], { cwd: bare });
73 // A bisection step runs a slice: its green proves nothing of the whole.
74 const key = tree.ok && x.only === undefined
75 ? checkKey({
76 repoId: repo.id,
77 treeOid: tree.value.trim(),
78 command,
79 install: manifest.install,
80 timeout: manifest.timeout,
81 egress: manifest.egress,
82 limits: manifest.limits,
83 base,
84 learned,
85 })
86 : undefined;
87 const earlier = key === undefined ? undefined : provenBy(app.db, key);
88 if (earlier !== undefined) {
89 write(`already proven green by ${earlier}`);
90 return { ok: true, proven: earlier };
91 }
92 const sent = await tryRunner(lane, x, {
93 run,
94 repo,
95 bare,
96 manifest,
97 command,
98 base,
99 builtOnBase,
100 learned,
101 tree,
102 key,
103 outside,
104 write,
105 signal,
106 });
107 if (sent !== undefined) return sent;
108 if (await reclaimIfFull(app)) write("disk under its free floor: swept unused checkouts");
109 const dir = join(checksRoot(app.config.dataDir), run.id);
110 mkdirSync(dir, { recursive: true });
111 write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine);
112 const exported = await exportTree(bare, run.commitOid, dir);
113 if (!exported.ok) return { ok: false, error: exported.error };
114 if (base !== undefined) {
115 const given = await giveHistory(bare, dir, base, run.commitOid);
116 if (!given.ok) {
117 await rm(dir, { recursive: true, force: true });
118 return { ok: false, error: given.error };
119 }
120 }
121 // What the base's own check left (`checks/cache.ts`): tsc's build info
122 // and the gate's per-file times, keyed by the commit this one builds on.
123 const cached = restoreCache(
124 app.config.dataDir,
125 repo.id,
126 builtOnBase === undefined ? [] : [builtOnBase],
127 dir,
128 );
129 if (cached !== undefined) write(`cache from ${cached.slice(0, 7)}`);
130 // Every check starts from the impact map the last green default-branch
131 // run measured; the fast tier selects by it.
132 try {
133 const seeded = seedImpact(app.config.dataDir, repo.id, dir);
134 if (seeded > 0) write(`impact map: ${seeded} tests, from the last green ${repo.defaultBranch}`);
135 } catch (error) {
136 write(`impact map not seeded: ${String(error)}`);
137 }
138 // The fast tier skips a test file whose inputs a default-branch run already proved green.
139 try {
140 const seeded = seedKept(app.config.dataDir, repo.id, dir);
141 if (seeded > 0) write(`kept results: ${seeded} tests, from ${repo.defaultBranch}'s measuring runs`);
142 } catch (error) {
143 write(`kept results not seeded: ${String(error)}`);
144 }
145 // The check is repository code: it runs in a cell of its own
146 // (`apps/contain.ts`), its checkout and a home its only places to write,
147 // the forge, the registry and what it declared its only way out.
148 // A Job holds egress only through a gate the forge opens for it
149 // (`checks/job_gate.ts`, #13251): a cluster that names none cannot
150 // honour forge.json, so the check stays in a cell, never unrestricted.
151 const fellBack = lane.placement.mode === "job" && lane.placement.gate === undefined;
152 // A check a runner did not take runs here as `process` does.
153 const placed: Placement = fellBack || lane.placement.mode === "runner" ? { mode: "process" } : lane.placement;
154 const job = placed.mode === "job" ? placed : undefined;
155 if (x.runId !== undefined)
156 noteRunProgress(app.db, x.runId, { placement: placed.mode });
157 if (fellBack)
158 write("placement: job skipped, this cluster names no egress gate for a Job (FORGE_CHECKS_EGRESS_GATE)");
159 // A Job is its own boundary: no cell here, the cluster and its gate hold it.
160 const cells = job === undefined ? await cellsAvailable() : undefined;
161 if (cells !== undefined) {
162 if (x.runId !== undefined)
163 noteRunProgress(app.db, x.runId, {
164 sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason },
165 });
166 if (!cells.ok)
167 write(`uncontained here: ${cells.reason}; the check runs unrestricted`);
168 } else {
169 if (x.runId !== undefined)
170 noteRunProgress(app.db, x.runId, { sandbox: { enforced: true, note: JOB_GATE_NOTE } });
171 write("placement: job");
172 }
173 const home = `${dir}.home`;
174 const cell: CellSpec | undefined = cells?.ok
175 ? {
176 dir,
177 writable: [],
178 home,
179 hide: cellHides(app),
180 egress: [...STEP_HOSTS, ...manifest.egress],
181 reach: [options.forgeUrl],
182 limits: resolveLimits("step", manifest.limits),
183 onUndeclared: (host, port) =>
184 noteReached(app.config.dataDir, repo.id, host, port),
185 // What an outside contributor's check may reach is held to, whatever the forge's setting.
186 egressMode: outside ? "enforce" : egressSetting(app.db),
187 label: `check:${run.id}`,
188 }
189 : undefined;
190 try {
191 // The pod's memory less the forge's own leaves too little for a cell
192 // (#12519): refused before a step starts, so the forge is never the one
193 // the OOM killer takes. The machine's, not the pull's: no red. A Job's
194 // memory is the cluster's, not this pod's.
195 const small = job === undefined
196 ? refuseSmallPod({ env: process.env["FORGE_CELL_MEMORY_MAX_MIB"], ...options.podRoom }, write)
197 : null;
198 if (small !== null) return small;
199 for (const line of limitWarnings(manifest.limits)) write(line);
200 const cellCpus = resolveLimits("step", manifest.limits).cpus;
201 const env = childEnv({
202 // The shard count an admin set (`core/capacity.ts`), else the
203 // default sized to what this check's cell holds, never the pod's.
204 ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)),
205 CI: "1",
206 FORGE_URL: options.forgeUrl,
207 FORGE_CHECK: run.id,
208 FORGE_COMMIT: x.commitOid,
209 FORGE_CHECK_IMPACT: IMPACT_FILE,
210 FORGE_CHECK_KEPT: KEPT_FILE,
211 ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }),
212 // The cell's CPU limit, which `availableParallelism()` may not
213 // report inside it; the shard count follows this (#2257).
214 FORGE_CELL_CPUS: String(cellCpus),
215 // The one place besides the checkout the cell can write on disk; its
216 // own /tmp is a 256 MiB tmpfs the shards would fill (#8679).
217 ...scratchEnv(cell === undefined ? undefined : home),
218 ...(base === undefined
219 ? {}
220 : {
221 FORGE_CHECK_BASE: base,
222 FORGE_CHECK_LEARNED: learned,
223 }),
224 // A cell inside a cell is not a cell. This check's own cell is the
225 // boundary; a forge the tests boot inside it would wrap its apps in
226 // nested cells, and those never answer in time. Its children run as
227 // plain processes here — as they do on a runner without bwrap.
228 FORGE_CELLS: "off",
229 });
230 let ran: Result<void, string>;
231 const checkLines: string[] = [];
232 // A bisection step runs only some files: what it shows green says nothing of the rest.
233 const wholeVerify = x.tier === "verify" && x.only === undefined;
234 // The gate is held to what a cell holds: the forge, the registry and
235 // what forge.json declares, an outside contributor's always enforced.
236 const gate = job?.gate === undefined
237 ? undefined
238 : openJobGate(
239 job.gate,
240 jobGatePolicy({
241 hosts: [...STEP_HOSTS, ...manifest.egress],
242 origins: [options.forgeUrl].map(origin).filter((o) => o !== undefined),
243 label: `check:${run.id}`,
244 outside,
245 setting: egressSetting(app.db),
246 note: (host, port) => noteReached(app.config.dataDir, repo.id, host, port),
247 }),
248 write,
249 );
250 if (gate !== undefined && !gate.ok) {
251 ran = err(gate.error);
252 } else if (job !== undefined) {
253 try {
254 ran = await runJobStep(job, {
255 runId: run.id,
256 dir,
257 commands: manifest.install === null ? [command] : [manifest.install, command],
258 env,
259 log: write,
260 output: checkLines,
261 signal,
262 timeoutMs: manifest.timeout * 1000,
263 limits: resolveLimits("step", manifest.limits),
264 onNode: (node) => {
265 if (x.runId !== undefined) noteRunProgress(app.db, x.runId, { node });
266 },
267 ...(gate === undefined ? {} : { proxy: gate.value.env }),
268 ...(options.job === undefined ? {} : { options: options.job }),
269 });
270 } finally {
271 gate?.value.close();
272 }
273 } else {
274 const installed = await runInstall(lane, { dir, env, cell, repo, runId: run.id, manifest, write, signal });
275 if (installed !== undefined) return installed;
276 // The check step's own lines are kept beside what `write` already
277 // logs, so a failure can be summarised once here — never by reading
278 // the log back, on this run or any later render of it. `runStep`
279 // echoes the command as its first line before any output of the
280 // command's own; that echo is not output to summarise from.
281 let sawEcho = false;
282 ran = await (options.checkStep ?? runStep)("check", command, {
283 cwd: dir,
284 env,
285 log: (line) => {
286 if (sawEcho) checkLines.push(line);
287 sawEcho = true;
288 write(line);
289 },
290 signal,
291 timeoutMs: manifest.timeout * 1000,
292 cell,
293 });
294 }
295 if (!ran.ok && ran.error === JOB_VANISHED) return { ok: false, error: JOB_VANISHED };
296 ran = await settleFlakes(
297 lane,
298 x,
299 { run, repo, bare, dir, env, cell, manifest, tip, inJob: job !== undefined, checkLines, write, signal },
300 ran,
301 );
302 if (!ran.ok) {
303 const { summary, detail } = summariseCheckFailure(
304 checkLines,
305 ran.error,
306 );
307 const killed = killedStep(ran.error);
308 // A red verification keeps the map it measured all the same, and
309 // names the files it failed in for the bisection (#7786).
310 const files = x.tier === "verify" ? failedFiles(checkLines) : [];
311 // Each failing file once more on its own (#12296): green alone is
312 // flaky under load, which a builder cannot fix; red alone stays red.
313 const flaky = await rerunRed(
314 { files, inJob: job !== undefined, killed, aborted: signal.aborted },
315 async (file, timeoutMs) => {
316 const again = await runStep("rerun", rerunFileCommand(file), {
317 cwd: dir,
318 env,
319 log: write,
320 signal,
321 timeoutMs: Math.min(timeoutMs, manifest.timeout * 1000),
322 cell,
323 });
324 return again.ok;
325 },
326 write,
327 );
328 if (x.tier === "verify" && x.keepImpact === true) {
329 try {
330 const kept = keepImpact(app.config.dataDir, repo.id, dir);
331 if (kept !== undefined) write(kept);
332 const results = keepKept(app.config.dataDir, repo.id, dir, true);
333 if (results !== undefined) write(results);
334 } catch (error) {
335 write(`impact map not kept: ${String(error)}`);
336 }
337 }
338 return {
339 ok: false,
340 error: killed ? ran.error : summary,
341 detail: withFlaky(redDetail(detail, files, checkLines), flaky),
342 ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}),
343 stage: checkStage(ran.error),
344 ...(killed ? { killed } : {}),
345 ...(stoppedForCpu(ran.error) ? { cpuStop: ran.error } : {}),
346 };
347 }
348 // Only a green check of the default branch is kept for the next.
349 if (
350 x.keep === true &&
351 keepCache(app.config.dataDir, repo.id, run.commitOid, dir)
352 )
353 write(`cache kept for ${run.commitOid.slice(0, 7)}`);
354 if (x.tier === "full" || x.keepImpact === true) {
355 try {
356 const kept = keepImpact(app.config.dataDir, repo.id, dir);
357 if (kept !== undefined) write(kept);
358 // Only the default branch's own run hands results back: a pull's tree could write any.
359 const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true);
360 if (results !== undefined) write(results);
361 } catch (error) {
362 write(`impact map not kept: ${String(error)}`);
363 }
364 }
365 if (key !== undefined) recordProven(app.db, key, x.id);
366 const cleared = clearedIn(checkLines);
367 return {
368 ok: true,
369 ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}),
370 ...(cleared === undefined ? {} : { cleared }),
371 };
372 } finally {
373 if (existsSync(dir)) await removeRunDir(dir);
374 await removeRunDir(home);
375 }
376 }
src/checks/scheduler.ts added +131
@@ -0,0 +1,131 @@
1
2 import { activeCrewRuns, crewHoldsCore, forgeCpus } from "../core/one_core.ts";
3 import { getRepoById } from "../core/repos.ts";
4 import { listActive, type Run } from "../core/runs.ts";
5 import { approvedOnce } from "./approved_first.ts";
6 import { outsideOnce } from "./outside_lane.ts";
7 import { redMain, unheld } from "./red_main.ts";
8 import { rebasesFirst } from "./finishing_first.ts";
9 import { inVerifyLane } from "./verify.ts";
10 import { appendRunLog } from "../runs/log.ts";
11 import { memoryHold } from "../runs/memory_hold.ts";
12 import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
13 import { usedRefusal } from "../apps/disk_guard.ts";
14 import { rankOnce } from "./ranked_first.ts";
15 import { gateQueued, gatesInFlight } from "./queue.ts";
16 import type { LaneState } from "./lane_state.ts";
17 import { slotted, capacity } from "./lane_state.ts";
18 import { pickNext, holdsLastSlot } from "./pick.ts";
19 import { start } from "./start.ts";
20 import { startVerify } from "./verify_lane.ts";
21
22 /**
23 * An account at its concurrent-checks limit (`settings/quotas.ts`) keeps
24 * the check queued, the count in its log once, until one of its own ends.
25 */
26 function atLimit(lane: LaneState, run: Run): boolean {
27 const { app } = lane;
28 const owner = getRepoById(app.db, run.repoId)?.ownerId;
29 if (owner === undefined) return false;
30 const full = runningLimitSentence(app.db, "checks", "concurrent checks", owner, () =>
31 runningRuns(app.db, "check", owner, lane.inFlight.keys()),
32 );
33 if (full === undefined) return false;
34 if (!lane.waiting.has(run.id)) {
35 lane.waiting.add(run.id);
36 appendRunLog(app.config.dataDir, run.id, `queued, not started: ${full}`);
37 }
38 return true;
39 }
40
41 /** `pickNext` among `queued`, passing over any whose account is at its limit. */
42 function startable(lane: LaneState, queued: readonly Run[]): Run | undefined {
43 const { app, options } = lane;
44 // At 80% used no check starts (#11799): they stay queued, one line each, and the tick asks
45 // again.
46 const full = usedRefusal(app.config.dataDir);
47 if (full !== undefined) {
48 for (const run of queued)
49 if (!lane.diskWaiting.has(run.id)) {
50 lane.diskWaiting.add(run.id);
51 appendRunLog(app.config.dataDir, run.id, `queued, ${full}`);
52 }
53 return undefined;
54 }
55 // A child near its memory limit runs one check at a time (`runs/memory_hold.ts`).
56 const tight = memoryHold(slotted(lane), options.pressure);
57 if (tight !== undefined) {
58 for (const run of queued)
59 if (!lane.memoryWaiting.has(run.id)) {
60 lane.memoryWaiting.add(run.id);
61 appendRunLog(app.config.dataDir, run.id, `queued, ${tight}`);
62 }
63 return undefined;
64 }
65 let rest = queued;
66 const isApproved = approvedOnce(app.db);
67 const rankOf = rankOnce(app.db);
68 const isOutside = outsideOnce(app.db);
69 let next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside);
70 while (next !== undefined && atLimit(lane, next)) {
71 const held = next;
72 rest = rest.filter((run) => run !== held);
73 next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside);
74 }
75 return next;
76 }
77
78 /**
79 * Fill every free slot with the queued check `pickNext` says deserves it —
80 * one the forge before this one never reached, one a forge stopping
81 * beside this one handed back, or simply the newest push once a slot
82 * opens up. Claiming is what decides between two forges, so calling this
83 * from both never runs the same check twice.
84 */
85 export function schedule(lane: LaneState): void {
86 const { app, options } = lane;
87 if (lane.stopped || lane.holding > 0) return;
88 const limit = capacity(lane);
89 if (options.gate === undefined) lane.gate.resize(limit);
90 // On one CPU a pull's check waits while a crew run works, for at most
91 // `holdMaxMs`; the verify lane, a rebase's check and a main check are the
92 // pipeline and never wait (#11599).
93 const crewHolds = crewHoldsCore((options.cpus ?? forgeCpus)(), options.activeCrewRuns ?? (() => activeCrewRuns(app.db)));
94 // The clock counts one continuous hold: it starts again once no crew run
95 // holds, and forgets checks that are no longer queued.
96 if (!crewHolds) lane.pullHold.reset();
97 else lane.pullHold.keep(new Set(listActive(app.db, "check").filter((run) => run.state === "queued").map((run) => run.id)));
98 startVerify(lane);
99 // Every merge-queue gate in flight holds a slot, a line's first too (#6376).
100 while (slotted(lane) < limit - gatesInFlight()) {
101 // A pull's check waits while its main is red (`checks/red_main.ts`).
102 const queued = unheld(
103 listActive(app.db, "check").filter(
104 (run) =>
105 run.state === "queued" &&
106 !lane.inFlight.has(run.id) &&
107 !inVerifyLane(run.env) &&
108 (run.notBefore === null || Date.parse(run.notBefore) <= Date.now()),
109 ),
110 (repoId) => redMain(app.db, repoId),
111 options.redHoldMs ?? 0,
112 );
113 // The limit is asked at the point of starting, one pick at a time:
114 // a started check is in flight before the next is asked about, so a
115 // burst of one account's checks cannot pass together. A rebase's
116 // check goes before any other (`finishing_first.ts`, #6140).
117 const { rebases, rest: others } = rebasesFirst(app.db, queued);
118 // While a gate is about to start, the last free slot is its: a pull's
119 // check waits for it, a rebase's and a main check do not.
120 const held = holdsLastSlot(limit - gatesInFlight() - slotted(lane), gateQueued());
121 const waiting = crewHolds
122 ? others.filter((run) => run.issueId === null || !lane.pullHold.held(run.id, (options.holdNow ?? Date.now)()))
123 : others;
124 const next = startable(lane, rebases) ?? startable(lane, held ? waiting.filter((run) => run.issueId === null) : waiting);
125 if (!next) return;
126 const repo = getRepoById(app.db, next.repoId);
127 if (!repo) return;
128 lane.pullHold.release(next.id);
129 start(lane, next, repo);
130 }
131 }
src/checks/service.ts +66 −2124
@@ -11,371 +11,39 @@
1111 * is that command's business.
1212 */
1313
14 import { pullDir, runDir } from "../core/pulls.ts";
15 import { cellHides } from "../spawn/hosted-mark.ts";
16 import { childLabel } from "../perf/child_time.ts";
17 import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
18 import { chmod, lstat, readdir, rm } from "node:fs/promises";
19 import { join } from "node:path";
14 import { holdClock } from "../core/one_core.ts";
2015 import type { App } from "../app.ts";
21 import { releaseOnGreen } from "../actions/holds.ts";
22 import { activeCrewRuns, crewHoldsCore, forgeCpus, holdClock } from "../core/one_core.ts";
23 import { announceRun } from "../actions/runs.ts";
24 import { announceShard } from "./shard_event.ts";
25 import type { ForgeEvent } from "../core/events.ts";
26 import { getPull, listOpenPulls, listPulls, type Pull } from "../core/pulls.ts";
27 import { isContributionFork } from "../core/contribute.ts";
28 import { issueMayRun } from "../core/pull_trust.ts";
29 import { outsideAgentWaits } from "../origins/guest.ts";
30 import { ASKS_NEXT, asksLine, heldPull } from "./held.ts";
31 import { OUTSIDE_CHECK_LINE, OUTSIDE_NO_GATE, OUTSIDE_UNCONTAINED } from "./outside.ts";
32 import { getRepoById, type Repo } from "../core/repos.ts";
33 import {
34 cancelRun,
35 getRun,
36 leaseLapsed,
37 cancelRunsForPull,
38 checksForHead,
39 claimRun,
40 createRun,
41 finishRun,
42 listActive,
43 noteRunProgress,
44 overranBefore,
45 setRunBase,
46 setRunCombined,
47 vanishedBefore,
48 VANISHED_SUMMARY,
49 infraBefore,
50 INFRA_SUMMARY,
51 CPU_STOP_SUMMARY,
52 DISK_USED_SUMMARY,
53 requeueCheck,
54 chainCutOffs,
55 CHECK_RELEASE_LIMIT,
56 RELEASE_SUMMARY,
57 REQUEUED_SUMMARY,
58 sweepAbandonedChecks,
59 type Run,
60 } from "../core/runs.ts";
61 import { Gate, runGit } from "../git/env.ts";
62 import { resolveOid } from "../git/merge.ts";
63 import { mergeForCheck } from "./merged.ts";
64 import { readCombined } from "../git/combined.ts";
65 import { approvedOnce } from "./approved_first.ts";
66 import { outsideOnce, ownersFirst } from "./outside_lane.ts";
67 import { checkOnApproval } from "./approval.ts";
68 import { BASE_MOVED, dueGreen, newestBaseFirst } from "./base_moved.ts";
69 import { checkKey, provenBy, recordProven } from "./proven.ts";
70 import { beginGate, endGate, gateHolding } from "./gate_record.ts";
71 import { giveHistory, headDepth } from "./history.ts";
72 import { readLearned } from "./learned.ts";
73 import { keepCache, restoreCache } from "./cache.ts";
74 import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
75 import { parseBunTestFailure } from "./bun-test.ts";
76 import { clearedIn } from "./base_red.ts";
77 import { recordCrashFlakes, passedAlone, recordFlakes, rerunCommand, rerunnable, pullOwnsRed } from "./flake.ts";
78 import { diskRecovered, withDisk } from "./disk_red.ts";
79 import { scratchEnv } from "./disk_full.ts";
80 import { ASKED, redMain, unheld } from "./red_main.ts";
81 import { proposeRevert } from "./revert_red.ts";
82 import { rebasesFirst } from "./finishing_first.ts";
83 import { IMPACT_DIR, IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
84 import { KEPT_DIR, KEPT_FILE, keepKept, seedKept } from "./kept-store.ts";
85 import { afterVerify, closeOnGreen } from "./culprit.ts";
86 import { noteLookPages } from "./look_pages.ts";
87 import { redDetail, reportGreen, reportRed } from "./report.ts";
88 import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts";
89 import {
90 bisectOrigin,
91 failedFiles,
92 inVerifyLane,
93 onlyEnv,
94 passedFiles,
95 VERIFY_ENV,
96 VERIFY_EVERY_MS,
97 VERIFY_REASON,
98 VERIFY_SWEEP_MS,
99 verifyDue,
100 } from "./verify.ts";
101 import { getIssueById } from "../core/issues.ts";
102 import { BUDGETS, budgetSummary, checkLimitMs, STARVATION_MS } from "../kernel/budgets.ts";
103 import { transact } from "../kernel/db.ts";
104 import { createLog } from "../kernel/log.ts";
105 import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts";
106 import { appendRunLog } from "../runs/log.ts";
107 import { memoryHold } from "../runs/memory_hold.ts";
108 import { cancelledLine } from "../runs/cancelled.ts";
109 import { trackRun, untrackRun } from "../runs/stoppable.ts";
110 import { workBegins, workEnds } from "../runs/ending.ts";
111 import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
112 import { exportTree, readFileAt } from "../apps/checkout.ts";
113 import { CHECK_TIMEOUT_MAX, parseCheck } from "../apps/manifest.ts";
114 import { workerCpus } from "../core/capacity.ts";
115 import { accountActive } from "../core/accounts.ts";
116 import { childEnv, killedStep, runStep } from "../apps/process.ts";
117 import { requeueCpuStop } from "./cpu_stop.ts";
118 import { isUsedRefusal, usedRefusal } from "../apps/disk_guard.ts";
119 import { noteReached } from "../apps/egress_report.ts";
120 import {
121 capacityCount,
122 noteBooted,
123 capacityEnv,
124 cpuLimit,
125 egressSetting,
126 } from "../core/capacity.ts";
127 import {
128 cellsAvailable,
129 origin,
130 limitWarnings,
131 resolveLimits,
132 STEP_HOSTS,
133 stoppedForCpu,
134 type CellSpec,
135 } from "../apps/contain.ts";
136 import { summariseCheckFailure } from "./summary.ts";
137 import { err, type Result } from "../kernel/result.ts";
138 import { JOB_GATE_NOTE, jobGatePolicy, openJobGate } from "./job_gate.ts";
139 import { placementFromEnv, type Placement } from "./placement.ts";
140 import { refuseSmallPod, type PodRoom } from "./pod_room.ts";
141 import { JOB_VANISHED, runJobStep } from "./job_step.ts";
142 import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts";
14316 import { sweepJobs } from "./runner_jobs.ts";
144 import {
145 cellInstall,
146 checkInstallCache,
147 meteredInstall,
148 installWithRetry,
149 prefillInstallCache,
150 withLock,
151 } from "./install.ts";
152 import { evictSoon, lend } from "./install_evict.ts";
153 import { checkStage, type GateStage } from "./gate_reason.ts";
154 import { rankOnce } from "./ranked_first.ts";
155 import { describeJob, LOCAL_RUNNER, type CheckOutcome, type Runner } from "./runner.ts";
156 import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts";
157 import type { JobOptions } from "../cluster/jobs.ts";
158 import type { CheckDetail } from "../core/runs.ts";
159 import { settleLandedRuns } from "../crew/interrupted.ts";
17 import { watchGates, extraGates } from "./queue.ts";
18 import { Gate } from "../git/env.ts";
19 import { capacityCount, noteBooted } from "../core/capacity.ts";
20 import { childLabel } from "../perf/child_time.ts";
21 import { VERIFY_SWEEP_MS } from "./verify.ts";
22 import { createLog } from "../kernel/log.ts";
16023 import { during } from "../perf/jobs.ts";
24 import type { ForgeEvent } from "../core/events.ts";
25 import { placementFromEnv } from "./placement.ts";
26 import { capacity, PULL_RESERVE, slotted, type ChecksHandle, type ChecksOptions, type LaneState } from "./lane_state.ts";
27 import { makeLocalRunner } from "./local_runner.ts";
28 import { sweepSoon } from "./sweep.ts";
29 import { request } from "./request.ts";
30 import { schedule } from "./scheduler.ts";
31 import { askVerifications, catchUp } from "./verify_lane.ts";
32 import { onEvent } from "./event_handler.ts";
33 import { gateRun } from "./gate_run.ts";
34 import { runTier } from "./run_tier.ts";
16135
16236 const log = createLog("checks");
16337
164 /** A gate this forge would not start, for it is stopping. */
165 const INTERRUPTED = "this forge is stopping";
166
167 export interface ChecksOptions {
168 /** How many checks may run at once. @default 1 */
169 readonly concurrency?: number | undefined;
170 /** How full this forge's memory is, 0 to 1: its cgroup's unless a test says otherwise. */
171 readonly pressure?: (() => number | undefined) | undefined;
172 /** The forge's public URL, told to the check as `FORGE_URL`. */
173 readonly forgeUrl: string;
174 /** The parent's gate, for a hosted child: its checks take their turn in the parent's lane. */
175 readonly gate?: Gate | undefined;
176 /** Whose checks these are, written first in each run's log (a hosted child's name). */
177 readonly label?: string | undefined;
178 /** How long a check may run before it is killed. @default BUDGETS.gate */
179 readonly budgetMs?: number | undefined;
180 /** How long a verification of the default branch may run. @default BUDGETS.verify */
181 readonly verifyBudgetMs?: number | undefined;
182 /**
183 * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0,
184 * nothing held; the forge says `RED_HOLD_MS`.
185 */
186 readonly redHoldMs?: number | undefined;
187 /**
188 * How long main's check after a land waits before it may start, so the
189 * steps of one candidate supersede each other while queued. @default 0,
190 * none; the forge says `LAND_SETTLE_MS`.
191 */
192 readonly landSettleMs?: number | undefined;
193 /**
194 * How long `stop` lets checks already running finish before it cuts them
195 * off and queues them again (#1796). Nothing new starts meanwhile.
196 * @default 0
197 */
198 readonly drainMs?: number | undefined;
199 /**
200 * Where a check's commands run (`checks/placement.ts`). @default from the
201 * environment: `process`.
202 */
203 readonly placement?: Placement | undefined;
204 /** The kube client a `job` placement speaks through; tests stand one in. */
205 readonly job?: JobOptions | undefined;
206 /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */
207 readonly installRetryMs?: number | undefined;
208 /**
209 * Runs the check step itself; tests stand one in for a stop the cell's watch makes
210 * (#11617). @default runStep
211 */
212 readonly checkStep?: typeof runStep | undefined;
213 /**
214 * Where the pod's memory.max is read and the forge's size
215 * (`checks/pod_room.ts`); tests stand in a small pod. @default the real ones
216 */
217 readonly podRoom?: Omit<PodRoom, "env"> | undefined;
218 /**
219 * How often the default branch is verified after landing at most
220 * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds
221 * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`.
222 */
223 readonly verifyEveryMs?: number | undefined;
224 /** Test seam: stands in for the tier the local runner runs, to see what reaches it. */
225 readonly runTier?: ((x: Execution) => Promise<CheckOutcome>) | undefined;
226 /** Test seam: how often the queue is looked at. @default TICK_MS */
227 readonly tickMs?: number | undefined;
228 /** The forge's CPU count; a 1-CPU forge holds pull checks for a crew run. @default forgeCpus */
229 readonly cpus?: (() => number) | undefined;
230 /** Crew runs working now. @default the running agent runs */
231 readonly activeCrewRuns?: (() => number) | undefined;
232 /** Test seams: how long a held pull check waits at most, and the clock it is judged by. */
233 readonly holdMaxMs?: number | undefined;
234 readonly holdNow?: (() => number) | undefined;
235 }
236
237 /** What one tier run is given (`runTier`). */
238 export interface Execution {
239 readonly bare: string;
240 readonly repo: Repo;
241 /** Names the checkout. */
242 readonly id: string;
243 readonly runId?: string;
244 readonly commitOid: string;
245 readonly pull?: Pull | undefined;
246 /**
247 * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after
248 * landing.
249 */
250 readonly tier: "run" | "fast" | "full" | "verify";
251 /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */
252 readonly only?: readonly string[];
253 /** A bisection step: the commit whose `check.verify` it runs. */
254 readonly commandsFrom?: string;
255 /** A check of the default branch's own push: its cache is kept. */
256 readonly keep?: boolean;
257 /** A green run keeps the impact map it measured; `full` always does. */
258 readonly keepImpact?: boolean;
259 /** The default branch's own run: only its log may open issues. */
260 readonly fileOrderDependent?: boolean;
261 /**
262 * The default branch's own run: a failure of one or two tests is re-run alone before it counts
263 * as red.
264 */
265 readonly isMain?: boolean;
266 readonly write: (line: string) => void;
267 readonly signal: AbortSignal;
268 }
269
270 /** The runner every check goes to: this process (`runner.ts`); `tier` runs one described check. */
271 export function makeLocalRunner(app: App, tier: (x: Execution) => Promise<CheckOutcome>): Runner {
272 return {
273 name: LOCAL_RUNNER,
274 async run(job, io) {
275 const repo = getRepoById(app.db, job.repoId);
276 if (repo === undefined) return { ok: false, error: "the repository is gone" };
277 const bare = await runDir(app.db, app.reposRoot, repo, job.issueId);
278 // Only a pull's own check is judged against the pull, resolved here and nowhere else; the
279 // verification lane never is.
280 const issue =
281 job.tier === "fast" && job.issueId !== null ? getIssueById(app.db, job.issueId) : undefined;
282 const pull = issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined;
283 return tier({
284 bare,
285 repo,
286 id: job.runId,
287 runId: job.runId,
288 commitOid: job.commitOid,
289 pull,
290 tier: job.tier,
291 ...(job.only === undefined ? {} : { only: job.only }),
292 ...(job.commandsFrom === undefined ? {} : { commandsFrom: job.commandsFrom }),
293 keep: job.keep,
294 keepImpact: job.keepImpact,
295 fileOrderDependent: job.fileOrderDependent,
296 isMain: job.isMain,
297 write: io.write,
298 signal: io.signal,
299 });
300 },
301 };
302 }
303
304 /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */
305 export interface GateDone {
306 ok: boolean;
307 error: string;
308 lines: string[];
309 interrupted?: boolean;
310 stage?: GateStage;
311 }
312
313 export interface ChecksHandle {
314 /**
315 * Ask for a check of one commit; the answer is the run, or nothing when the commit declares no
316 * check.
317 */
318 request(
319 repo: Repo,
320 env: string,
321 commitOid: string,
322 reason: string,
323 issueId?: string,
324 actorId?: string,
325 ): Promise<Run | undefined>;
326 /**
327 * The full tier on a merge, before it is made (#1198): `commitOid` is the
328 * merge of the pull with its base, run in a clone of its own. Nothing is
329 * queued and no run is written; the answer is the verdict. `interrupted`
330 * is a gate that did not finish for a reason that is not the code — this
331 * forge stopping, or the cluster losing its Job — and is no verdict (#3155).
332 * `signal` aborting (the queue's deadline, #3430) cuts it off the same way.
333 */
334 gate(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone>;
335 /**
336 * The gate of `commitOid` this forge is running now, to wait on rather
337 * than start again (#6611); undefined when none is.
338 */
339 adopt?(commitOid: string): Promise<GateDone> | undefined;
340 /**
341 * Stop checking. Checks under way get `drainMs` to finish unless `drain`
342 * is false (a person's Ctrl-C): then they are cut off and queued again now.
343 */
344 stop(options?: { drain?: boolean }): Promise<void>;
345 /**
346 * Check slots free now for a merge-queue gate beyond a line's first
347 * (#4053): what neither a running check nor such a gate holds, less
348 * `PULL_RESERVE` kept for pull checks.
349 */
350 spareSlots(): number;
351 /** The gate checks take their turn at; a hosted child's checks share it. */
352 readonly lane: Gate;
353 /**
354 * Ask for every verification after landing that is due now
355 * (`checks/verify.ts`); the answer is the runs asked for. The forge's
356 * own timer calls it when `verifyEveryMs` is set.
357 */
358 verifyDue(now?: number): Promise<Run[]>;
359 }
360
361 /**
362 * Check slots the merge queue's extra gates never take (#4053): pull checks
363 * always have at least these, however long the queue.
364 */
365 export const PULL_RESERVE = 2;
366
367 /**
368 * How long main's check after a land waits to start (#4061). The merge queue
369 * lands a candidate's steps seconds apart, and only its tip was gated: a
370 * step's check that started at once ran the full suite on a commit the next
371 * step replaced, up to seven of them filling the lane for nine minutes. A
372 * step's check that waits is superseded by the next while still queued, and
373 * the tip's is proven green by the gate.
374 */
375 export const LAND_SETTLE_MS = 30_000;
376
377 /** The longest a failed test is given to pass alone, in seconds. */
378 const RERUN_TIMEOUT_S = 300;
38 // The service is cut along its seams, one concept to a module, and these are the names it has
39 // always been imported by.
40 export type { ChecksHandle, ChecksOptions, GateDone } from "./lane_state.ts";
41 export { LAND_SETTLE_MS, PULL_RESERVE } from "./lane_state.ts";
42 export type { Execution } from "./local_runner.ts";
43 export { makeLocalRunner } from "./local_runner.ts";
44 export { holdsLastSlot, pickNext } from "./pick.ts";
45 export { requeueAbandonedChecks } from "./abandoned.ts";
46 export { reclaimIfFull, removeRunDir, sweepCheckouts, sweepSoon } from "./sweep.ts";
37947
38048 /** How often queued checks another forge left are looked for. */
38149 const TICK_MS = 5_000;
@@ -392,202 +60,8 @@ const TICK_MS = 5_000;
39260 export { STARVATION_MS } from "../kernel/budgets.ts";
39361
39462 /**
395 * Whether a pull's check must pass over the free slots left (`free`): while
396 * a merge-queue gate is about to start (`gateQueued`), the last one is the
397 * gate's.
63 * The one place a checks service is kept per forge: `checksFor` reads it, `startChecks` writes it.
39864 */
399 export function holdsLastSlot(free: number, gateWaits: boolean): boolean {
400 return gateWaits && free <= 1;
401 }
402
403 /**
404 * Which queued check takes a slot when one frees: the one requested most
405 * recently, so a push's own check is never stuck behind pulls that queued
406 * hours earlier — unless something has already waited `STARVATION_MS` or
407 * more, in which case the longest-waiting check goes first regardless of
408 * how fresh the rest are. Checks of approved pulls (`approved`) go before
409 * the rest, newest first among themselves (#6396): a check that is the last
410 * thing between a pull and landing is not held up by a new pull's. Ties
411 * (down to the same instant) fall back to id order, which is itself
412 * creation order (`kernel/id.ts`). Among the rest, and among the approved,
413 * the pull whose issue stands highest on the owner's ranked list (`rank`,
414 * 1 at the top) goes first, the unranked last, newest first only as the
415 * tie-break (#12230).
416 */
417 export function pickNext(
418 queued: readonly Run[],
419 now: number = Date.now(),
420 approved: (run: Run) => boolean = () => false,
421 rank: (run: Run) => number | undefined = () => undefined,
422 outside: (run: Run) => boolean = () => false,
423 ): Run | undefined {
424 // An unapproved outside contributor's check goes after everyone else's (`outside_lane.ts`).
425 queued = ownersFirst(queued, outside);
426 if (queued.length === 0) return undefined;
427 const newer = (a: Run, b: Run): boolean =>
428 a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id);
429 let oldest = queued[0]!;
430 const place = (run: Run): number => rank(run) ?? Number.MAX_SAFE_INTEGER;
431 const better = (a: Run, b: Run): boolean => place(a) < place(b) || (place(a) === place(b) && newer(a, b));
432 let newest: Run | undefined;
433 let newestApproved: Run | undefined;
434 for (const run of queued) {
435 if (newer(oldest, run)) oldest = run;
436 if (newest === undefined || better(run, newest)) newest = run;
437 if (approved(run) && (newestApproved === undefined || better(run, newestApproved)))
438 newestApproved = run;
439 }
440 if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest;
441 return newestApproved ?? newest;
442 }
443
444 /**
445 * Settle the checks a dead forge left running, and say so on the bus: a
446 * check whose commit still heads an open pull is queued again for whichever
447 * checks service is up to run, so the change keeps reading *working*.
448 * Called on boot and on the way by whichever forge is alive. A cancelled
449 * check is not announced, as a superseded one is not: `run.finished` reads
450 * as a verdict on the change, and the forge dying is none.
451 */
452 export function requeueAbandonedChecks(app: App, staleMs: number): number {
453 const swept = sweepAbandonedChecks(app.db, staleMs);
454 for (const { cancelled, again } of swept.requeued) {
455 appendRunLog(app.config.dataDir, cancelled.id, REQUEUED_SUMMARY);
456 announceRun(app, again, again.actorId ?? undefined);
457 }
458 for (const run of swept.cancelled)
459 appendRunLog(app.config.dataDir, run.id, run.summary);
460 for (const run of swept.failed) {
461 appendRunLog(app.config.dataDir, run.id, run.summary);
462 announceRun(app, run);
463 }
464 // On the same timer: what a dead or lapsed run left on the disk. Removing
465 // a checkout is thousands of unlinks: off the loop, never in front of /healthz (#7688).
466 sweepSoon(app);
467 return swept.requeued.length + swept.failed.length + swept.cancelled.length;
468 }
469
470 /**
471 * Remove a run's directory even when the cell left read-only files in it (a
472 * bun cache is): `rmSync` cannot unlink inside a directory without write
473 * permission, so permissions are restored and the removal tried again. A
474 * removal that still fails is logged, never swallowed.
475 */
476 export async function removeRunDir(path: string): Promise<void> {
477 try {
478 await rm(path, { recursive: true, force: true });
479 return;
480 } catch {
481 /* fall through to restore permissions */
482 }
483 const open = async (p: string): Promise<void> => {
484 try {
485 if ((await lstat(p)).isSymbolicLink()) return;
486 if ((await lstat(p)).isDirectory()) {
487 await chmod(p, 0o700);
488 for (const n of await readdir(p)) await open(join(p, n));
489 }
490 } catch {
491 /* gone, or not ours: the retry says */
492 }
493 };
494 await open(path);
495 try {
496 await rm(path, { recursive: true, force: true });
497 } catch (error) {
498 log.error("could not remove a run directory", { path, error: String(error) });
499 }
500 }
501
502 /** The sweep of the checkouts under way, if any: one at a time. */
503 let sweeping: Promise<void> | undefined;
504
505 /**
506 * Sweep the checkouts in the background and answer when done. A sweep
507 * already running is the one answered, not doubled; a failure is logged.
508 */
509 export function sweepSoon(app: App, now: number = Date.now()): Promise<void> {
510 sweeping ??= sweepCheckouts(app, now)
511 .catch((error) => log.error("could not sweep the checkouts", { error: String(error) }))
512 .finally(() => {
513 sweeping = undefined;
514 });
515 return sweeping;
516 }
517
518 /** Where a check's checkout lives while it runs. */
519 const checksRoot = (dataDir: string): string => join(dataDir, "checks");
520
521 /**
522 * What is kept under the checks' root that is no checkout: every
523 * repository's impact map (`impact-store.ts`) and kept results
524 * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`),
525 * the gates' logs (`gate_log.ts`) and the last gate's results
526 * (`gate_last.ts`). The sweep took each for a stranger's checkout and
527 * removed it at every boot and every 30 seconds after: a map lived half a
528 * minute, so no check was ever seeded with one (#12178).
529 */
530 const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]);
531
532 /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */
533 const GATE_DIR = /^gate-([0-9a-f]{12})$/;
534
535 /**
536 * Whether a gate's checkout, this forge's or a peer's during a roll, is younger than a gate may
537 * run.
538 */
539 function gateMayBeRunning(path: string, now: number): boolean {
540 try {
541 return now - lstatSync(path).mtimeMs < GATE_DEADLINE_MS;
542 } catch {
543 return false;
544 }
545 }
546
547 /**
548 * Remove the checkouts nobody is using: one whose run is not running with a
549 * live lease, or has no run at all. A forge booting during a roll shares this
550 * directory and database with the old one, whose checks are still running in
551 * theirs; those are left alone. A gate writes no run: its checkout is left
552 * while it is younger than a gate's deadline, and past that while its
553 * `check_gate` row is open, for as long as a check may run. What the checks
554 * keep between runs ({@link STORES}) is no checkout and is never swept.
555 */
556 export async function sweepCheckouts(app: App, now: number = Date.now()): Promise<void> {
557 const root = checksRoot(app.config.dataDir);
558 if (!existsSync(root)) return;
559 const since = new Date(now - CHECK_TIMEOUT_MAX * 1000);
560 for (const name of readdirSync(root)) {
561 if (STORES.has(name)) continue;
562 // A gate has no run row: its checkout is spared until no gate could still be using it.
563 if (name.startsWith("gate-") && gateMayBeRunning(join(root, name), now)) continue;
564 // `<id>.home` is the home of run `<id>`: it lives and dies with it.
565 const id = name.endsWith(".home") ? name.slice(0, -5) : name;
566 const gate = GATE_DIR.exec(id);
567 if (gate !== null) {
568 if (!gateHolding(app.db, gate[1]!, since)) await removeRunDir(join(root, name));
569 continue;
570 }
571 const run = getRun(app.db, id);
572 const live =
573 run !== undefined &&
574 run.state === "running" &&
575 !leaseLapsed(run, STALE_AFTER_MS, now);
576 if (!live) await removeRunDir(join(root, name));
577 }
578 }
579
580 /**
581 * Under the free floor, give back what no check is using before taking more
582 * (#6945): checkouts leaked by a crash sat on the disk until the next boot
583 * while every check failed red on it. Answers whether it swept.
584 */
585 export async function reclaimIfFull(app: App, now: number = Date.now()): Promise<boolean> {
586 if (diskRecovered(app.config.dataDir)) return false;
587 await sweepCheckouts(app, now);
588 return true;
589 }
590
59165 const handles = new WeakMap<App, ChecksHandle>();
59266
59367 /** The checks service of this forge, if one was started. */
@@ -604,1273 +78,36 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
60478 // by the next check, not the next boot. `concurrency` is the boot default.
60579 const booted = options.concurrency ?? 1;
60680 noteBooted(app.db, "checks.concurrency", String(booted));
607 const capacity = (): number =>
608 capacityCount(app.db, "checks.concurrency", booted);
609 const gate = options.gate ?? new Gate(capacity());
61081 const placement = options.placement ?? placementFromEnv();
61182 // Jobs a forge that died left on the runners' desk: their checks are queued again elsewhere.
61283 if (placement.mode === "runner") sweepJobs(app.db);
613 /**
614 * Full gates on land in flight: not slots in the queue, but stopped with
615 * it — after the drain, as a check is. By commit, so a merge queue that
616 * carries on adopts one rather than gate the same tip again (#6611).
617 */
618 const gates = new Map<AbortController, { commitOid: string; done: Promise<GateDone> }>();
619 const inFlight = new Map<
620 string,
621 { promise: Promise<void>; abort: AbortController }
622 >();
623 /** Checks already told why they wait, so the log says it once. */
624 const waiting = new Set<string>();
625 /**
626 * Queued checks a full disk holds that have said so once (#11799); apart from `waiting`, so
627 * neither line hides the other.
628 */
629 const diskWaiting = new Set<string>();
630 const memoryWaiting = new Set<string>();
631 /** The verification-lane run in flight, which holds no check slot. */
632 let verifying: string | undefined;
633 const slotted = (): number => inFlight.size - (verifying === undefined ? 0 : 1);
634 const settle = options.landSettleMs ?? 0;
635 let stopped = false;
636 /** While above 0, a batch of requests is being queued and `schedule` waits for its end. */
637 let holding = 0;
84 const lane: LaneState = {
85 app,
86 options,
87 gate: options.gate ?? new Gate(capacityCount(app.db, "checks.concurrency", booted)),
88 placement,
89 gates: new Map(),
90 inFlight: new Map(),
91 waiting: new Set(),
92 diskWaiting: new Set(),
93 memoryWaiting: new Set(),
94 verifying: undefined,
95 settle: options.landSettleMs ?? 0,
96 stopped: false,
97 holding: 0,
98 pullHold: holdClock(options.holdMaxMs),
99 runner: makeLocalRunner(app, (x) => (options.runTier ?? ((y) => runTier(lane, y)))(x)),
100 schedule: () => schedule(lane),
101 request: (...ask) => request(lane, ...ask),
102 };
638103
639104 sweepSoon(app);
640105
641 async function request(
642 repo: Repo,
643 env: string,
644 commitOid: string,
645 reason: string,
646 issueId?: string,
647 actorId?: string,
648 notBefore?: string,
649 ): Promise<Run | undefined> {
650 // A suspended or removed owner's code is not run, not even to check it.
651 if (stopped || !accountActive(app.db, repo.ownerId)) return undefined;
652 // A guest's fork runs nothing: its code runs only as a pull a person approved.
653 if (isContributionFork(app.db, repo.id)) return undefined;
654 // A pull from someone who cannot write runs nothing until a writer
655 // approves this exact head (`core/pull_trust.ts`).
656 if (!issueMayRun(app.db, repo, issueId, commitOid)) {
657 log.info("check waits for a writer's approval", {
658 repo: `${repo.ownerName}/${repo.name}`,
659 env,
660 commit: commitOid.slice(0, 7),
661 });
662 return undefined;
663 }
664 // A sealed pull's commit is where its work is, never in the base (`pullDir`).
665 const bare = await runDir(app.db, app.reposRoot, repo, issueId);
666 const [forgeJson, packageJson] = await Promise.all([
667 readFileAt(bare, commitOid, "forge.json"),
668 readFileAt(bare, commitOid, "package.json"),
669 ]);
670 const declared = parseCheck(forgeJson, packageJson);
671 if (declared.ok && declared.value === undefined) return undefined;
672 // Main is verified after landing only where it says how (#7786).
673 if (env === VERIFY_ENV && (!declared.ok || declared.value?.verify == null)) return undefined;
674 // One check per environment at a time: a newer commit supersedes what
675 // is still waiting, and the same commit is not checked twice. Looked for
676 // and written in one write transaction, so a second forge process over
677 // this database (a roll) cannot queue the same check beside it.
678 const superseded: Run[] = [];
679 const { run, fresh } = transact(app.db, () => {
680 for (const active of listActive(app.db, "check")) {
681 if (active.repoId !== repo.id || active.env !== env) continue;
682 // A person asking for a check a red main holds queues it as theirs,
683 // which runs (`checks/red_main.ts`); any other repeat is the same check.
684 if (
685 active.commitOid === commitOid &&
686 !(reason === ASKED && active.state === "queued" && active.reason !== ASKED)
687 )
688 return { run: active, fresh: false };
689 // A pull's running check is for a head nobody will land either:
690 // it is stopped too, below, once the row says so.
691 if (active.state === "queued" || issueId !== undefined) {
692 const gone = cancelRun(
693 app.db,
694 active.id,
695 `superseded by ${commitOid.slice(0, 7)}`,
696 );
697 if (gone && active.state === "running") superseded.push(gone);
698 }
699 }
700 const created = createRun(app.db, {
701 repoId: repo.id,
702 kind: "check",
703 env,
704 commitOid,
705 reason,
706 ...(issueId === undefined ? {} : { issueId }),
707 ...(actorId === undefined ? {} : { actorId }),
708 ...(notBefore === undefined ? {} : { notBefore }),
709 });
710 return { run: created, fresh: true };
711 });
712 // The row is cancelled, so the check's own end finishes nothing and
713 // announces nothing; the abort is what frees its slot and its checkout.
714 for (const old of superseded) inFlight.get(old.id)?.abort.abort();
715 if (!fresh) return run;
716 announceRun(app, run, actorId);
717 schedule();
718 return run;
719 }
720
721 /** Run a queued check here, whichever forge queued it. */
722 function start(run: Run, repo: Repo): void {
723 if (inFlight.has(run.id)) return;
724 waiting.delete(run.id);
725 diskWaiting.delete(run.id);
726 memoryWaiting.delete(run.id);
727 const abort = new AbortController();
728 const low = inVerifyLane(run.env);
729 if (low) verifying = run.id;
730 // A person's cancel (`cancelRepoRun`) reaches it through this.
731 trackRun(run.id, abort);
732 workBegins(app.db, run.id);
733 const promise = execute(run, repo, abort.signal, !low)
734 .catch((error) =>
735 log.error("check failed to execute", {
736 run: run.id,
737 error: String(error),
738 }),
739 )
740 .finally(() => {
741 untrackRun(run.id, abort);
742 // Every process the check started has ended: a cancel waiting on
743 // that is heard now (#13291).
744 workEnds(app.db, run.id);
745 inFlight.delete(run.id);
746 if (verifying === run.id) verifying = undefined;
747 // A pull's green that ran while its base moved is of the old tip: asked again once.
748 if (run.issueId !== null && !stopped)
749 void recheckFinished(run.id, repo).catch((error) =>
750 log.error("check could not be asked again after the base moved", { run: run.id, error: String(error) }),
751 );
752 // The slot this run held is free again: give it to whichever queued
753 // check now deserves it most, not whoever asked first.
754 schedule();
755 });
756 inFlight.set(run.id, { promise, abort });
757 }
758
759 /**
760 * An account at its concurrent-checks limit (`settings/quotas.ts`) keeps
761 * the check queued, the count in its log once, until one of its own ends.
762 */
763 function atLimit(run: Run): boolean {
764 const owner = getRepoById(app.db, run.repoId)?.ownerId;
765 if (owner === undefined) return false;
766 const full = runningLimitSentence(app.db, "checks", "concurrent checks", owner, () =>
767 runningRuns(app.db, "check", owner, inFlight.keys()),
768 );
769 if (full === undefined) return false;
770 if (!waiting.has(run.id)) {
771 waiting.add(run.id);
772 appendRunLog(app.config.dataDir, run.id, `queued, not started: ${full}`);
773 }
774 return true;
775 }
776
777 /**
778 * Fill every free slot with the queued check `pickNext` says deserves it —
779 * one the forge before this one never reached, one a forge stopping
780 * beside this one handed back, or simply the newest push once a slot
781 * opens up. Claiming is what decides between two forges, so calling this
782 * from both never runs the same check twice.
783 */
784 /** `pickNext` among `queued`, passing over any whose account is at its limit. */
785 function startable(queued: readonly Run[]): Run | undefined {
786 // At 80% used no check starts (#11799): they stay queued, one line each, and the tick asks
787 // again.
788 const full = usedRefusal(app.config.dataDir);
789 if (full !== undefined) {
790 for (const run of queued)
791 if (!diskWaiting.has(run.id)) {
792 diskWaiting.add(run.id);
793 appendRunLog(app.config.dataDir, run.id, `queued, ${full}`);
794 }
795 return undefined;
796 }
797 // A child near its memory limit runs one check at a time (`runs/memory_hold.ts`).
798 const tight = memoryHold(slotted(), options.pressure);
799 if (tight !== undefined) {
800 for (const run of queued)
801 if (!memoryWaiting.has(run.id)) {
802 memoryWaiting.add(run.id);
803 appendRunLog(app.config.dataDir, run.id, `queued, ${tight}`);
804 }
805 return undefined;
806 }
807 let rest = queued;
808 const isApproved = approvedOnce(app.db);
809 const rankOf = rankOnce(app.db);
810 const isOutside = outsideOnce(app.db);
811 let next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside);
812 while (next !== undefined && atLimit(next)) {
813 const held = next;
814 rest = rest.filter((run) => run !== held);
815 next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside);
816 }
817 return next;
818 }
819
820 const pullHold = holdClock(options.holdMaxMs);
821
822 function schedule(): void {
823 if (stopped || holding > 0) return;
824 const limit = capacity();
825 if (options.gate === undefined) gate.resize(limit);
826 // On one CPU a pull's check waits while a crew run works, for at most
827 // `holdMaxMs`; the verify lane, a rebase's check and a main check are the
828 // pipeline and never wait (#11599).
829 const crewHolds = crewHoldsCore((options.cpus ?? forgeCpus)(), options.activeCrewRuns ?? (() => activeCrewRuns(app.db)));
830 // The clock counts one continuous hold: it starts again once no crew run
831 // holds, and forgets checks that are no longer queued.
832 if (!crewHolds) pullHold.reset();
833 else pullHold.keep(new Set(listActive(app.db, "check").filter((run) => run.state === "queued").map((run) => run.id)));
834 startVerify();
835 // Every merge-queue gate in flight holds a slot, a line's first too (#6376).
836 while (slotted() < limit - gatesInFlight()) {
837 // A pull's check waits while its main is red (`checks/red_main.ts`).
838 const queued = unheld(
839 listActive(app.db, "check").filter(
840 (run) =>
841 run.state === "queued" &&
842 !inFlight.has(run.id) &&
843 !inVerifyLane(run.env) &&
844 (run.notBefore === null || Date.parse(run.notBefore) <= Date.now()),
845 ),
846 (repoId) => redMain(app.db, repoId),
847 options.redHoldMs ?? 0,
848 );
849 // The limit is asked at the point of starting, one pick at a time:
850 // a started check is in flight before the next is asked about, so a
851 // burst of one account's checks cannot pass together. A rebase's
852 // check goes before any other (`finishing_first.ts`, #6140).
853 const { rebases, rest: others } = rebasesFirst(app.db, queued);
854 // While a gate is about to start, the last free slot is its: a pull's
855 // check waits for it, a rebase's and a main check do not.
856 const held = holdsLastSlot(limit - gatesInFlight() - slotted(), gateQueued());
857 const waiting = crewHolds
858 ? others.filter((run) => run.issueId === null || !pullHold.held(run.id, (options.holdNow ?? Date.now)()))
859 : others;
860 const next = startable(rebases) ?? startable(held ? waiting.filter((run) => run.issueId === null) : waiting);
861 if (!next) return;
862 const repo = getRepoById(app.db, next.repoId);
863 if (!repo) return;
864 pullHold.release(next.id);
865 start(next, repo);
866 }
867 }
868
869 /**
870 * The verification lane (#7786): one run at a time, forge-wide, beside
871 * the check slots rather than in one, so no landing and no pull's check
872 * ever waits for it. A bisection step goes before a new verification.
873 */
874 function startVerify(): void {
875 if (verifying !== undefined) return;
876 const queued = listActive(app.db, "check").filter(
877 (run) => run.state === "queued" && inVerifyLane(run.env) && !inFlight.has(run.id),
878 );
879 const next =
880 queued.find((run) => bisectOrigin(run.reason) !== undefined) ?? queued[0];
881 if (next === undefined) return;
882 const repo = getRepoById(app.db, next.repoId);
883 if (repo !== undefined) start(next, repo);
884 }
885
886 /** Every verification due now: main moved since the last, and that one is old enough. */
887 async function askVerifications(now = Date.now()): Promise<Run[]> {
888 const everyMs = options.verifyEveryMs ?? VERIFY_EVERY_MS;
889 const since = new Date(now - 24 * 3_600_000).toISOString();
890 // The newest default-branch check of each repository that had one today.
891 const tips = app.db
892 .query<{ repo_id: string; commit_oid: string }, [string]>(
893 `SELECT r.repo_id, r.commit_oid FROM run r JOIN repo ON repo.id = r.repo_id
894 WHERE r.id IN (SELECT MAX(x.id) FROM run x JOIN repo y ON y.id = x.repo_id
895 WHERE x.kind = 'check' AND x.issue_id IS NULL
896 AND x.env = y.default_branch AND x.created_at >= ?1
897 GROUP BY x.repo_id)`,
898 )
899 .all(since);
900 const asked: Run[] = [];
901 for (const tip of tips) {
902 const last = app.db
903 .query<{ commit_oid: string; created_at: string }, [string]>(
904 `SELECT commit_oid, created_at FROM run
905 WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}'
906 ORDER BY id DESC LIMIT 1`,
907 )
908 .get(tip.repo_id);
909 const active = listActive(app.db, "check").some(
910 (run) => run.repoId === tip.repo_id && inVerifyLane(run.env),
911 );
912 const due = verifyDue({
913 tip: tip.commit_oid,
914 last: last === null ? undefined : { commitOid: last.commit_oid, createdAt: last.created_at },
915 active,
916 now,
917 everyMs,
918 });
919 if (!due) continue;
920 const repo = getRepoById(app.db, tip.repo_id);
921 if (repo === undefined) continue;
922 const run = await request(repo, VERIFY_ENV, tip.commit_oid, VERIFY_REASON);
923 if (run !== undefined) asked.push(run);
924 }
925 return asked;
926 }
927
928 /**
929 * An open pull whose head was never checked — the event that should have
930 * asked landed while a forge was on its way out, or before this forge
931 * existed — is asked for on boot. A pull that stays unchecked would never
932 * land on green, however green it is.
933 */
934 async function catchUp(): Promise<void> {
935 const heads = app.db
936 .query<
937 { repo_id: string; issue_id: string; number: number; head_oid: string },
938 []
939 >(
940 `SELECT issue.repo_id, issue.id AS issue_id, issue.number, pull.head_oid
941 FROM pull JOIN issue ON issue.id = pull.issue_id
942 WHERE issue.state = 'open' AND pull.merged_at IS NULL
943 ORDER BY issue.created_at DESC LIMIT 200`,
944 )
945 .all();
946 for (const head of heads) {
947 if (head.head_oid === "") continue;
948 if (checksForHead(app.db, head.issue_id, head.head_oid).length > 0)
949 continue;
950 const repo = getRepoById(app.db, head.repo_id);
951 if (!repo) continue;
952 try {
953 await request(
954 repo,
955 `pull-${head.number}`,
956 head.head_oid,
957 "boot",
958 head.issue_id,
959 );
960 } catch (error) {
961 log.error("check could not be requested on boot", {
962 pull: head.number,
963 error: String(error),
964 });
965 }
966 }
967 }
968
969 async function execute(
970 run: Run,
971 repo: Repo,
972 signal: AbortSignal,
973 slot = true,
974 ): Promise<void> {
975 // The verification lane takes no slot of the gate (`startVerify`).
976 const release = slot ? await gate.acquire() : () => {};
977 try {
978 // A slot that opens while this forge is stopping is not taken: the run
979 // stays queued for the forge that carries on.
980 if (stopped) return;
981 // Asked again as it starts: a check queued before a rule changed, or
982 // handed back by a forge that stopped, runs only what may run now.
983 if (!issueMayRun(app.db, repo, run.issueId, run.commitOid)) {
984 cancelRun(app.db, run.id, "waiting for a writer to approve this head");
985 return;
986 }
987 const claimed = claimRun(app.db, run.id);
988 if (!claimed) return;
989 const lease = holdLease(app.db, run.id);
990 announceRun(app, claimed);
991 const write = (line: string): void => {
992 appendRunLog(app.config.dataDir, run.id, line);
993 announceShard(app, run, line);
994 };
995 if (options.label !== undefined)
996 write(`forge ${options.label} — checked in its parent's lane`);
997 const started = Date.now();
998 // The budget: past it the check is aborted, whatever it is doing.
999 const limit = checkLimitMs(run.env, {
1000 gate: options.budgetMs ?? BUDGETS.gate,
1001 verify: options.verifyBudgetMs ?? BUDGETS.verify,
1002 review: BUDGETS.review,
1003 });
1004 const budget = new AbortController();
1005 let overran = false;
1006 const forward = (): void => budget.abort();
1007 signal.addEventListener("abort", forward, { once: true });
1008 const clock = setTimeout(() => {
1009 overran = true;
1010 budget.abort();
1011 }, limit);
1012 const outcome = await check(claimed, repo, write, budget.signal).finally(
1013 () => {
1014 clearTimeout(clock);
1015 signal.removeEventListener("abort", forward);
1016 lease();
1017 },
1018 );
1019 // Where a check cut off here is asked again. A verification is of the
1020 // default branch as it stands: it goes back at the branch's newest
1021 // checked commit, the one `askVerifications` would ask of. Left at the
1022 // commit it began on, each restart and each overrun began it there
1023 // again, no other was asked while it was active, and main went
1024 // unverified from 01:48Z to 03:37Z on 2026-11-14 (#12178).
1025 const againAt = (): string | undefined =>
1026 run.env === VERIFY_ENV
1027 ? app.db
1028 .query<{ commit_oid: string }, [string, string]>(
1029 `SELECT commit_oid FROM run WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL
1030 AND env = ?2 ORDER BY id DESC LIMIT 1`,
1031 )
1032 .get(repo.id, repo.defaultBranch)?.commit_oid || undefined
1033 : undefined;
1034 if (overran) {
1035 const summary = budgetSummary(
1036 "a full gate",
1037 Date.now() - started,
1038 limit,
1039 );
1040 write(summary);
1041 if (!overranBefore(app.db, claimed)) {
1042 const moved = requeueCheck(app.db, run.id, undefined, summary, againAt());
1043 if (moved) {
1044 announceRun(app, moved.again, run.actorId ?? undefined);
1045 return;
1046 }
1047 }
1048 // The second overrun: a failure, and the card the vitals carry.
1049 const failed = finishRun(app.db, run.id, {
1050 state: "failed",
1051 summary,
1052 });
1053 if (failed) announceRun(app, failed);
1054 return;
1055 }
1056 // Ended from outside while it ran: its pull landed or closed, and
1057 // the row says so. That is not this check failing.
1058 // A person's cancel ends the log with who cancelled it (#11229).
1059 const row = getRun(app.db, run.id);
1060 if (!outcome.ok && row?.state === "cancelled") {
1061 write(cancelledLine(row) ?? `superseded: ${row.summary}`);
1062 return;
1063 }
1064 if (!outcome.ok && stopped && signal.aborted) {
1065 // Cut off by this forge stopping, not by the check: the same check
1066 // goes back on the queue, and the forge that carries on runs it.
1067 // Past the release ceiling the shutdown is what this check hangs or
1068 // slows: it fails, saying so, instead of riding every release.
1069 if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) {
1070 const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`;
1071 write(summary);
1072 const failed = finishRun(app.db, run.id, { state: "failed", summary });
1073 if (failed) announceRun(app, failed);
1074 return;
1075 }
1076 const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt());
1077 if (!moved) return;
1078 write(RELEASE_SUMMARY);
1079 announceRun(app, moved.again, run.actorId ?? undefined);
1080 return;
1081 }
1082 // The cluster lost the Job (evicted, node gone): interrupted, not
1083 // failed, and queued once more as after a restart; the second loss fails.
1084 if (!outcome.ok && outcome.error === JOB_VANISHED && !vanishedBefore(app.db, claimed)) {
1085 const moved = requeueCheck(app.db, run.id, undefined, VANISHED_SUMMARY);
1086 if (moved) {
1087 write(VANISHED_SUMMARY);
1088 announceRun(app, moved.again, run.actorId ?? undefined);
1089 return;
1090 }
1091 }
1092 // The install failed twice on what the machine did, not the code: the
1093 // check is queued once more and nothing reads it as red (#5093).
1094 if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) {
1095 const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY);
1096 if (moved) {
1097 write(INFRA_SUMMARY);
1098 announceRun(app, moved.again, run.actorId ?? undefined);
1099 return;
1100 }
1101 }
1102 // The disk was 80% used when the step would have started (#11799):
1103 // the machine's wait. Queued again, not red; the pick holds it until there is room.
1104 if (!outcome.ok && isUsedRefusal(outcome.error)) {
1105 const moved = requeueCheck(app.db, run.id, undefined, DISK_USED_SUMMARY);
1106 if (moved) {
1107 write(DISK_USED_SUMMARY);
1108 announceRun(app, moved.again, run.actorId ?? undefined);
1109 return;
1110 }
1111 }
1112 // Stopped for CPU by the cell's watch: the machine's, not the pull's.
1113 // Queued once more on its own, and no builder hears check_failed for
1114 // it; a second stop in a row is reported as any red is (#11617).
1115 if (!outcome.ok && outcome.cpuStop !== undefined) {
1116 const moved = requeueCpuStop(app.db, claimed, outcome.cpuStop);
1117 if (moved) {
1118 write(CPU_STOP_SUMMARY);
1119 announceRun(app, moved.again, run.actorId ?? undefined);
1120 return;
1121 }
1122 }
1123 const seconds = Math.round((Date.now() - started) / 1000);
1124 const done = finishRun(app.db, run.id, {
1125 state: outcome.ok ? "ok" : "failed",
1126 summary: outcome.ok
1127 ? outcome.proven === undefined
1128 ? `check passed in ${seconds}s${outcome.cleared === undefined ? "" : `; ${outcome.cleared}`}`
1129 : `already proven green by ${outcome.proven}`
1130 : `check failed: ${outcome.error}`,
1131 ...(outcome.ok
1132 ? {}
1133 : { checkDetail: withDisk(outcome.detail, app.config.dataDir) }),
1134 });
1135 write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`);
1136 // The pages the gate chose to picture are said on the pull (#13373), worked out by the
1137 // forge from the pull's commits, never read from the log the branch's code writes.
1138 if (done) void noteLookPages(app, repo, done).catch((error) => log.error("look pages note failed", { error: String(error) }));
1139 if (done) announceRun(app, done);
1140 if (done && outcome.ok) releaseOnGreen(app, done);
1141 if (!outcome.ok && run.issueId === null && run.env === repo.defaultBranch)
1142 proposeRevert(app, repo.id).catch((error) =>
1143 log.error("revert proposal failed", { error: String(error) }),
1144 );
1145 // A red verification after landing names its culprit (#7786).
1146 if (done && run.env === VERIFY_ENV) {
1147 reportRed(app, repo, done).forEach(write);
1148 // A file the run printed passing closes its open report issue (#9524).
1149 reportGreen(app, repo, done, outcome.passed ?? []).forEach(write);
1150 // A green that ran every file a culprit's issue names closes it (#11395).
1151 closeOnGreen(app, repo, done, outcome.passed ?? []).forEach(write);
1152 }
1153 if (done && inVerifyLane(run.env))
1154 await afterVerify(app, repo, done, (r, env, oid, reason) => request(r, env, oid, reason))
1155 .then((lines) => lines.forEach(write))
1156 .catch((error) => write(`culprit not named: ${String(error)}`));
1157 } finally {
1158 release();
1159 }
1160 }
1161
1162 const localRunner = makeLocalRunner(app, (x) => (options.runTier ?? runTier)(x));
1163
1164 /** Describe a claimed check as data and hand it to the runner. */
1165 async function check(
1166 run: Run,
1167 repo: Repo,
1168 write: (line: string) => void,
1169 signal: AbortSignal,
1170 ): Promise<CheckOutcome> {
1171 const issue = run.issueId === null ? undefined : getIssueById(app.db, run.issueId);
1172 const lane = inVerifyLane(run.env);
1173 // A bisection step runs only the red's failed files, with the red's own command: an older
1174 // commit may not say how to verify (#7786).
1175 const origin =
1176 lane && run.env !== VERIFY_ENV ? getRun(app.db, bisectOrigin(run.reason) ?? "") : undefined;
1177 const job = describeJob(run, {
1178 defaultBranch: repo.defaultBranch,
1179 verifyEnv: VERIFY_ENV,
1180 inVerifyLane: lane,
1181 pulled: issue?.kind === "pull",
1182 origin:
1183 origin === undefined
1184 ? undefined
1185 : { files: origin.checkDetail?.files ?? [], commitOid: origin.commitOid },
1186 });
1187 if ("error" in job) return { ok: false, error: job.error };
1188 return localRunner.run(job, { write, signal });
1189 }
1190
1191 async function runTier(
1192 x: Execution,
1193 ): Promise<
1194 | {
1195 ok: true;
1196 proven?: string;
1197 passed?: readonly string[];
1198 /** A red it cleared as main's (#10304): `base_red.ts`'s words, kept in the summary. */
1199 cleared?: string;
1200 }
1201 | {
1202 ok: false;
1203 error: string;
1204 detail?: CheckDetail | undefined;
1205 /** A whole verification: the files its log ran to the end green (#9524). */
1206 passed?: readonly string[];
1207 infrastructure?: boolean;
1208 /** The check step's error when the cell's watch stopped it for CPU (#11617). */
1209 cpuStop?: string;
1210 /** Where it stopped red (#5276); none is before any step ran. */
1211 stage?: GateStage;
1212 /** The check step was ended by a signal from outside: no verdict (#6722). */
1213 killed?: boolean;
1214 }
1215 > {
1216 const { bare, repo, write, signal } = x;
1217 // A pull's fast check is of the pull as it would land: its head merged
1218 // into the base's tip. The gate comes from that tree, so a pull forked
1219 // before the base learned something is judged with it.
1220 let source = x.commitOid;
1221 let tip: string | undefined;
1222 // The commit the checked tree builds on: the base tip for a merge made
1223 // now, the base a combined commit names for a kept fork's pull. The fast
1224 // tier diffs against it, so a combined built before main moved is not
1225 // charged with main's later changes.
1226 let builtOnBase: string | undefined;
1227 let mergeLine = "";
1228 if (x.tier === "fast" && x.pull !== undefined) {
1229 tip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`);
1230 if (tip === undefined)
1231 return { ok: false, error: `the base ${x.pull.baseBranch} could not be found` };
1232 // A kept fork's pull is checked on its combined commit (#9932): the
1233 // tree that lands, built on the base tip it names (`combined/N`).
1234 const combined = x.pull.crossRepo
1235 ? await readCombined(bare, x.pull.number, x.commitOid)
1236 : undefined;
1237 if (combined !== undefined && combined.oid !== x.commitOid) {
1238 source = combined.oid;
1239 const builtOn = combined.baseOid === "" ? tip : combined.baseOid;
1240 if (x.runId !== undefined) {
1241 setRunBase(app.db, x.runId, builtOn);
1242 setRunCombined(app.db, x.runId, combined.oid);
1243 }
1244 builtOnBase = builtOn;
1245 mergeLine = `checked as combined with ${x.pull.baseBranch} at ${builtOn.slice(0, 7)} (${combined.oid.slice(0, 7)})`;
1246 } else {
1247 const merged = await mergeForCheck(bare, x.pull.baseBranch, tip, x.commitOid);
1248 if (!merged.ok) return { ok: false, error: merged.error };
1249 source = merged.commit;
1250 if (x.runId !== undefined) setRunBase(app.db, x.runId, tip);
1251 builtOnBase = tip;
1252 mergeLine = merged.line;
1253 }
1254 }
1255 const run = { id: x.id, commitOid: source };
1256 const [forgeJson, packageJson] = await Promise.all([
1257 readFileAt(bare, run.commitOid, "forge.json"),
1258 readFileAt(bare, run.commitOid, "package.json"),
1259 ]);
1260 const declared = parseCheck(forgeJson, packageJson);
1261 if (!declared.ok)
1262 return {
1263 ok: false,
1264 error:
1265 declared.error.kind === "invalid"
1266 ? declared.error.message
1267 : "this commit declares no check",
1268 };
1269 if (declared.value === undefined)
1270 return { ok: false, error: "this commit declares no check" };
1271 // A pull does not choose its own gate: the commands and the timeout come
1272 // from the base branch's `forge.json` where the pull left it, and so do
1273 // the hosts its steps may reach and the limits its cell is given
1274 // (`checks/held.ts`). The branch's own supplies only its install step.
1275 let manifest = declared.value;
1276 let baseTip: string | undefined;
1277 /**
1278 * An outside agent's pull nobody has approved: its code is checked, inside a cell or not at
1279 * all.
1280 */
1281 let outside = false;
1282 if (x.pull !== undefined) {
1283 baseTip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`);
1284 if (baseTip === undefined)
1285 return { ok: false, error: "the pull's base branch could not be found" };
1286 const [baseJson, basePackage] = await Promise.all([
1287 readFileAt(bare, baseTip, "forge.json"),
1288 readFileAt(bare, baseTip, "package.json"),
1289 ]);
1290 const gate = parseCheck(baseJson, basePackage);
1291 if (!gate.ok)
1292 return {
1293 ok: false,
1294 error:
1295 gate.error.kind === "invalid"
1296 ? `${x.pull.baseBranch}: ${gate.error.message}`
1297 : "the base branch declares no check",
1298 };
1299 const held = heldPull(manifest, baseJson, "check");
1300 manifest = held.manifest;
1301 if (gate.value !== undefined) {
1302 manifest = {
1303 ...manifest,
1304 run: gate.value.run,
1305 fast: gate.value.fast,
1306 full: gate.value.full,
1307 timeout: gate.value.timeout,
1308 };
1309 write(
1310 `tier ${x.tier} from ${x.pull.baseBranch}@${baseTip.slice(0, 7)}`,
1311 );
1312 // An outside agent's pull nobody has approved is checked at once, with nobody's
1313 // permission: in a cell, held to that list whatever the forge's egress setting, until a
1314 // writer approves the pull (`origins/guest.ts`).
1315 if (outsideAgentWaits(app.db, x.pull)) {
1316 // Uncontained, a check is a plain process beside the forge's own data: a writer's code
1317 // may be; an outside contributor's waits for a writer here, and says how to go on.
1318 if (!(await cellsAvailable()).ok) return { ok: false, error: OUTSIDE_UNCONTAINED };
1319 outside = true;
1320 write(OUTSIDE_CHECK_LINE(x.pull.baseBranch));
1321 }
1322 } else if (outsideAgentWaits(app.db, x.pull)) {
1323 // No gate on the base branch to hold it to: an outside contributor's pull does not bring
1324 // its own.
1325 return { ok: false, error: OUTSIDE_NO_GATE(x.pull.baseBranch) };
1326 }
1327 // What it asked for and does not have yet, said where a refused request is looked for.
1328 if (held.asks !== undefined) write(`${asksLine(held.asks, x.pull.baseBranch)} ${ASKS_NEXT}`);
1329 }
1330 if (x.commandsFrom !== undefined) {
1331 const from = parseCheck(await readFileAt(bare, x.commandsFrom, "forge.json"), undefined);
1332 if (from.ok && from.value !== undefined) manifest = { ...manifest, verify: from.value.verify, timeout: from.value.timeout };
1333 }
1334 if (x.tier === "verify" && manifest.verify === null)
1335 return { ok: false, error: "this commit declares no check.verify" };
1336 const command =
1337 x.tier === "full"
1338 ? manifest.full
1339 : x.tier === "fast"
1340 ? (manifest.fast ?? manifest.run)
1341 : x.tier === "verify"
1342 ? manifest.verify!
1343 : manifest.run;
1344 // The fast tier diffs what the pull changes against the commit it builds
1345 // on, which the clone is given as a repository of its own: `origin/main`
1346 // is not a ref there.
1347 const base = manifest.fast !== null ? builtOnBase : undefined;
1348 // The tree this runs on under this gate has been green before: that is
1349 // the verdict, and nothing is checked out (#1795).
1350 const learned =
1351 base === undefined ? "" : readLearned(app.config.dataDir, x.pull!.id).join(",");
1352 const tree = await runGit(["rev-parse", `${run.commitOid}^{tree}`], { cwd: bare });
1353 // A bisection step runs a slice: its green proves nothing of the whole.
1354 const key = tree.ok && x.only === undefined
1355 ? checkKey({
1356 repoId: repo.id,
1357 treeOid: tree.value.trim(),
1358 command,
1359 install: manifest.install,
1360 timeout: manifest.timeout,
1361 egress: manifest.egress,
1362 limits: manifest.limits,
1363 base,
1364 learned,
1365 })
1366 : undefined;
1367 const earlier = key === undefined ? undefined : provenBy(app.db, key);
1368 if (earlier !== undefined) {
1369 write(`already proven green by ${earlier}`);
1370 return { ok: true, proven: earlier };
1371 }
1372 // A runner that joined takes the steps (`runner_step.ts`, #13652): only a check that hands
1373 // back nothing but its log and its verdict, and only while a runner is connected. Not taken,
1374 // or an answer that is not the check's own verdict, and it runs here as it always has.
1375 if (placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) {
1376 const lines: string[] = [];
1377 const sent = await runRunnerStep(app, {
1378 runId: x.runId,
1379 repoId: repo.id,
1380 dir: bare,
1381 job: {
1382 run: x.runId,
1383 repo: `${repo.ownerName}/${repo.name}`,
1384 commit: run.commitOid,
1385 ...(base === undefined ? {} : { base }),
1386 depth: base === undefined ? 1 : await headDepth(bare, base, run.commitOid),
1387 steps: [
1388 ...(manifest.install === null ? [] : [{ name: "install" as const, command: manifest.install }]),
1389 { name: "check" as const, command },
1390 ],
1391 // Only the check's own variables cross, never the forge's; the runner adds its cell's.
1392 env: {
1393 CI: "1",
1394 FORGE_URL: options.forgeUrl,
1395 FORGE_CHECK: run.id,
1396 FORGE_COMMIT: x.commitOid,
1397 FORGE_CHECK_IMPACT: IMPACT_FILE,
1398 FORGE_CHECK_KEPT: KEPT_FILE,
1399 ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }),
1400 FORGE_CELLS: "off",
1401 },
1402 limits: {
1403 ...(manifest.limits.cpus === undefined ? {} : { cpus: manifest.limits.cpus }),
1404 ...(manifest.limits.memoryMiB === undefined ? {} : { memoryMiB: manifest.limits.memoryMiB }),
1405 ...(manifest.limits.diskMiB === undefined ? {} : { diskMiB: manifest.limits.diskMiB }),
1406 },
1407 timeoutS: manifest.timeout,
1408 egress: [...STEP_HOSTS, ...manifest.egress],
1409 egressMode: outside ? "enforce" : egressSetting(app.db),
1410 // An outside contributor's code runs in a cell or not at all, there as here.
1411 cell: outside ? "required" : "preferred",
1412 cache: { repo: repo.id, tree: tree.ok ? tree.value.trim() : "", base: builtOnBase ?? "" },
1413 },
1414 log: write,
1415 output: lines,
1416 signal,
1417 waitMs: placement.waitMs,
1418 });
1419 const verdict = sent.taken ? runnerVerdict(sent, lines, signal.aborted) : sent.why;
1420 if (typeof verdict === "string") write(`placement: runner; ${verdict}; running here`);
1421 else {
1422 if (sent.taken && sent.runner !== "")
1423 noteRunProgress(app.db, x.runId, {
1424 placement: "runner",
1425 node: sent.runner,
1426 sandbox: { enforced: sent.contained, note: sent.note },
1427 });
1428 try {
1429 recordCrashFlakes(app.db, app.bus, repo, lines, { runId: run.id, commit: run.commitOid, pull: true }, verdict.ok);
1430 } catch (error) {
1431 write(`crash flakes not recorded: ${String(error)}`);
1432 }
1433 if (!verdict.ok) return verdict;
1434 if (key !== undefined) recordProven(app.db, key, x.id);
1435 const cleared = clearedIn(lines);
1436 return { ok: true, ...(cleared === undefined ? {} : { cleared }) };
1437 }
1438 }
1439 if (await reclaimIfFull(app)) write("disk under its free floor: swept unused checkouts");
1440 const dir = join(checksRoot(app.config.dataDir), run.id);
1441 mkdirSync(dir, { recursive: true });
1442 write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine);
1443 const exported = await exportTree(bare, run.commitOid, dir);
1444 if (!exported.ok) return { ok: false, error: exported.error };
1445 if (base !== undefined) {
1446 const given = await giveHistory(bare, dir, base, run.commitOid);
1447 if (!given.ok) {
1448 await rm(dir, { recursive: true, force: true });
1449 return { ok: false, error: given.error };
1450 }
1451 }
1452 // What the base's own check left (`checks/cache.ts`): tsc's build info
1453 // and the gate's per-file times, keyed by the commit this one builds on.
1454 const cached = restoreCache(
1455 app.config.dataDir,
1456 repo.id,
1457 builtOnBase === undefined ? [] : [builtOnBase],
1458 dir,
1459 );
1460 if (cached !== undefined) write(`cache from ${cached.slice(0, 7)}`);
1461 // Every check starts from the impact map the last green default-branch
1462 // run measured; the fast tier selects by it.
1463 try {
1464 const seeded = seedImpact(app.config.dataDir, repo.id, dir);
1465 if (seeded > 0) write(`impact map: ${seeded} tests, from the last green ${repo.defaultBranch}`);
1466 } catch (error) {
1467 write(`impact map not seeded: ${String(error)}`);
1468 }
1469 // The fast tier skips a test file whose inputs a default-branch run already proved green.
1470 try {
1471 const seeded = seedKept(app.config.dataDir, repo.id, dir);
1472 if (seeded > 0) write(`kept results: ${seeded} tests, from ${repo.defaultBranch}'s measuring runs`);
1473 } catch (error) {
1474 write(`kept results not seeded: ${String(error)}`);
1475 }
1476 // The check is repository code: it runs in a cell of its own
1477 // (`apps/contain.ts`), its checkout and a home its only places to write,
1478 // the forge, the registry and what it declared its only way out.
1479 // A Job holds egress only through a gate the forge opens for it
1480 // (`checks/job_gate.ts`, #13251): a cluster that names none cannot
1481 // honour forge.json, so the check stays in a cell, never unrestricted.
1482 const fellBack = placement.mode === "job" && placement.gate === undefined;
1483 // A check a runner did not take runs here as `process` does.
1484 const placed: Placement = fellBack || placement.mode === "runner" ? { mode: "process" } : placement;
1485 const job = placed.mode === "job" ? placed : undefined;
1486 if (x.runId !== undefined)
1487 noteRunProgress(app.db, x.runId, { placement: placed.mode });
1488 if (fellBack)
1489 write("placement: job skipped, this cluster names no egress gate for a Job (FORGE_CHECKS_EGRESS_GATE)");
1490 // A Job is its own boundary: no cell here, the cluster and its gate hold it.
1491 const cells = job === undefined ? await cellsAvailable() : undefined;
1492 if (cells !== undefined) {
1493 if (x.runId !== undefined)
1494 noteRunProgress(app.db, x.runId, {
1495 sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason },
1496 });
1497 if (!cells.ok)
1498 write(`uncontained here: ${cells.reason}; the check runs unrestricted`);
1499 } else {
1500 if (x.runId !== undefined)
1501 noteRunProgress(app.db, x.runId, { sandbox: { enforced: true, note: JOB_GATE_NOTE } });
1502 write("placement: job");
1503 }
1504 const home = `${dir}.home`;
1505 const cell: CellSpec | undefined = cells?.ok
1506 ? {
1507 dir,
1508 writable: [],
1509 home,
1510 hide: cellHides(app),
1511 egress: [...STEP_HOSTS, ...manifest.egress],
1512 reach: [options.forgeUrl],
1513 limits: resolveLimits("step", manifest.limits),
1514 onUndeclared: (host, port) =>
1515 noteReached(app.config.dataDir, repo.id, host, port),
1516 // What an outside contributor's check may reach is held to, whatever the forge's setting.
1517 egressMode: outside ? "enforce" : egressSetting(app.db),
1518 label: `check:${run.id}`,
1519 }
1520 : undefined;
1521 try {
1522 // The pod's memory less the forge's own leaves too little for a cell
1523 // (#12519): refused before a step starts, so the forge is never the one
1524 // the OOM killer takes. The machine's, not the pull's: no red. A Job's
1525 // memory is the cluster's, not this pod's.
1526 const small = job === undefined
1527 ? refuseSmallPod({ env: process.env["FORGE_CELL_MEMORY_MAX_MIB"], ...options.podRoom }, write)
1528 : null;
1529 if (small !== null) return small;
1530 for (const line of limitWarnings(manifest.limits)) write(line);
1531 const cellCpus = resolveLimits("step", manifest.limits).cpus;
1532 const env = childEnv({
1533 // The shard count an admin set (`core/capacity.ts`), else the
1534 // default sized to what this check's cell holds, never the pod's.
1535 ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)),
1536 CI: "1",
1537 FORGE_URL: options.forgeUrl,
1538 FORGE_CHECK: run.id,
1539 FORGE_COMMIT: x.commitOid,
1540 FORGE_CHECK_IMPACT: IMPACT_FILE,
1541 FORGE_CHECK_KEPT: KEPT_FILE,
1542 ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }),
1543 // The cell's CPU limit, which `availableParallelism()` may not
1544 // report inside it; the shard count follows this (#2257).
1545 FORGE_CELL_CPUS: String(cellCpus),
1546 // The one place besides the checkout the cell can write on disk; its
1547 // own /tmp is a 256 MiB tmpfs the shards would fill (#8679).
1548 ...scratchEnv(cell === undefined ? undefined : home),
1549 ...(base === undefined
1550 ? {}
1551 : {
1552 FORGE_CHECK_BASE: base,
1553 FORGE_CHECK_LEARNED: learned,
1554 }),
1555 // A cell inside a cell is not a cell. This check's own cell is the
1556 // boundary; a forge the tests boot inside it would wrap its apps in
1557 // nested cells, and those never answer in time. Its children run as
1558 // plain processes here — as they do on a runner without bwrap.
1559 FORGE_CELLS: "off",
1560 });
1561 let ran: Result<void, string>;
1562 const checkLines: string[] = [];
1563 // A bisection step runs only some files: what it shows green says nothing of the rest.
1564 const wholeVerify = x.tier === "verify" && x.only === undefined;
1565 // The gate is held to what a cell holds: the forge, the registry and
1566 // what forge.json declares, an outside contributor's always enforced.
1567 const gate = job?.gate === undefined
1568 ? undefined
1569 : openJobGate(
1570 job.gate,
1571 jobGatePolicy({
1572 hosts: [...STEP_HOSTS, ...manifest.egress],
1573 origins: [options.forgeUrl].map(origin).filter((o) => o !== undefined),
1574 label: `check:${run.id}`,
1575 outside,
1576 setting: egressSetting(app.db),
1577 note: (host, port) => noteReached(app.config.dataDir, repo.id, host, port),
1578 }),
1579 write,
1580 );
1581 if (gate !== undefined && !gate.ok) {
1582 ran = err(gate.error);
1583 } else if (job !== undefined) {
1584 try {
1585 ran = await runJobStep(job, {
1586 runId: run.id,
1587 dir,
1588 commands: manifest.install === null ? [command] : [manifest.install, command],
1589 env,
1590 log: write,
1591 output: checkLines,
1592 signal,
1593 timeoutMs: manifest.timeout * 1000,
1594 limits: resolveLimits("step", manifest.limits),
1595 onNode: (node) => {
1596 if (x.runId !== undefined) noteRunProgress(app.db, x.runId, { node });
1597 },
1598 ...(gate === undefined ? {} : { proxy: gate.value.env }),
1599 ...(options.job === undefined ? {} : { options: options.job }),
1600 });
1601 } finally {
1602 gate?.value.close();
1603 }
1604 } else {
1605 if (manifest.install !== null) {
1606 const install = manifest.install;
1607 // A cell's home is its own cache. Uncontained, the repository's
1608 // cache is shared, so installs into it take turns (`checks/install.ts`).
1609 const shared =
1610 cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined;
1611 // A cell installs from that same cache, bound read-only (#7543):
1612 // the forge fills it first, on the host, so no pull writes what
1613 // the gate on main installs. Cluster jobs (`runJobStep`) are
1614 // unchanged: a Job has its own cache and never takes this path.
1615 const warm = cell === undefined ? undefined : checkInstallCache(app.config.dataDir, repo.id);
1616 // Set once the fill has run: a cell installs from the shared cache
1617 // only when the forge filled it; otherwise it keeps a cache of its own.
1618 let filled = false;
1619 const once = (log: (line: string) => void): Promise<Result<void, string>> => {
1620 const base = shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared };
1621 const from = cellInstall(install, base, filled ? warm : undefined);
1622 const revealed = cell !== undefined && filled && warm !== undefined ? { ...cell, reveal: [warm] } : cell;
1623 return runStep("install", from.command, {
1624 cwd: dir,
1625 env: from.env,
1626 log,
1627 signal,
1628 timeoutMs: manifest.timeout * 1000,
1629 cell: revealed,
1630 });
1631 };
1632 // What the cell's proxy received, for the one line that shows the cache
1633 // working (#7546). Uncontained, nothing is logged: the shared cache
1634 // is not metered, and its growth is wrong under concurrent checks.
1635 // Stopped in `finally`, so a throw never leaves the label metered;
1636 // over a MiB from the npm registry adds the miss line (#8382).
1637 const installed = await meteredInstall(
1638 cell === undefined ? undefined : `check:${run.id}`,
1639 repo.id,
1640 () =>
1641 installWithRetry(
1642 async (log) => {
1643 if (shared === undefined) {
1644 if (warm !== undefined) {
1645 const fill = await prefillInstallCache({
1646 cache: warm,
1647 dir,
1648 env,
1649 log,
1650 signal,
1651 timeoutMs: manifest.timeout * 1000,
1652 });
1653 if (!fill.ok) return fill;
1654 filled = fill.value;
1655 void evictSoon(warm, process.env);
1656 }
1657 return once(log);
1658 }
1659 mkdirSync(shared, { recursive: true });
1660 const held = await withLock(
1661 `${shared}.lock`,
1662 async () => {
1663 const done = await once(log);
1664 if (done.ok) lend(shared, dir);
1665 return done;
1666 },
1667 { signal },
1668 );
1669 if (held.ok && held.value.ok) void evictSoon(shared, process.env);
1670 return held.ok ? held.value : held;
1671 },
1672 { log: write, signal, retryMs: options.installRetryMs },
1673 ),
1674 write,
1675 { fetched: true },
1676 );
1677 if (!installed.ok)
1678 return {
1679 ok: false,
1680 error: installed.error,
1681 infrastructure: installed.infrastructure,
1682 stage: "install",
1683 };
1684 }
1685 // The check step's own lines are kept beside what `write` already
1686 // logs, so a failure can be summarised once here — never by reading
1687 // the log back, on this run or any later render of it. `runStep`
1688 // echoes the command as its first line before any output of the
1689 // command's own; that echo is not output to summarise from.
1690 let sawEcho = false;
1691 ran = await (options.checkStep ?? runStep)("check", command, {
1692 cwd: dir,
1693 env,
1694 log: (line) => {
1695 if (sawEcho) checkLines.push(line);
1696 sawEcho = true;
1697 write(line);
1698 },
1699 signal,
1700 timeoutMs: manifest.timeout * 1000,
1701 cell,
1702 });
1703 }
1704 if (!ran.ok && ran.error === JOB_VANISHED) return { ok: false, error: JOB_VANISHED };
1705 // A file the gate cleared because it passed alone is still a failure
1706 // to track: the cell cannot reach the forge, so it is filed from here.
1707 try {
1708 // Only the default branch's own run files: a pull branch controls
1709 // its check's output, and the issues are authored as the owner.
1710 const all =
1711 x.fileOrderDependent === true
1712 ? clearedFiles(checkLines)
1713 : [];
1714 const cleared = all.slice(0, MAX_FILED);
1715 if (cleared.length > 0) {
1716 fileOrderDependent(app.db, app.bus, repo, cleared, {
1717 runId: run.id,
1718 commit: run.commitOid,
1719 });
1720 write(`filed: ${cleared.length} test file(s) that fail only beside others`);
1721 if (all.length > cleared.length)
1722 write(`not filed: ${all.length - cleared.length} more (at most ${MAX_FILED} per run)`);
1723 }
1724 } catch (error) {
1725 write(`order-dependent tests not filed: ${String(error)}`);
1726 }
1727 // Main's red costs the forge every pull's check (#6643): one or two
1728 // failed tests are re-run alone, on this commit in this cell, before
1729 // main counts as red. Passing alone is a flake: green, and recorded.
1730 // A pull's red is the same unless the pull touched the test (#7426).
1731 if (!ran.ok && job === undefined && (x.isMain === true || tip !== undefined)) {
1732 try {
1733 let tests = rerunnable(checkLines, parseBunTestFailure(checkLines));
1734 if (tests !== undefined && x.isMain !== true) {
1735 const diff = await runGit(
1736 ["diff", "--name-only", "-z", `${tip}...${x.commitOid}`],
1737 { cwd: bare },
1738 );
1739 const changed = diff.ok ? diff.value.split("\0").filter((p) => p !== "") : undefined;
1740 const read = (path: string): string | undefined => {
1741 try {
1742 return readFileSync(join(dir, path), "utf8");
1743 } catch {
1744 return undefined;
1745 }
1746 };
1747 if (pullOwnsRed(tests, changed, read)) {
1748 write("not re-run alone: the pull changed a failed test or what it imports");
1749 tests = undefined;
1750 }
1751 }
1752 if (tests !== undefined) {
1753 write(`re-running ${tests.length} failed test(s) alone`);
1754 let alone = true;
1755 for (const test of tests) {
1756 const out: string[] = [];
1757 const again = await runStep("flake", rerunCommand(test), {
1758 cwd: dir,
1759 env,
1760 log: (line) => {
1761 out.push(line);
1762 write(line);
1763 },
1764 signal,
1765 timeoutMs: Math.min(manifest.timeout, RERUN_TIMEOUT_S) * 1000,
1766 cell,
1767 });
1768 if (!again.ok || !passedAlone(out)) {
1769 alone = false;
1770 write(`"${test.name}" fails alone too: ${x.isMain === true ? "main" : "the check"} is red`);
1771 break;
1772 }
1773 }
1774 if (alone) {
1775 recordFlakes(app.db, app.bus, repo, tests, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true });
1776 for (const t of tests) write(`flake: ${t.file} > ${t.name} passed alone`);
1777 write(`flaky: ${tests.map((t) => `"${t.name}"`).join(", ")} failed in the full run and passed alone; recorded, ${x.isMain === true ? "main" : "the check"} stays green`);
1778 ran = { ok: true, value: undefined };
1779 }
1780 }
1781 } catch (error) {
1782 write(`flake re-run failed: ${String(error)}`);
1783 }
1784 }
1785 // A shard that crashed outside any test and ran green once more (#7781).
1786 {
1787 try {
1788 recordCrashFlakes(app.db, app.bus, repo, checkLines, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }, ran.ok);
1789 } catch (error) {
1790 write(`crash flakes not recorded: ${String(error)}`);
1791 }
1792 }
1793 if (!ran.ok) {
1794 const { summary, detail } = summariseCheckFailure(
1795 checkLines,
1796 ran.error,
1797 );
1798 const killed = killedStep(ran.error);
1799 // A red verification keeps the map it measured all the same, and
1800 // names the files it failed in for the bisection (#7786).
1801 const files = x.tier === "verify" ? failedFiles(checkLines) : [];
1802 // Each failing file once more on its own (#12296): green alone is
1803 // flaky under load, which a builder cannot fix; red alone stays red.
1804 const flaky = await rerunRed(
1805 { files, inJob: job !== undefined, killed, aborted: signal.aborted },
1806 async (file, timeoutMs) => {
1807 const again = await runStep("rerun", rerunFileCommand(file), {
1808 cwd: dir,
1809 env,
1810 log: write,
1811 signal,
1812 timeoutMs: Math.min(timeoutMs, manifest.timeout * 1000),
1813 cell,
1814 });
1815 return again.ok;
1816 },
1817 write,
1818 );
1819 if (x.tier === "verify" && x.keepImpact === true) {
1820 try {
1821 const kept = keepImpact(app.config.dataDir, repo.id, dir);
1822 if (kept !== undefined) write(kept);
1823 const results = keepKept(app.config.dataDir, repo.id, dir, true);
1824 if (results !== undefined) write(results);
1825 } catch (error) {
1826 write(`impact map not kept: ${String(error)}`);
1827 }
1828 }
1829 return {
1830 ok: false,
1831 error: killed ? ran.error : summary,
1832 detail: withFlaky(redDetail(detail, files, checkLines), flaky),
1833 ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}),
1834 stage: checkStage(ran.error),
1835 ...(killed ? { killed } : {}),
1836 ...(stoppedForCpu(ran.error) ? { cpuStop: ran.error } : {}),
1837 };
1838 }
1839 // Only a green check of the default branch is kept for the next.
1840 if (
1841 x.keep === true &&
1842 keepCache(app.config.dataDir, repo.id, run.commitOid, dir)
1843 )
1844 write(`cache kept for ${run.commitOid.slice(0, 7)}`);
1845 if (x.tier === "full" || x.keepImpact === true) {
1846 try {
1847 const kept = keepImpact(app.config.dataDir, repo.id, dir);
1848 if (kept !== undefined) write(kept);
1849 // Only the default branch's own run hands results back: a pull's tree could write any.
1850 const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true);
1851 if (results !== undefined) write(results);
1852 } catch (error) {
1853 write(`impact map not kept: ${String(error)}`);
1854 }
1855 }
1856 if (key !== undefined) recordProven(app.db, key, x.id);
1857 const cleared = clearedIn(checkLines);
1858 return {
1859 ok: true,
1860 ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}),
1861 ...(cleared === undefined ? {} : { cleared }),
1862 };
1863 } finally {
1864 if (existsSync(dir)) await removeRunDir(dir);
1865 await removeRunDir(home);
1866 }
1867 }
1868
1869106 // What starts a check: a branch moving, a pull opening, a fork's push
1870107 // moving a pull's head. The commit checked is always the one the pull
1871108 // proposes now.
1872109 const unsubscribe = app.bus.subscribe((event: ForgeEvent) => {
1873 onEvent(event).catch((error) =>
110 onEvent(lane, event).catch((error) =>
1874111 log.error("checks event handler failed", {
1875112 kind: event.kind,
1876113 error: String(error),
@@ -1878,335 +115,40 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
1878115 );
1879116 });
1880117
1881 async function onEvent(event: ForgeEvent): Promise<void> {
1882 if (stopped || event.repoId === null) return;
1883 const repo = getRepoById(app.db, event.repoId);
1884 if (!repo) return;
1885 const actor = event.actorId ?? undefined;
1886 switch (event.kind) {
1887 case "push": {
1888 // A fork that silently followed holds its parent's commit, byte for
1889 // byte: checking the copy proves nothing the parent's own check did
1890 // not, once per fork, on every landed pull. A merge of the parent
1891 // into the fork's own work is a new commit, and is checked.
1892 if (
1893 typeof event.payload["followed"] === "string" &&
1894 event.payload["merged"] !== true
1895 )
1896 return;
1897 const updates = event.payload["updates"];
1898 if (!Array.isArray(updates)) return;
1899 for (const update of updates as { ref?: unknown; newOid?: unknown }[]) {
1900 if (
1901 typeof update.ref !== "string" ||
1902 typeof update.newOid !== "string"
1903 )
1904 continue;
1905 if (
1906 !update.ref.startsWith("refs/heads/") ||
1907 /^0+$/.test(update.newOid)
1908 )
1909 continue;
1910 const branch = update.ref.slice("refs/heads/".length);
1911 if (branch === repo.defaultBranch)
1912 await request(
1913 repo,
1914 branch,
1915 update.newOid,
1916 "push",
1917 undefined,
1918 actor,
1919 );
1920 await recheckBase(repo, branch, update.newOid);
1921 for (const pull of listPulls(app.db, repo.id, {
1922 sealed: true,
1923 state: "open",
1924 limit: 200,
1925 })) {
1926 if (pull.crossRepo || pull.headBranch !== branch) continue;
1927 await request(
1928 repo,
1929 `pull-${pull.number}`,
1930 update.newOid,
1931 "push",
1932 pull.id,
1933 actor,
1934 );
1935 }
1936 }
1937 return;
1938 }
1939 case "pull.opened":
1940 case "pull.reopened": {
1941 const number = event.payload["number"];
1942 if (typeof number !== "number") return;
1943 const pull = getPull(app.db, repo.id, number);
1944 if (!pull) return;
1945 const head = await headOf(repo, pull);
1946 if (head !== undefined)
1947 await request(
1948 repo,
1949 `pull-${pull.number}`,
1950 head,
1951 "pull",
1952 pull.id,
1953 actor,
1954 );
1955 return;
1956 }
1957 case "pull.pushed": {
1958 const number = event.payload["number"];
1959 const headOid = event.payload["headOid"];
1960 if (typeof number !== "number" || typeof headOid !== "string") return;
1961 const pull = getPull(app.db, repo.id, number);
1962 if (!pull) return;
1963 await request(
1964 repo,
1965 `pull-${pull.number}`,
1966 headOid,
1967 "push",
1968 pull.id,
1969 actor,
1970 );
1971 return;
1972 }
1973 // An approval is what lets a reader's pull run: check its head now.
1974 case "pull.run_granted":
1975 case "pull.reviewed": {
1976 const number = event.payload["number"];
1977 if (typeof number !== "number") return;
1978 if (
1979 event.kind === "pull.reviewed" &&
1980 event.payload["state"] !== "approved"
1981 )
1982 return;
1983 const pull = getPull(app.db, repo.id, number);
1984 if (!pull || pull.state !== "open") return;
1985 const head = await headOf(repo, pull);
1986 if (head === undefined) return;
1987 // A head already green is not checked again for nothing (#4391).
1988 const env = `pull-${pull.number}`;
1989 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
1990 const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
1991 // The merge queue gates only an autonomous repository's lands (`crew/autonomy.ts` admit).
1992 const landGated = repo.autonomy === "autonomous" && repo.autonomyBy !== null;
1993 if (!checkOnApproval(checksForHead(app.db, pull.id, head), env, tip, landGated))
1994 return;
1995 await request(repo, env, head, "approved", pull.id, actor);
1996 return;
1997 }
1998 // Landed or closed: nothing left to check, so nothing waits in the lane.
1999 case "pull.merged":
2000 case "pull.closed": {
2001 const number = event.payload["number"];
2002 if (typeof number !== "number") return;
2003 const pull = getPull(app.db, repo.id, number);
2004 if (!pull) return;
2005 const cancelled = cancelRunsForPull(
2006 app.db,
2007 pull.id,
2008 event.kind === "pull.merged" ? "pull landed" : "pull closed",
2009 );
2010 for (const run of cancelled) {
2011 inFlight.get(run.id)?.abort.abort();
2012 announceRun(app, run);
2013 }
2014 // A builder run still in flight when its pull lands reads as done now (#12049).
2015 if (event.kind === "pull.merged")
2016 for (const id of settleLandedRuns(app.db, { issueId: pull.id, headBranch: pull.headBranch, repoId: repo.id, number })) {
2017 const done = getRun(app.db, id);
2018 if (done) announceRun(app, done);
2019 }
2020 // A land moves the default branch without a push event: check its
2021 // new head, or a red main stays red until someone pushes to it. The
2022 // check settles first: a candidate's next step supersedes it.
2023 const mergeOid = event.payload["mergeOid"];
2024 if (
2025 event.kind === "pull.merged" &&
2026 pull.baseBranch === repo.defaultBranch &&
2027 typeof mergeOid === "string" &&
2028 /^[0-9a-f]{40}$/.test(mergeOid)
2029 )
2030 await request(
2031 repo,
2032 repo.defaultBranch,
2033 mergeOid,
2034 "push",
2035 undefined,
2036 actor,
2037 settle > 0 ? new Date(Date.now() + settle).toISOString() : undefined,
2038 );
2039 if (event.kind === "pull.merged" && typeof mergeOid === "string" && /^[0-9a-f]{40}$/.test(mergeOid))
2040 await recheckBase(repo, pull.baseBranch, mergeOid);
2041 return;
2042 }
2043 default:
2044 return;
2045 }
2046 }
2047
2048 /**
2049 * The base moved (#12533): every open pull on it whose green was of an
2050 * older tip is checked again, once per move (`base_moved.ts`), newest base
2051 * first (`newestBaseFirst`). A queued check merges the newest tip when it
2052 * starts, so a later move supersedes nothing and queues nothing. Skipped
2053 * where the merge queue gates lands: its gate merges with the tip itself.
2054 */
2055 async function recheckBase(repo: Repo, branch: string, tip: string): Promise<void> {
2056 if (landGated(repo)) return;
2057 const due: { number: number; green: Run; pull: Pull }[] = [];
2058 for (const pull of listOpenPulls(app.db, repo.id)) {
2059 if (pull.baseBranch !== branch) continue;
2060 const green = dueOf(pull, tip);
2061 if (green !== undefined) due.push({ number: pull.number, green, pull });
2062 }
2063 await askInOrder(repo, newestBaseFirst(due).map((item) => item.pull));
2064 }
2065
2066 const landGated = (repo: Repo): boolean => repo.autonomy === "autonomous" && repo.autonomyBy !== null;
2067
2068 function dueOf(pull: Pull, tip: string): Run | undefined {
2069 if (pull.state !== "open" || pull.headOid === "" || pull.mergedAt !== null) return undefined;
2070 return dueGreen(checksForHead(app.db, pull.id, pull.headOid), `pull-${pull.number}`, tip, false);
2071 }
2072
2073 /**
2074 * Asks a re-check for each pull, the first to take a slot first. The queue
2075 * starts the newest request first (`pickNext`), so they are asked last-first
2076 * with scheduling held until every one is queued: a free slot does not go
2077 * to whichever happened to be asked before the rest.
2078 */
2079 async function askInOrder(repo: Repo, pulls: readonly Pull[]): Promise<void> {
2080 if (pulls.length === 0) return;
2081 holding++;
2082 try {
2083 for (const pull of [...pulls].reverse()) {
2084 try {
2085 await request(repo, `pull-${pull.number}`, pull.headOid, BASE_MOVED, pull.id);
2086 } catch (error) {
2087 log.error("check could not be asked again after the base moved", { pull: pull.number, error: String(error) });
2088 }
2089 }
2090 } finally {
2091 holding--;
2092 schedule();
2093 }
2094 }
2095
2096 /** A check that finished green while its base moved is asked again, once, on the tip now. */
2097 async function recheckFinished(runId: string, repo: Repo): Promise<void> {
2098 // The common path, a red or a gated repository, reads no ref and no pull.
2099 if (landGated(repo)) return;
2100 const run = getRun(app.db, runId);
2101 const number = /^pull-(\d+)$/.exec(run?.env ?? "")?.[1];
2102 if (run === undefined || number === undefined || run.state !== "ok" || run.baseOid === "") return;
2103 const pull = getPull(app.db, repo.id, Number(number));
2104 if (!pull || pull.id !== run.issueId || pull.headOid !== run.commitOid) return;
2105 // A branch's tip is a file read (`resolveOid`), no process.
2106 const tip = await resolveOid(await pullDir(app.db, app.reposRoot, repo, pull), `refs/heads/${pull.baseBranch}`);
2107 if (tip === undefined || tip === run.baseOid) return;
2108 if (dueOf(pull, tip) !== undefined) await askInOrder(repo, [pull]);
2109 }
2110
2111 async function headOf(repo: Repo, pull: Pull): Promise<string | undefined> {
2112 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
2113 const ref = pull.crossRepo
2114 ? `refs/pull/${pull.number}/head`
2115 : `refs/heads/${pull.headBranch}`;
2116 return (await resolveOid(dir, ref)) ?? (pull.headOid || undefined);
2117 }
2118
2119118 // A hosted child's lane runs on its parent's thread: its ticks are named for it
2120119 // (`perf/child_time.ts`).
2121120 const owner = options.label === undefined ? undefined : childLabel(options.label);
2122121 const timer = setInterval(() => {
2123 if (!stopped) during("lane", schedule, owner, app);
122 if (!lane.stopped) during("lane", lane.schedule, owner, app);
2124123 }, options.tickMs ?? TICK_MS);
2125124 // Main is verified after landing on a schedule, when the forge says how often.
2126125 const sweeper =
2127126 options.verifyEveryMs === undefined
2128127 ? undefined
2129128 : setInterval(() => {
2130 if (!stopped)
2131 during("verify-sweep", askVerifications, owner, app).catch((error) =>
129 if (!lane.stopped)
130 during("verify-sweep", () => askVerifications(lane), owner, app).catch((error) =>
2132131 log.error("verification not asked", { error: String(error) }),
2133132 );
2134133 }, Math.min(VERIFY_SWEEP_MS, options.verifyEveryMs));
2135134 // A gate's slot frees: the pull check held for it starts now, not at the tick.
2136 const unwatch = watchGates(schedule);
2137 during("lane", schedule, owner, app);
2138 void during("lane-catch-up", catchUp, owner, app);
135 const unwatch = watchGates(lane.schedule);
136 during("lane", lane.schedule, owner, app);
137 void during("lane-catch-up", () => catchUp(lane), owner, app);
2139138
2140 /** One gate's run on the lane: the full tier on `commitOid`, traced, cut off by `controller`. */
2141 async function gateOnce(
2142 repo: Repo,
2143 pull: Pull,
2144 commitOid: string,
2145 controller: AbortController,
2146 signal: AbortSignal | undefined,
2147 ): Promise<GateDone> {
2148 const lines: string[] = [];
2149 const abandon = (): void => controller.abort();
2150 signal?.addEventListener("abort", abandon, { once: true });
2151 // The slot minutes it spends are the flow's gate minutes (#6426).
2152 const traced = beginGate(app.db, repo.id, pull.id, commitOid);
2153 try {
2154 const bare = await pullDir(app.db, app.reposRoot, repo, pull);
2155 const done = await runTier({
2156 bare,
2157 repo,
2158 id: gateRunId(commitOid),
2159 commitOid,
2160 pull,
2161 tier: "full",
2162 write: (line) => lines.push(line),
2163 signal: controller.signal,
2164 });
2165 if (done.ok) return { ok: true, error: "", lines };
2166 const stage = done.stage ?? "setup";
2167 // Cut off by this forge stopping, the Job lost under it, the check
2168 // killed by a signal from outside (the pod going, the OOM killer), or
2169 // an install the machine failed twice: the code was not judged, so
2170 // this is no red. A gate the drain lets finish is judged (#6611, #6722).
2171 if (
2172 controller.signal.aborted ||
2173 done.error === JOB_VANISHED ||
2174 done.infrastructure === true ||
2175 done.killed === true
2176 )
2177 return { ok: false, error: done.error, lines, interrupted: true, stage };
2178 return { ok: false, error: done.error, lines, stage };
2179 } finally {
2180 endGate(app.db, traced);
2181 signal?.removeEventListener("abort", abandon);
2182 gates.delete(controller);
2183 }
2184 }
2185
2186 /**
2187 * A gate asked of a forge already stopping is not started: the forge that carries on asks it
2188 * again.
2189 */
2190 function gateRun(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone> {
2191 if (stopped) return Promise.resolve({ ok: false, error: INTERRUPTED, lines: [], interrupted: true });
2192 const controller = new AbortController();
2193 const done = gateOnce(repo, pull, commitOid, controller, signal);
2194 gates.set(controller, { commitOid, done });
2195 return done;
2196 }
2197139
2198140 const handle: ChecksHandle = {
2199 request,
2200 gate: gateRun,
141 request: lane.request,
142 gate: (repo, pull, commitOid, signal) => gateRun(lane, repo, pull, commitOid, signal),
2201143 adopt: (commitOid) => {
2202 for (const g of gates.values()) if (g.commitOid === commitOid) return g.done;
144 for (const g of lane.gates.values()) if (g.commitOid === commitOid) return g.done;
2203145 return undefined;
2204146 },
2205 spareSlots: () => Math.max(0, capacity() - PULL_RESERVE - slotted() - extraGates()),
2206 verifyDue: (now) => askVerifications(now),
2207 lane: gate,
147 spareSlots: () => Math.max(0, capacity(lane) - PULL_RESERVE - slotted(lane) - extraGates()),
148 verifyDue: (now) => askVerifications(lane, now),
149 lane: lane.gate,
2208150 stop: async (stopOptions) => {
2209 stopped = true;
151 lane.stopped = true;
2210152 clearInterval(timer);
2211153 if (sweeper !== undefined) clearInterval(sweeper);
2212154 unsubscribe();
@@ -2216,11 +158,11 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
2216158 // A merge-queue gate is under way too: its verdict, if it comes in the
2217159 // drain, is kept for the forge that carries on (#6611).
2218160 const running = Promise.all([
2219 ...[...inFlight.values()].map((f) => f.promise),
2220 ...[...gates.values()].map((g) => g.done),
161 ...[...lane.inFlight.values()].map((f) => f.promise),
162 ...[...lane.gates.values()].map((g) => g.done),
2221163 ]);
2222164 const drainMs = stopOptions?.drain === false ? 0 : (options.drainMs ?? 0);
2223 if (drainMs > 0 && inFlight.size + gates.size > 0) {
165 if (drainMs > 0 && lane.inFlight.size + lane.gates.size > 0) {
2224166 let timer: ReturnType<typeof setTimeout> | undefined;
2225167 await Promise.race([
2226168 running,
@@ -2230,8 +172,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
2230172 ]);
2231173 clearTimeout(timer);
2232174 }
2233 for (const { abort } of inFlight.values()) abort.abort();
2234 for (const abort of gates.keys()) abort.abort();
175 for (const { abort } of lane.inFlight.values()) abort.abort();
176 for (const abort of lane.gates.keys()) abort.abort();
2235177 await running;
2236178 },
2237179 };
src/checks/start.ts added +51
@@ -0,0 +1,51 @@
1
2 import type { Repo } from "../core/repos.ts";
3 import type { Run } from "../core/runs.ts";
4 import { inVerifyLane } from "./verify.ts";
5 import { createLog } from "../kernel/log.ts";
6 import { trackRun, untrackRun } from "../runs/stoppable.ts";
7 import { workBegins, workEnds } from "../runs/ending.ts";
8 import type { LaneState } from "./lane_state.ts";
9 import { execute } from "./execute.ts";
10 import { recheckFinished } from "./recheck_base.ts";
11
12 const log = createLog("checks");
13
14 /** Run a queued check here, whichever forge queued it. */
15 export function start(lane: LaneState, run: Run, repo: Repo): void {
16 const { app } = lane;
17 if (lane.inFlight.has(run.id)) return;
18 lane.waiting.delete(run.id);
19 lane.diskWaiting.delete(run.id);
20 lane.memoryWaiting.delete(run.id);
21 const abort = new AbortController();
22 const low = inVerifyLane(run.env);
23 if (low) lane.verifying = run.id;
24 // A person's cancel (`cancelRepoRun`) reaches it through this.
25 trackRun(run.id, abort);
26 workBegins(app.db, run.id);
27 const promise = execute(lane, run, repo, abort.signal, !low)
28 .catch((error) =>
29 log.error("check failed to execute", {
30 run: run.id,
31 error: String(error),
32 }),
33 )
34 .finally(() => {
35 untrackRun(run.id, abort);
36 // Every process the check started has ended: a cancel waiting on
37 // that is heard now (#13291).
38 workEnds(app.db, run.id);
39 lane.inFlight.delete(run.id);
40 if (lane.verifying === run.id) lane.verifying = undefined;
41 // A pull's green that ran while its base moved is of the old tip: asked again once.
42 if (run.issueId !== null && !lane.stopped)
43 void recheckFinished(lane, run.id, repo).catch((error) =>
44 log.error("check could not be asked again after the base moved", { run: run.id, error: String(error) }),
45 );
46 // The slot this run held is free again: give it to whichever queued
47 // check now deserves it most, not whoever asked first.
48 lane.schedule();
49 });
50 lane.inFlight.set(run.id, { promise, abort });
51 }
src/checks/sweep.ts added +138
@@ -0,0 +1,138 @@
1
2 import { existsSync, lstatSync, readdirSync } from "node:fs";
3 import { chmod, lstat, readdir, rm } from "node:fs/promises";
4 import { join } from "node:path";
5 import type { App } from "../app.ts";
6 import { getRun, leaseLapsed } from "../core/runs.ts";
7 import { gateHolding } from "./gate_record.ts";
8 import { diskRecovered } from "./disk_red.ts";
9 import { IMPACT_DIR } from "./impact-store.ts";
10 import { KEPT_DIR } from "./kept-store.ts";
11 import { createLog } from "../kernel/log.ts";
12 import { STALE_AFTER_MS } from "../runs/lease.ts";
13 import { CHECK_TIMEOUT_MAX } from "../apps/manifest.ts";
14 import { GATE_DEADLINE_MS } from "./queue.ts";
15
16
17 const log = createLog("checks");
18
19 /**
20 * Remove a run's directory even when the cell left read-only files in it (a
21 * bun cache is): `rmSync` cannot unlink inside a directory without write
22 * permission, so permissions are restored and the removal tried again. A
23 * removal that still fails is logged, never swallowed.
24 */
25 export async function removeRunDir(path: string): Promise<void> {
26 try {
27 await rm(path, { recursive: true, force: true });
28 return;
29 } catch {
30 /* fall through to restore permissions */
31 }
32 const open = async (p: string): Promise<void> => {
33 try {
34 if ((await lstat(p)).isSymbolicLink()) return;
35 if ((await lstat(p)).isDirectory()) {
36 await chmod(p, 0o700);
37 for (const n of await readdir(p)) await open(join(p, n));
38 }
39 } catch {
40 /* gone, or not ours: the retry says */
41 }
42 };
43 await open(path);
44 try {
45 await rm(path, { recursive: true, force: true });
46 } catch (error) {
47 log.error("could not remove a run directory", { path, error: String(error) });
48 }
49 }
50
51 /** The sweep of the checkouts under way, if any: one at a time. */
52 let sweeping: Promise<void> | undefined;
53
54 /**
55 * Sweep the checkouts in the background and answer when done. A sweep
56 * already running is the one answered, not doubled; a failure is logged.
57 */
58 export function sweepSoon(app: App, now: number = Date.now()): Promise<void> {
59 sweeping ??= sweepCheckouts(app, now)
60 .catch((error) => log.error("could not sweep the checkouts", { error: String(error) }))
61 .finally(() => {
62 sweeping = undefined;
63 });
64 return sweeping;
65 }
66
67 /** Where a check's checkout lives while it runs. */
68 export const checksRoot = (dataDir: string): string => join(dataDir, "checks");
69
70 /**
71 * What is kept under the checks' root that is no checkout: every
72 * repository's impact map (`impact-store.ts`) and kept results
73 * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`),
74 * the gates' logs (`gate_log.ts`) and the last gate's results
75 * (`gate_last.ts`). The sweep took each for a stranger's checkout and
76 * removed it at every boot and every 30 seconds after: a map lived half a
77 * minute, so no check was ever seeded with one (#12178).
78 */
79 const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]);
80
81 /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */
82 const GATE_DIR = /^gate-([0-9a-f]{12})$/;
83
84 /**
85 * Whether a gate's checkout, this forge's or a peer's during a roll, is younger than a gate may
86 * run.
87 */
88 function gateMayBeRunning(path: string, now: number): boolean {
89 try {
90 return now - lstatSync(path).mtimeMs < GATE_DEADLINE_MS;
91 } catch {
92 return false;
93 }
94 }
95
96 /**
97 * Remove the checkouts nobody is using: one whose run is not running with a
98 * live lease, or has no run at all. A forge booting during a roll shares this
99 * directory and database with the old one, whose checks are still running in
100 * theirs; those are left alone. A gate writes no run: its checkout is left
101 * while it is younger than a gate's deadline, and past that while its
102 * `check_gate` row is open, for as long as a check may run. What the checks
103 * keep between runs ({@link STORES}) is no checkout and is never swept.
104 */
105 export async function sweepCheckouts(app: App, now: number = Date.now()): Promise<void> {
106 const root = checksRoot(app.config.dataDir);
107 if (!existsSync(root)) return;
108 const since = new Date(now - CHECK_TIMEOUT_MAX * 1000);
109 for (const name of readdirSync(root)) {
110 if (STORES.has(name)) continue;
111 // A gate has no run row: its checkout is spared until no gate could still be using it.
112 if (name.startsWith("gate-") && gateMayBeRunning(join(root, name), now)) continue;
113 // `<id>.home` is the home of run `<id>`: it lives and dies with it.
114 const id = name.endsWith(".home") ? name.slice(0, -5) : name;
115 const gate = GATE_DIR.exec(id);
116 if (gate !== null) {
117 if (!gateHolding(app.db, gate[1]!, since)) await removeRunDir(join(root, name));
118 continue;
119 }
120 const run = getRun(app.db, id);
121 const live =
122 run !== undefined &&
123 run.state === "running" &&
124 !leaseLapsed(run, STALE_AFTER_MS, now);
125 if (!live) await removeRunDir(join(root, name));
126 }
127 }
128
129 /**
130 * Under the free floor, give back what no check is using before taking more
131 * (#6945): checkouts leaked by a crash sat on the disk until the next boot
132 * while every check failed red on it. Answers whether it swept.
133 */
134 export async function reclaimIfFull(app: App, now: number = Date.now()): Promise<boolean> {
135 if (diskRecovered(app.config.dataDir)) return false;
136 await sweepCheckouts(app, now);
137 return true;
138 }
src/checks/tier_flake.ts added +140
@@ -0,0 +1,140 @@
1
2 import { readFileSync } from "node:fs";
3 import { join } from "node:path";
4 import type { Repo } from "../core/repos.ts";
5 import { runGit } from "../git/env.ts";
6 import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
7 import { parseBunTestFailure } from "./bun-test.ts";
8 import { recordCrashFlakes, passedAlone, recordFlakes, rerunCommand, rerunnable, pullOwnsRed } from "./flake.ts";
9 import { childEnv, runStep } from "../apps/process.ts";
10 import type { CellSpec } from "../apps/contain.ts";
11 import type { Result } from "../kernel/result.ts";
12 import type { LaneState } from "./lane_state.ts";
13 import type { Execution } from "./local_runner.ts";
14 import type { CheckManifest } from "../apps/manifest.ts";
15
16 /** The longest a failed test is given to pass alone, in seconds. */
17 const RERUN_TIMEOUT_S = 300;
18
19 /** What the flake passes look at once the check step has ended. */
20 export interface FlakeCheck {
21 readonly run: { readonly id: string; readonly commitOid: string };
22 readonly repo: Repo;
23 readonly bare: string;
24 readonly dir: string;
25 readonly env: ReturnType<typeof childEnv>;
26 readonly cell: CellSpec | undefined;
27 readonly manifest: CheckManifest;
28 /** The base tip a pull's check merged onto; none for a check of a branch. */
29 readonly tip: string | undefined;
30 /** Whether the check ran in a cluster Job, where it is not re-run here. */
31 readonly inJob: boolean;
32 readonly checkLines: readonly string[];
33 readonly write: (line: string) => void;
34 readonly signal: AbortSignal;
35 }
36
37 /**
38 * What a red check's own log can still excuse: a file filed because it fails only beside others,
39 * a failed test that passes alone (a flake, so green), a shard that crashed outside any test.
40 * Answers the check step's result, made green when the red was a flake.
41 */
42 export async function settleFlakes(
43 lane: LaneState,
44 x: Execution,
45 c: FlakeCheck,
46 result: Result<void, string>,
47 ): Promise<Result<void, string>> {
48 const { app } = lane;
49 const { run, repo, bare, dir, env, cell, manifest, tip, inJob, checkLines, write, signal } = c;
50 let ran = result;
51 // A file the gate cleared because it passed alone is still a failure
52 // to track: the cell cannot reach the forge, so it is filed from here.
53 try {
54 // Only the default branch's own run files: a pull branch controls
55 // its check's output, and the issues are authored as the owner.
56 const all =
57 x.fileOrderDependent === true
58 ? clearedFiles(checkLines)
59 : [];
60 const cleared = all.slice(0, MAX_FILED);
61 if (cleared.length > 0) {
62 fileOrderDependent(app.db, app.bus, repo, cleared, {
63 runId: run.id,
64 commit: run.commitOid,
65 });
66 write(`filed: ${cleared.length} test file(s) that fail only beside others`);
67 if (all.length > cleared.length)
68 write(`not filed: ${all.length - cleared.length} more (at most ${MAX_FILED} per run)`);
69 }
70 } catch (error) {
71 write(`order-dependent tests not filed: ${String(error)}`);
72 }
73 // Main's red costs the forge every pull's check (#6643): one or two
74 // failed tests are re-run alone, on this commit in this cell, before
75 // main counts as red. Passing alone is a flake: green, and recorded.
76 // A pull's red is the same unless the pull touched the test (#7426).
77 if (!ran.ok && !inJob && (x.isMain === true || tip !== undefined)) {
78 try {
79 let tests = rerunnable(checkLines, parseBunTestFailure(checkLines));
80 if (tests !== undefined && x.isMain !== true) {
81 const diff = await runGit(
82 ["diff", "--name-only", "-z", `${tip}...${x.commitOid}`],
83 { cwd: bare },
84 );
85 const changed = diff.ok ? diff.value.split("\0").filter((p) => p !== "") : undefined;
86 const read = (path: string): string | undefined => {
87 try {
88 return readFileSync(join(dir, path), "utf8");
89 } catch {
90 return undefined;
91 }
92 };
93 if (pullOwnsRed(tests, changed, read)) {
94 write("not re-run alone: the pull changed a failed test or what it imports");
95 tests = undefined;
96 }
97 }
98 if (tests !== undefined) {
99 write(`re-running ${tests.length} failed test(s) alone`);
100 let alone = true;
101 for (const test of tests) {
102 const out: string[] = [];
103 const again = await runStep("flake", rerunCommand(test), {
104 cwd: dir,
105 env,
106 log: (line) => {
107 out.push(line);
108 write(line);
109 },
110 signal,
111 timeoutMs: Math.min(manifest.timeout, RERUN_TIMEOUT_S) * 1000,
112 cell,
113 });
114 if (!again.ok || !passedAlone(out)) {
115 alone = false;
116 write(`"${test.name}" fails alone too: ${x.isMain === true ? "main" : "the check"} is red`);
117 break;
118 }
119 }
120 if (alone) {
121 recordFlakes(app.db, app.bus, repo, tests, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true });
122 for (const t of tests) write(`flake: ${t.file} > ${t.name} passed alone`);
123 write(`flaky: ${tests.map((t) => `"${t.name}"`).join(", ")} failed in the full run and passed alone; recorded, ${x.isMain === true ? "main" : "the check"} stays green`);
124 ran = { ok: true, value: undefined };
125 }
126 }
127 } catch (error) {
128 write(`flake re-run failed: ${String(error)}`);
129 }
130 }
131 // A shard that crashed outside any test and ran green once more (#7781).
132 {
133 try {
134 recordCrashFlakes(app.db, app.bus, repo, checkLines, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }, ran.ok);
135 } catch (error) {
136 write(`crash flakes not recorded: ${String(error)}`);
137 }
138 }
139 return ran;
140 }
src/checks/tier_install.ts added +114
@@ -0,0 +1,114 @@
1
2 import { mkdirSync } from "node:fs";
3 import type { Repo } from "../core/repos.ts";
4 import { childEnv, runStep } from "../apps/process.ts";
5 import type { CellSpec } from "../apps/contain.ts";
6 import type { Result } from "../kernel/result.ts";
7 import { cellInstall, checkInstallCache, meteredInstall, installWithRetry, prefillInstallCache, withLock } from "./install.ts";
8 import { evictSoon, lend } from "./install_evict.ts";
9 import type { CheckOutcome } from "./runner.ts";
10 import type { LaneState } from "./lane_state.ts";
11 import type { CheckManifest } from "../apps/manifest.ts";
12
13 /** What an install step is run with. */
14 export interface InstallStep {
15 readonly dir: string;
16 readonly env: ReturnType<typeof childEnv>;
17 readonly cell: CellSpec | undefined;
18 readonly repo: Repo;
19 readonly runId: string;
20 readonly manifest: CheckManifest;
21 readonly write: (line: string) => void;
22 readonly signal: AbortSignal;
23 }
24
25 /**
26 * The check's install step, here, in its cell or on the shared cache. Answers a failure, else
27 * nothing.
28 */
29 export async function runInstall(lane: LaneState, c: InstallStep): Promise<CheckOutcome | undefined> {
30 const { app, options } = lane;
31 const { dir, env, cell, repo, manifest, write, signal } = c;
32 const run = { id: c.runId };
33 if (manifest.install !== null) {
34 const install = manifest.install;
35 // A cell's home is its own cache. Uncontained, the repository's
36 // cache is shared, so installs into it take turns (`checks/install.ts`).
37 const shared =
38 cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined;
39 // A cell installs from that same cache, bound read-only (#7543):
40 // the forge fills it first, on the host, so no pull writes what
41 // the gate on main installs. Cluster jobs (`runJobStep`) are
42 // unchanged: a Job has its own cache and never takes this path.
43 const warm = cell === undefined ? undefined : checkInstallCache(app.config.dataDir, repo.id);
44 // Set once the fill has run: a cell installs from the shared cache
45 // only when the forge filled it; otherwise it keeps a cache of its own.
46 let filled = false;
47 const once = (log: (line: string) => void): Promise<Result<void, string>> => {
48 const base = shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared };
49 const from = cellInstall(install, base, filled ? warm : undefined);
50 const revealed = cell !== undefined && filled && warm !== undefined ? { ...cell, reveal: [warm] } : cell;
51 return runStep("install", from.command, {
52 cwd: dir,
53 env: from.env,
54 log,
55 signal,
56 timeoutMs: manifest.timeout * 1000,
57 cell: revealed,
58 });
59 };
60 // What the cell's proxy received, for the one line that shows the cache
61 // working (#7546). Uncontained, nothing is logged: the shared cache
62 // is not metered, and its growth is wrong under concurrent checks.
63 // Stopped in `finally`, so a throw never leaves the label metered;
64 // over a MiB from the npm registry adds the miss line (#8382).
65 const installed = await meteredInstall(
66 cell === undefined ? undefined : `check:${run.id}`,
67 repo.id,
68 () =>
69 installWithRetry(
70 async (log) => {
71 if (shared === undefined) {
72 if (warm !== undefined) {
73 const fill = await prefillInstallCache({
74 cache: warm,
75 dir,
76 env,
77 log,
78 signal,
79 timeoutMs: manifest.timeout * 1000,
80 });
81 if (!fill.ok) return fill;
82 filled = fill.value;
83 void evictSoon(warm, process.env);
84 }
85 return once(log);
86 }
87 mkdirSync(shared, { recursive: true });
88 const held = await withLock(
89 `${shared}.lock`,
90 async () => {
91 const done = await once(log);
92 if (done.ok) lend(shared, dir);
93 return done;
94 },
95 { signal },
96 );
97 if (held.ok && held.value.ok) void evictSoon(shared, process.env);
98 return held.ok ? held.value : held;
99 },
100 { log: write, signal, retryMs: options.installRetryMs },
101 ),
102 write,
103 { fetched: true },
104 );
105 if (!installed.ok)
106 return {
107 ok: false,
108 error: installed.error,
109 infrastructure: installed.infrastructure,
110 stage: "install",
111 };
112 }
113 return undefined;
114 }
src/checks/tier_manifest.ts added +106
@@ -0,0 +1,106 @@
1
2 import { outsideAgentWaits } from "../origins/guest.ts";
3 import { ASKS_NEXT, asksLine, heldPull } from "./held.ts";
4 import { OUTSIDE_CHECK_LINE, OUTSIDE_NO_GATE, OUTSIDE_UNCONTAINED } from "./outside.ts";
5 import { resolveOid } from "../git/merge.ts";
6 import { readFileAt } from "../apps/checkout.ts";
7 import { parseCheck } from "../apps/manifest.ts";
8 import { cellsAvailable } from "../apps/contain.ts";
9 import type { LaneState } from "./lane_state.ts";
10 import type { Execution } from "./local_runner.ts";
11 import type { CheckManifest } from "../apps/manifest.ts";
12
13 /** The manifest a check runs by, and whether an outside contributor's code is being run. */
14 export type TierManifest =
15 | { readonly ok: true; readonly manifest: CheckManifest; readonly outside: boolean }
16 | { readonly ok: false; readonly error: string };
17
18 /**
19 * The manifest this check runs by. A pull does not choose its own gate: the commands and the
20 * timeout come from the base branch (`checks/held.ts`).
21 */
22 export async function resolveManifest(lane: LaneState, x: Execution, source: string): Promise<TierManifest> {
23 const { app } = lane;
24 const { bare, write } = x;
25 const [forgeJson, packageJson] = await Promise.all([
26 readFileAt(bare, source, "forge.json"),
27 readFileAt(bare, source, "package.json"),
28 ]);
29 const declared = parseCheck(forgeJson, packageJson);
30 if (!declared.ok)
31 return {
32 ok: false,
33 error:
34 declared.error.kind === "invalid"
35 ? declared.error.message
36 : "this commit declares no check",
37 };
38 if (declared.value === undefined)
39 return { ok: false, error: "this commit declares no check" };
40 // A pull does not choose its own gate: the commands and the timeout come
41 // from the base branch's `forge.json` where the pull left it, and so do
42 // the hosts its steps may reach and the limits its cell is given
43 // (`checks/held.ts`). The branch's own supplies only its install step.
44 let manifest = declared.value;
45 let baseTip: string | undefined;
46 /**
47 * An outside agent's pull nobody has approved: its code is checked, inside a cell or not at
48 * all.
49 */
50 let outside = false;
51 if (x.pull !== undefined) {
52 baseTip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`);
53 if (baseTip === undefined)
54 return { ok: false, error: "the pull's base branch could not be found" };
55 const [baseJson, basePackage] = await Promise.all([
56 readFileAt(bare, baseTip, "forge.json"),
57 readFileAt(bare, baseTip, "package.json"),
58 ]);
59 const gate = parseCheck(baseJson, basePackage);
60 if (!gate.ok)
61 return {
62 ok: false,
63 error:
64 gate.error.kind === "invalid"
65 ? `${x.pull.baseBranch}: ${gate.error.message}`
66 : "the base branch declares no check",
67 };
68 const held = heldPull(manifest, baseJson, "check");
69 manifest = held.manifest;
70 if (gate.value !== undefined) {
71 manifest = {
72 ...manifest,
73 run: gate.value.run,
74 fast: gate.value.fast,
75 full: gate.value.full,
76 timeout: gate.value.timeout,
77 };
78 write(
79 `tier ${x.tier} from ${x.pull.baseBranch}@${baseTip.slice(0, 7)}`,
80 );
81 // An outside agent's pull nobody has approved is checked at once, with nobody's
82 // permission: in a cell, held to that list whatever the forge's egress setting, until a
83 // writer approves the pull (`origins/guest.ts`).
84 if (outsideAgentWaits(app.db, x.pull)) {
85 // Uncontained, a check is a plain process beside the forge's own data: a writer's code
86 // may be; an outside contributor's waits for a writer here, and says how to go on.
87 if (!(await cellsAvailable()).ok) return { ok: false, error: OUTSIDE_UNCONTAINED };
88 outside = true;
89 write(OUTSIDE_CHECK_LINE(x.pull.baseBranch));
90 }
91 } else if (outsideAgentWaits(app.db, x.pull)) {
92 // No gate on the base branch to hold it to: an outside contributor's pull does not bring
93 // its own.
94 return { ok: false, error: OUTSIDE_NO_GATE(x.pull.baseBranch) };
95 }
96 // What it asked for and does not have yet, said where a refused request is looked for.
97 if (held.asks !== undefined) write(`${asksLine(held.asks, x.pull.baseBranch)} ${ASKS_NEXT}`);
98 }
99 if (x.commandsFrom !== undefined) {
100 const from = parseCheck(await readFileAt(bare, x.commandsFrom, "forge.json"), undefined);
101 if (from.ok && from.value !== undefined) manifest = { ...manifest, verify: from.value.verify, timeout: from.value.timeout };
102 }
103 if (x.tier === "verify" && manifest.verify === null)
104 return { ok: false, error: "this commit declares no check.verify" };
105 return { ok: true, manifest, outside };
106 }
src/checks/tier_runner.ts added +112
@@ -0,0 +1,112 @@
1
2 import type { Repo } from "../core/repos.ts";
3 import { noteRunProgress } from "../core/runs.ts";
4 import { runGit } from "../git/env.ts";
5 import { recordProven } from "./proven.ts";
6 import { headDepth } from "./history.ts";
7 import { clearedIn } from "./base_red.ts";
8 import { recordCrashFlakes } from "./flake.ts";
9 import { IMPACT_FILE } from "./impact-store.ts";
10 import { KEPT_FILE } from "./kept-store.ts";
11 import { egressSetting } from "../core/capacity.ts";
12 import { STEP_HOSTS } from "../apps/contain.ts";
13 import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts";
14 import type { CheckOutcome } from "./runner.ts";
15 import type { LaneState } from "./lane_state.ts";
16 import type { Execution } from "./local_runner.ts";
17 import type { CheckManifest } from "../apps/manifest.ts";
18
19 /** What a check handed to a runner is made of. */
20 export interface RunnerCheck {
21 readonly run: { readonly id: string; readonly commitOid: string };
22 readonly repo: Repo;
23 readonly bare: string;
24 readonly manifest: CheckManifest;
25 readonly command: string;
26 readonly base: string | undefined;
27 readonly builtOnBase: string | undefined;
28 readonly learned: string;
29 readonly tree: Awaited<ReturnType<typeof runGit>>;
30 readonly key: string | undefined;
31 readonly outside: boolean;
32 readonly write: (line: string) => void;
33 readonly signal: AbortSignal;
34 }
35
36 /**
37 * Give the check to a runner that joined, if one takes it (`runner_step.ts`, #13652). Answers its
38 * verdict, or nothing when it was not taken or answered with something that is not the check's own
39 * verdict: it then runs here as it always has.
40 */
41 export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck): Promise<CheckOutcome | undefined> {
42 const { app, options } = lane;
43 const { run, repo, bare, manifest, command, base, builtOnBase, learned, tree, key, outside, write, signal } = c;
44 // A runner that joined takes the steps (`runner_step.ts`, #13652): only a check that hands
45 // back nothing but its log and its verdict, and only while a runner is connected. Not taken,
46 // or an answer that is not the check's own verdict, and it runs here as it always has.
47 if (lane.placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) {
48 const lines: string[] = [];
49 const sent = await runRunnerStep(app, {
50 runId: x.runId,
51 repoId: repo.id,
52 dir: bare,
53 job: {
54 run: x.runId,
55 repo: `${repo.ownerName}/${repo.name}`,
56 commit: run.commitOid,
57 ...(base === undefined ? {} : { base }),
58 depth: base === undefined ? 1 : await headDepth(bare, base, run.commitOid),
59 steps: [
60 ...(manifest.install === null ? [] : [{ name: "install" as const, command: manifest.install }]),
61 { name: "check" as const, command },
62 ],
63 // Only the check's own variables cross, never the forge's; the runner adds its cell's.
64 env: {
65 CI: "1",
66 FORGE_URL: options.forgeUrl,
67 FORGE_CHECK: run.id,
68 FORGE_COMMIT: x.commitOid,
69 FORGE_CHECK_IMPACT: IMPACT_FILE,
70 FORGE_CHECK_KEPT: KEPT_FILE,
71 ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }),
72 FORGE_CELLS: "off",
73 },
74 limits: {
75 ...(manifest.limits.cpus === undefined ? {} : { cpus: manifest.limits.cpus }),
76 ...(manifest.limits.memoryMiB === undefined ? {} : { memoryMiB: manifest.limits.memoryMiB }),
77 ...(manifest.limits.diskMiB === undefined ? {} : { diskMiB: manifest.limits.diskMiB }),
78 },
79 timeoutS: manifest.timeout,
80 egress: [...STEP_HOSTS, ...manifest.egress],
81 egressMode: outside ? "enforce" : egressSetting(app.db),
82 // An outside contributor's code runs in a cell or not at all, there as here.
83 cell: outside ? "required" : "preferred",
84 cache: { repo: repo.id, tree: tree.ok ? tree.value.trim() : "", base: builtOnBase ?? "" },
85 },
86 log: write,
87 output: lines,
88 signal,
89 waitMs: lane.placement.waitMs,
90 });
91 const verdict = sent.taken ? runnerVerdict(sent, lines, signal.aborted) : sent.why;
92 if (typeof verdict === "string") write(`placement: runner; ${verdict}; running here`);
93 else {
94 if (sent.taken && sent.runner !== "")
95 noteRunProgress(app.db, x.runId, {
96 placement: "runner",
97 node: sent.runner,
98 sandbox: { enforced: sent.contained, note: sent.note },
99 });
100 try {
101 recordCrashFlakes(app.db, app.bus, repo, lines, { runId: run.id, commit: run.commitOid, pull: true }, verdict.ok);
102 } catch (error) {
103 write(`crash flakes not recorded: ${String(error)}`);
104 }
105 if (!verdict.ok) return verdict;
106 if (key !== undefined) recordProven(app.db, key, x.id);
107 const cleared = clearedIn(lines);
108 return { ok: true, ...(cleared === undefined ? {} : { cleared }) };
109 }
110 }
111 return undefined;
112 }
src/checks/tier_source.ts added +66
@@ -0,0 +1,66 @@
1
2 import { setRunBase, setRunCombined } from "../core/runs.ts";
3 import { resolveOid } from "../git/merge.ts";
4 import { mergeForCheck } from "./merged.ts";
5 import { readCombined } from "../git/combined.ts";
6 import type { LaneState } from "./lane_state.ts";
7 import type { Execution } from "./local_runner.ts";
8
9 /** The commit a tier checks, and what it builds on. */
10 export type TierSource =
11 | {
12 readonly ok: true;
13 readonly source: string;
14 readonly tip: string | undefined;
15 readonly builtOnBase: string | undefined;
16 readonly mergeLine: string;
17 }
18 | { readonly ok: false; readonly error: string };
19
20 /**
21 * The commit this tier checks. A pull's fast check is of the pull as it would land, and a kept
22 * fork's is of its combined commit; any other check is of the commit asked.
23 */
24 export async function resolveSource(lane: LaneState, x: Execution): Promise<TierSource> {
25 const { app } = lane;
26 const { bare } = x;
27 // A pull's fast check is of the pull as it would land: its head merged
28 // into the base's tip. The gate comes from that tree, so a pull forked
29 // before the base learned something is judged with it.
30 let source = x.commitOid;
31 let tip: string | undefined;
32 // The commit the checked tree builds on: the base tip for a merge made
33 // now, the base a combined commit names for a kept fork's pull. The fast
34 // tier diffs against it, so a combined built before main moved is not
35 // charged with main's later changes.
36 let builtOnBase: string | undefined;
37 let mergeLine = "";
38 if (x.tier === "fast" && x.pull !== undefined) {
39 tip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`);
40 if (tip === undefined)
41 return { ok: false, error: `the base ${x.pull.baseBranch} could not be found` };
42 // A kept fork's pull is checked on its combined commit (#9932): the
43 // tree that lands, built on the base tip it names (`combined/N`).
44 const combined = x.pull.crossRepo
45 ? await readCombined(bare, x.pull.number, x.commitOid)
46 : undefined;
47 if (combined !== undefined && combined.oid !== x.commitOid) {
48 source = combined.oid;
49 const builtOn = combined.baseOid === "" ? tip : combined.baseOid;
50 if (x.runId !== undefined) {
51 setRunBase(app.db, x.runId, builtOn);
52 setRunCombined(app.db, x.runId, combined.oid);
53 }
54 builtOnBase = builtOn;
55 mergeLine = `checked as combined with ${x.pull.baseBranch} at ${builtOn.slice(0, 7)} (${combined.oid.slice(0, 7)})`;
56 } else {
57 const merged = await mergeForCheck(bare, x.pull.baseBranch, tip, x.commitOid);
58 if (!merged.ok) return { ok: false, error: merged.error };
59 source = merged.commit;
60 if (x.runId !== undefined) setRunBase(app.db, x.runId, tip);
61 builtOnBase = tip;
62 mergeLine = merged.line;
63 }
64 }
65 return { ok: true, source, tip, builtOnBase, mergeLine };
66 }
src/checks/verify_lane.ts added +112
@@ -0,0 +1,112 @@
1
2 import { getRepoById } from "../core/repos.ts";
3 import { checksForHead, listActive, type Run } from "../core/runs.ts";
4 import { bisectOrigin, inVerifyLane, VERIFY_ENV, VERIFY_EVERY_MS, VERIFY_REASON, verifyDue } from "./verify.ts";
5 import { createLog } from "../kernel/log.ts";
6 import type { LaneState } from "./lane_state.ts";
7 import { start } from "./start.ts";
8
9 const log = createLog("checks");
10
11 /**
12 * The verification lane (#7786): one run at a time, forge-wide, beside
13 * the check slots rather than in one, so no landing and no pull's check
14 * ever waits for it. A bisection step goes before a new verification.
15 */
16 export function startVerify(lane: LaneState): void {
17 const { app } = lane;
18 if (lane.verifying !== undefined) return;
19 const queued = listActive(app.db, "check").filter(
20 (run) => run.state === "queued" && inVerifyLane(run.env) && !lane.inFlight.has(run.id),
21 );
22 const next =
23 queued.find((run) => bisectOrigin(run.reason) !== undefined) ?? queued[0];
24 if (next === undefined) return;
25 const repo = getRepoById(app.db, next.repoId);
26 if (repo !== undefined) start(lane, next, repo);
27 }
28
29 /** Every verification due now: main moved since the last, and that one is old enough. */
30 export async function askVerifications(lane: LaneState, now = Date.now()): Promise<Run[]> {
31 const { app, options } = lane;
32 const everyMs = options.verifyEveryMs ?? VERIFY_EVERY_MS;
33 const since = new Date(now - 24 * 3_600_000).toISOString();
34 // The newest default-branch check of each repository that had one today.
35 const tips = app.db
36 .query<{ repo_id: string; commit_oid: string }, [string]>(
37 `SELECT r.repo_id, r.commit_oid FROM run r JOIN repo ON repo.id = r.repo_id
38 WHERE r.id IN (SELECT MAX(x.id) FROM run x JOIN repo y ON y.id = x.repo_id
39 WHERE x.kind = 'check' AND x.issue_id IS NULL
40 AND x.env = y.default_branch AND x.created_at >= ?1
41 GROUP BY x.repo_id)`,
42 )
43 .all(since);
44 const asked: Run[] = [];
45 for (const tip of tips) {
46 const last = app.db
47 .query<{ commit_oid: string; created_at: string }, [string]>(
48 `SELECT commit_oid, created_at FROM run
49 WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}'
50 ORDER BY id DESC LIMIT 1`,
51 )
52 .get(tip.repo_id);
53 const active = listActive(app.db, "check").some(
54 (run) => run.repoId === tip.repo_id && inVerifyLane(run.env),
55 );
56 const due = verifyDue({
57 tip: tip.commit_oid,
58 last: last === null ? undefined : { commitOid: last.commit_oid, createdAt: last.created_at },
59 active,
60 now,
61 everyMs,
62 });
63 if (!due) continue;
64 const repo = getRepoById(app.db, tip.repo_id);
65 if (repo === undefined) continue;
66 const run = await lane.request(repo, VERIFY_ENV, tip.commit_oid, VERIFY_REASON);
67 if (run !== undefined) asked.push(run);
68 }
69 return asked;
70 }
71
72 /**
73 * An open pull whose head was never checked — the event that should have
74 * asked landed while a forge was on its way out, or before this forge
75 * existed — is asked for on boot. A pull that stays unchecked would never
76 * land on green, however green it is.
77 */
78 export async function catchUp(lane: LaneState): Promise<void> {
79 const { app } = lane;
80 const heads = app.db
81 .query<
82 { repo_id: string; issue_id: string; number: number; head_oid: string },
83 []
84 >(
85 `SELECT issue.repo_id, issue.id AS issue_id, issue.number, pull.head_oid
86 FROM pull JOIN issue ON issue.id = pull.issue_id
87 WHERE issue.state = 'open' AND pull.merged_at IS NULL
88 ORDER BY issue.created_at DESC LIMIT 200`,
89 )
90 .all();
91 for (const head of heads) {
92 if (head.head_oid === "") continue;
93 if (checksForHead(app.db, head.issue_id, head.head_oid).length > 0)
94 continue;
95 const repo = getRepoById(app.db, head.repo_id);
96 if (!repo) continue;
97 try {
98 await lane.request(
99 repo,
100 `pull-${head.number}`,
101 head.head_oid,
102 "boot",
103 head.issue_id,
104 );
105 } catch (error) {
106 log.error("check could not be requested on boot", {
107 pull: head.number,
108 error: String(error),
109 });
110 }
111 }
112 }
src/kernel/destroy_reach.ts +2 −2
@@ -68,8 +68,8 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
6868 "src/git/seed_history.ts:prune",
6969 ]),
7070 ...unitsOn("scratch under the checks root, per run or commit (checkouts, caches, locks, the install cache): never a forge's or an app's directory", [
71 "src/checks/service.ts:startChecks",
72 "src/checks/service.ts:removeRunDir",
71 "src/checks/run_tier.ts:runTier",
72 "src/checks/sweep.ts:removeRunDir",
7373 "src/checks/cache.ts:restoreCache",
7474 "src/checks/cache.ts:keepCache",
7575 "src/checks/impact-store.ts:seedImpact",
test/checks-service-split.test.ts added +59
@@ -0,0 +1,59 @@
1 /**
2 * `src/checks/service.ts` was one 2,240-line closure; it is cut along its seams (#14018). What
3 * the cut keeps: every name it was imported by, one module holding the handle registry behind
4 * `checksFor`, and no module over 600 lines.
5 */
6
7 import { describe, expect, test } from "bun:test";
8 import { readdirSync, readFileSync } from "node:fs";
9 import { join } from "node:path";
10 import type { App } from "../src/app.ts";
11 import * as service from "../src/checks/service.ts";
12 import type { ChecksHandle } from "../src/checks/service.ts";
13
14 const dir = join(import.meta.dir, "..", "src", "checks");
15 const modules = readdirSync(dir).filter((name) => name.endsWith(".ts"));
16
17 describe("the checks service is split", () => {
18 test("the names service.ts was imported by are still exported from it", () => {
19 for (const name of [
20 "startChecks",
21 "checksFor",
22 "provideChecksForTest",
23 "pickNext",
24 "holdsLastSlot",
25 "requeueAbandonedChecks",
26 "removeRunDir",
27 "sweepSoon",
28 "sweepCheckouts",
29 "reclaimIfFull",
30 "makeLocalRunner",
31 "PULL_RESERVE",
32 "LAND_SETTLE_MS",
33 "STARVATION_MS",
34 ])
35 expect(name in service).toBe(true);
36 });
37
38 test("the handle registry behind checksFor lives in exactly one module", () => {
39 const holders = modules.filter((name) => /new WeakMap\s*<\s*App\s*,\s*ChecksHandle/.test(readFileSync(join(dir, name), "utf8")));
40 expect(holders).toEqual(["service.ts"]);
41 });
42
43 test("checksFor answers the handle a test provided, per app", () => {
44 const one = {} as App;
45 const other = {} as App;
46 const handle = { spareSlots: () => 3 } as ChecksHandle;
47 expect(service.checksFor(one)).toBeUndefined();
48 service.provideChecksForTest(one, handle);
49 expect(service.checksFor(one)).toBe(handle);
50 expect(service.checksFor(other)).toBeUndefined();
51 });
52
53 test("no module of the lane is over 600 lines", () => {
54 const long = modules
55 .map((name) => [name, readFileSync(join(dir, name), "utf8").split("\n").length] as const)
56 .filter(([name, lines]) => lines > 600 && ["service.ts", "run_tier.ts", "scheduler.ts", "event_handler.ts", "execute.ts", "lane_state.ts"].includes(name));
57 expect(long).toEqual([]);
58 });
59 });
test/timers-named.test.ts +1 −1
@@ -134,7 +134,7 @@ const ALLOWED: readonly Allowed[] = [
134134 ["src/looks/thumb.ts", "() => reject("],
135135 ["src/looks/capture.ts", "() => reject("],
136136 ["src/actions/search.ts", "() => resolve(\"timeout\")"],
137 ["src/checks/service.ts", "() => { overran = true; budget.abort(); }"],
137 ["src/checks/execute.ts", "() => { overran = true; budget.abort(); }"],
138138 ["src/kernel/mail.ts", "() => finish(false)"],
139139 ["src/apps/domains.ts", "() => reject("],
140140 ["src/apps/preview_spec.ts", "() => { stop.abort(); reject("],

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs