Public: anyone can read this; the crew can change it.
src/checks/service.ts is split into small modules named for one concept, with no behaviour change #14019
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14018-src-checks-service-ts-is-split-into-smal → main 1 commit fork 1 thread · 1 unresolved
24 files
changed +2543
−2128
AGENTS.md +3 −1
Diff not shown — a line 2,989 characters wide (limit 2,000). +3 −1
View filesrc/checks/abandoned.ts added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | ||
| 2 | import type { App } from "../app.ts"; | |
| 3 | import { announceRun } from "../actions/runs.ts"; | |
| 4 | import { REQUEUED_SUMMARY, sweepAbandonedChecks } from "../core/runs.ts"; | |
| 5 | import { appendRunLog } from "../runs/log.ts"; | |
| 6 | import { sweepSoon } from "./sweep.ts"; | |
| 7 | ||
| 8 | /** | |
| 9 | * Settle the checks a dead forge left running, and say so on the bus: a | |
| 10 | * check whose commit still heads an open pull is queued again for whichever | |
| 11 | * checks service is up to run, so the change keeps reading *working*. | |
| 12 | * Called on boot and on the way by whichever forge is alive. A cancelled | |
| 13 | * check is not announced, as a superseded one is not: `run.finished` reads | |
| 14 | * as a verdict on the change, and the forge dying is none. | |
| 15 | */ | |
| 16 | export function requeueAbandonedChecks(app: App, staleMs: number): number { | |
| 17 | const swept = sweepAbandonedChecks(app.db, staleMs); | |
| 18 | for (const { cancelled, again } of swept.requeued) { | |
| 19 | appendRunLog(app.config.dataDir, cancelled.id, REQUEUED_SUMMARY); | |
| 20 | announceRun(app, again, again.actorId ?? undefined); | |
| 21 | } | |
| 22 | for (const run of swept.cancelled) | |
| 23 | appendRunLog(app.config.dataDir, run.id, run.summary); | |
| 24 | for (const run of swept.failed) { | |
| 25 | appendRunLog(app.config.dataDir, run.id, run.summary); | |
| 26 | announceRun(app, run); | |
| 27 | } | |
| 28 | // On the same timer: what a dead or lapsed run left on the disk. Removing | |
| 29 | // a checkout is thousands of unlinks: off the loop, never in front of /healthz (#7688). | |
| 30 | sweepSoon(app); | |
| 31 | return swept.requeued.length + swept.failed.length + swept.cancelled.length; | |
| 32 | } |
src/checks/event_handler.ts added +189
| @@ -0,0 +1,189 @@ | ||
| 1 | ||
| 2 | import { pullDir } from "../core/pulls.ts"; | |
| 3 | import type { App } from "../app.ts"; | |
| 4 | import { announceRun } from "../actions/runs.ts"; | |
| 5 | import type { ForgeEvent } from "../core/events.ts"; | |
| 6 | import { getPull, listPulls, type Pull } from "../core/pulls.ts"; | |
| 7 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 8 | import { getRun, cancelRunsForPull, checksForHead } from "../core/runs.ts"; | |
| 9 | import { resolveOid } from "../git/merge.ts"; | |
| 10 | import { checkOnApproval } from "./approval.ts"; | |
| 11 | import { settleLandedRuns } from "../crew/interrupted.ts"; | |
| 12 | import type { LaneState } from "./lane_state.ts"; | |
| 13 | import { recheckBase } from "./recheck_base.ts"; | |
| 14 | ||
| 15 | export async function onEvent(lane: LaneState, event: ForgeEvent): Promise<void> { | |
| 16 | const { app } = lane; | |
| 17 | if (lane.stopped || event.repoId === null) return; | |
| 18 | const repo = getRepoById(app.db, event.repoId); | |
| 19 | if (!repo) return; | |
| 20 | const actor = event.actorId ?? undefined; | |
| 21 | switch (event.kind) { | |
| 22 | case "push": { | |
| 23 | // A fork that silently followed holds its parent's commit, byte for | |
| 24 | // byte: checking the copy proves nothing the parent's own check did | |
| 25 | // not, once per fork, on every landed pull. A merge of the parent | |
| 26 | // into the fork's own work is a new commit, and is checked. | |
| 27 | if ( | |
| 28 | typeof event.payload["followed"] === "string" && | |
| 29 | event.payload["merged"] !== true | |
| 30 | ) | |
| 31 | return; | |
| 32 | const updates = event.payload["updates"]; | |
| 33 | if (!Array.isArray(updates)) return; | |
| 34 | for (const update of updates as { ref?: unknown; newOid?: unknown }[]) { | |
| 35 | if ( | |
| 36 | typeof update.ref !== "string" || | |
| 37 | typeof update.newOid !== "string" | |
| 38 | ) | |
| 39 | continue; | |
| 40 | if ( | |
| 41 | !update.ref.startsWith("refs/heads/") || | |
| 42 | /^0+$/.test(update.newOid) | |
| 43 | ) | |
| 44 | continue; | |
| 45 | const branch = update.ref.slice("refs/heads/".length); | |
| 46 | if (branch === repo.defaultBranch) | |
| 47 | await lane.request( | |
| 48 | repo, | |
| 49 | branch, | |
| 50 | update.newOid, | |
| 51 | "push", | |
| 52 | undefined, | |
| 53 | actor, | |
| 54 | ); | |
| 55 | await recheckBase(lane, repo, branch, update.newOid); | |
| 56 | for (const pull of listPulls(app.db, repo.id, { | |
| 57 | sealed: true, | |
| 58 | state: "open", | |
| 59 | limit: 200, | |
| 60 | })) { | |
| 61 | if (pull.crossRepo || pull.headBranch !== branch) continue; | |
| 62 | await lane.request( | |
| 63 | repo, | |
| 64 | `pull-${pull.number}`, | |
| 65 | update.newOid, | |
| 66 | "push", | |
| 67 | pull.id, | |
| 68 | actor, | |
| 69 | ); | |
| 70 | } | |
| 71 | } | |
| 72 | return; | |
| 73 | } | |
| 74 | case "pull.opened": | |
| 75 | case "pull.reopened": { | |
| 76 | const number = event.payload["number"]; | |
| 77 | if (typeof number !== "number") return; | |
| 78 | const pull = getPull(app.db, repo.id, number); | |
| 79 | if (!pull) return; | |
| 80 | const head = await headOf(app, repo, pull); | |
| 81 | if (head !== undefined) | |
| 82 | await lane.request( | |
| 83 | repo, | |
| 84 | `pull-${pull.number}`, | |
| 85 | head, | |
| 86 | "pull", | |
| 87 | pull.id, | |
| 88 | actor, | |
| 89 | ); | |
| 90 | return; | |
| 91 | } | |
| 92 | case "pull.pushed": { | |
| 93 | const number = event.payload["number"]; | |
| 94 | const headOid = event.payload["headOid"]; | |
| 95 | if (typeof number !== "number" || typeof headOid !== "string") return; | |
| 96 | const pull = getPull(app.db, repo.id, number); | |
| 97 | if (!pull) return; | |
| 98 | await lane.request( | |
| 99 | repo, | |
| 100 | `pull-${pull.number}`, | |
| 101 | headOid, | |
| 102 | "push", | |
| 103 | pull.id, | |
| 104 | actor, | |
| 105 | ); | |
| 106 | return; | |
| 107 | } | |
| 108 | // An approval is what lets a reader's pull run: check its head now. | |
| 109 | case "pull.run_granted": | |
| 110 | case "pull.reviewed": { | |
| 111 | const number = event.payload["number"]; | |
| 112 | if (typeof number !== "number") return; | |
| 113 | if ( | |
| 114 | event.kind === "pull.reviewed" && | |
| 115 | event.payload["state"] !== "approved" | |
| 116 | ) | |
| 117 | return; | |
| 118 | const pull = getPull(app.db, repo.id, number); | |
| 119 | if (!pull || pull.state !== "open") return; | |
| 120 | const head = await headOf(app, repo, pull); | |
| 121 | if (head === undefined) return; | |
| 122 | // A head already green is not checked again for nothing (#4391). | |
| 123 | const env = `pull-${pull.number}`; | |
| 124 | const dir = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 125 | const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`); | |
| 126 | // The merge queue gates only an autonomous repository's lands (`crew/autonomy.ts` admit). | |
| 127 | const landGated = repo.autonomy === "autonomous" && repo.autonomyBy !== null; | |
| 128 | if (!checkOnApproval(checksForHead(app.db, pull.id, head), env, tip, landGated)) | |
| 129 | return; | |
| 130 | await lane.request(repo, env, head, "approved", pull.id, actor); | |
| 131 | return; | |
| 132 | } | |
| 133 | // Landed or closed: nothing left to check, so nothing waits in the lane. | |
| 134 | case "pull.merged": | |
| 135 | case "pull.closed": { | |
| 136 | const number = event.payload["number"]; | |
| 137 | if (typeof number !== "number") return; | |
| 138 | const pull = getPull(app.db, repo.id, number); | |
| 139 | if (!pull) return; | |
| 140 | const cancelled = cancelRunsForPull( | |
| 141 | app.db, | |
| 142 | pull.id, | |
| 143 | event.kind === "pull.merged" ? "pull landed" : "pull closed", | |
| 144 | ); | |
| 145 | for (const run of cancelled) { | |
| 146 | lane.inFlight.get(run.id)?.abort.abort(); | |
| 147 | announceRun(app, run); | |
| 148 | } | |
| 149 | // A builder run still in flight when its pull lands reads as done now (#12049). | |
| 150 | if (event.kind === "pull.merged") | |
| 151 | for (const id of settleLandedRuns(app.db, { issueId: pull.id, headBranch: pull.headBranch, repoId: repo.id, number })) { | |
| 152 | const done = getRun(app.db, id); | |
| 153 | if (done) announceRun(app, done); | |
| 154 | } | |
| 155 | // A land moves the default branch without a push event: check its | |
| 156 | // new head, or a red main stays red until someone pushes to it. The | |
| 157 | // check settles first: a candidate's next step supersedes it. | |
| 158 | const mergeOid = event.payload["mergeOid"]; | |
| 159 | if ( | |
| 160 | event.kind === "pull.merged" && | |
| 161 | pull.baseBranch === repo.defaultBranch && | |
| 162 | typeof mergeOid === "string" && | |
| 163 | /^[0-9a-f]{40}$/.test(mergeOid) | |
| 164 | ) | |
| 165 | await lane.request( | |
| 166 | repo, | |
| 167 | repo.defaultBranch, | |
| 168 | mergeOid, | |
| 169 | "push", | |
| 170 | undefined, | |
| 171 | actor, | |
| 172 | lane.settle > 0 ? new Date(Date.now() + lane.settle).toISOString() : undefined, | |
| 173 | ); | |
| 174 | if (event.kind === "pull.merged" && typeof mergeOid === "string" && /^[0-9a-f]{40}$/.test(mergeOid)) | |
| 175 | await recheckBase(lane, repo, pull.baseBranch, mergeOid); | |
| 176 | return; | |
| 177 | } | |
| 178 | default: | |
| 179 | return; | |
| 180 | } | |
| 181 | } | |
| 182 | ||
| 183 | async function headOf(app: App, repo: Repo, pull: Pull): Promise<string | undefined> { | |
| 184 | const dir = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 185 | const ref = pull.crossRepo | |
| 186 | ? `refs/pull/${pull.number}/head` | |
| 187 | : `refs/heads/${pull.headBranch}`; | |
| 188 | return (await resolveOid(dir, ref)) ?? (pull.headOid || undefined); | |
| 189 | } |
src/checks/execute.ts added +250
| @@ -0,0 +1,250 @@ | ||
| 1 | ||
| 2 | import { releaseOnGreen } from "../actions/holds.ts"; | |
| 3 | import { announceRun } from "../actions/runs.ts"; | |
| 4 | import { announceShard } from "./shard_event.ts"; | |
| 5 | import { issueMayRun } from "../core/pull_trust.ts"; | |
| 6 | import type { Repo } from "../core/repos.ts"; | |
| 7 | import { cancelRun, getRun, claimRun, finishRun, overranBefore, vanishedBefore, VANISHED_SUMMARY, infraBefore, INFRA_SUMMARY, CPU_STOP_SUMMARY, DISK_USED_SUMMARY, requeueCheck, chainCutOffs, CHECK_RELEASE_LIMIT, RELEASE_SUMMARY, type Run } from "../core/runs.ts"; | |
| 8 | import { withDisk } from "./disk_red.ts"; | |
| 9 | import { proposeRevert } from "./revert_red.ts"; | |
| 10 | import { afterVerify, closeOnGreen } from "./culprit.ts"; | |
| 11 | import { noteLookPages } from "./look_pages.ts"; | |
| 12 | import { reportGreen, reportRed } from "./report.ts"; | |
| 13 | import { bisectOrigin, inVerifyLane, VERIFY_ENV } from "./verify.ts"; | |
| 14 | import { getIssueById } from "../core/issues.ts"; | |
| 15 | import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts"; | |
| 16 | import { createLog } from "../kernel/log.ts"; | |
| 17 | import { holdLease } from "../runs/lease.ts"; | |
| 18 | import { appendRunLog } from "../runs/log.ts"; | |
| 19 | import { cancelledLine } from "../runs/cancelled.ts"; | |
| 20 | import { requeueCpuStop } from "./cpu_stop.ts"; | |
| 21 | import { isUsedRefusal } from "../apps/disk_guard.ts"; | |
| 22 | import { JOB_VANISHED } from "./job_step.ts"; | |
| 23 | import { describeJob, type CheckOutcome } from "./runner.ts"; | |
| 24 | import type { LaneState } from "./lane_state.ts"; | |
| 25 | ||
| 26 | const log = createLog("checks"); | |
| 27 | ||
| 28 | export async function execute( | |
| 29 | lane: LaneState, | |
| 30 | run: Run, | |
| 31 | repo: Repo, | |
| 32 | signal: AbortSignal, | |
| 33 | slot = true, | |
| 34 | ): Promise<void> { | |
| 35 | const { app, options } = lane; | |
| 36 | // The verification lane takes no slot of the gate (`startVerify`). | |
| 37 | const release = slot ? await lane.gate.acquire() : () => {}; | |
| 38 | try { | |
| 39 | // A slot that opens while this forge is stopping is not taken: the run | |
| 40 | // stays queued for the forge that carries on. | |
| 41 | if (lane.stopped) return; | |
| 42 | // Asked again as it starts: a check queued before a rule changed, or | |
| 43 | // handed back by a forge that stopped, runs only what may run now. | |
| 44 | if (!issueMayRun(app.db, repo, run.issueId, run.commitOid)) { | |
| 45 | cancelRun(app.db, run.id, "waiting for a writer to approve this head"); | |
| 46 | return; | |
| 47 | } | |
| 48 | const claimed = claimRun(app.db, run.id); | |
| 49 | if (!claimed) return; | |
| 50 | const lease = holdLease(app.db, run.id); | |
| 51 | announceRun(app, claimed); | |
| 52 | const write = (line: string): void => { | |
| 53 | appendRunLog(app.config.dataDir, run.id, line); | |
| 54 | announceShard(app, run, line); | |
| 55 | }; | |
| 56 | if (options.label !== undefined) | |
| 57 | write(`forge ${options.label} — checked in its parent's lane`); | |
| 58 | const started = Date.now(); | |
| 59 | // The budget: past it the check is aborted, whatever it is doing. | |
| 60 | const limit = checkLimitMs(run.env, { | |
| 61 | gate: options.budgetMs ?? BUDGETS.gate, | |
| 62 | verify: options.verifyBudgetMs ?? BUDGETS.verify, | |
| 63 | review: BUDGETS.review, | |
| 64 | }); | |
| 65 | const budget = new AbortController(); | |
| 66 | let overran = false; | |
| 67 | const forward = (): void => budget.abort(); | |
| 68 | signal.addEventListener("abort", forward, { once: true }); | |
| 69 | const clock = setTimeout(() => { | |
| 70 | overran = true; | |
| 71 | budget.abort(); | |
| 72 | }, limit); | |
| 73 | const outcome = await check(lane, claimed, repo, write, budget.signal).finally( | |
| 74 | () => { | |
| 75 | clearTimeout(clock); | |
| 76 | signal.removeEventListener("abort", forward); | |
| 77 | lease(); | |
| 78 | }, | |
| 79 | ); | |
| 80 | // Where a check cut off here is asked again. A verification is of the | |
| 81 | // default branch as it stands: it goes back at the branch's newest | |
| 82 | // checked commit, the one `askVerifications` would ask of. Left at the | |
| 83 | // commit it began on, each restart and each overrun began it there | |
| 84 | // again, no other was asked while it was active, and main went | |
| 85 | // unverified from 01:48Z to 03:37Z on 2026-11-14 (#12178). | |
| 86 | const againAt = (): string | undefined => | |
| 87 | run.env === VERIFY_ENV | |
| 88 | ? app.db | |
| 89 | .query<{ commit_oid: string }, [string, string]>( | |
| 90 | `SELECT commit_oid FROM run WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL | |
| 91 | AND env = ?2 ORDER BY id DESC LIMIT 1`, | |
| 92 | ) | |
| 93 | .get(repo.id, repo.defaultBranch)?.commit_oid || undefined | |
| 94 | : undefined; | |
| 95 | if (overran) { | |
| 96 | const summary = budgetSummary( | |
| 97 | "a full gate", | |
| 98 | Date.now() - started, | |
| 99 | limit, | |
| 100 | ); | |
| 101 | write(summary); | |
| 102 | if (!overranBefore(app.db, claimed)) { | |
| 103 | const moved = requeueCheck(app.db, run.id, undefined, summary, againAt()); | |
| 104 | if (moved) { | |
| 105 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 106 | return; | |
| 107 | } | |
| 108 | } | |
| 109 | // The second overrun: a failure, and the card the vitals carry. | |
| 110 | const failed = finishRun(app.db, run.id, { | |
| 111 | state: "failed", | |
| 112 | summary, | |
| 113 | }); | |
| 114 | if (failed) announceRun(app, failed); | |
| 115 | return; | |
| 116 | } | |
| 117 | // Ended from outside while it ran: its pull landed or closed, and | |
| 118 | // the row says so. That is not this check failing. | |
| 119 | // A person's cancel ends the log with who cancelled it (#11229). | |
| 120 | const row = getRun(app.db, run.id); | |
| 121 | if (!outcome.ok && row?.state === "cancelled") { | |
| 122 | write(cancelledLine(row) ?? `superseded: ${row.summary}`); | |
| 123 | return; | |
| 124 | } | |
| 125 | if (!outcome.ok && lane.stopped && signal.aborted) { | |
| 126 | // Cut off by this forge stopping, not by the check: the same check | |
| 127 | // goes back on the queue, and the forge that carries on runs it. | |
| 128 | // Past the release ceiling the shutdown is what this check hangs or | |
| 129 | // slows: it fails, saying so, instead of riding every release. | |
| 130 | if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) { | |
| 131 | const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`; | |
| 132 | write(summary); | |
| 133 | const failed = finishRun(app.db, run.id, { state: "failed", summary }); | |
| 134 | if (failed) announceRun(app, failed); | |
| 135 | return; | |
| 136 | } | |
| 137 | const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt()); | |
| 138 | if (!moved) return; | |
| 139 | write(RELEASE_SUMMARY); | |
| 140 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 141 | return; | |
| 142 | } | |
| 143 | // The cluster lost the Job (evicted, node gone): interrupted, not | |
| 144 | // failed, and queued once more as after a restart; the second loss fails. | |
| 145 | if (!outcome.ok && outcome.error === JOB_VANISHED && !vanishedBefore(app.db, claimed)) { | |
| 146 | const moved = requeueCheck(app.db, run.id, undefined, VANISHED_SUMMARY); | |
| 147 | if (moved) { | |
| 148 | write(VANISHED_SUMMARY); | |
| 149 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 150 | return; | |
| 151 | } | |
| 152 | } | |
| 153 | // The install failed twice on what the machine did, not the code: the | |
| 154 | // check is queued once more and nothing reads it as red (#5093). | |
| 155 | if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) { | |
| 156 | const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY); | |
| 157 | if (moved) { | |
| 158 | write(INFRA_SUMMARY); | |
| 159 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 160 | return; | |
| 161 | } | |
| 162 | } | |
| 163 | // The disk was 80% used when the step would have started (#11799): | |
| 164 | // the machine's wait. Queued again, not red; the pick holds it until there is room. | |
| 165 | if (!outcome.ok && isUsedRefusal(outcome.error)) { | |
| 166 | const moved = requeueCheck(app.db, run.id, undefined, DISK_USED_SUMMARY); | |
| 167 | if (moved) { | |
| 168 | write(DISK_USED_SUMMARY); | |
| 169 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 170 | return; | |
| 171 | } | |
| 172 | } | |
| 173 | // Stopped for CPU by the cell's watch: the machine's, not the pull's. | |
| 174 | // Queued once more on its own, and no builder hears check_failed for | |
| 175 | // it; a second stop in a row is reported as any red is (#11617). | |
| 176 | if (!outcome.ok && outcome.cpuStop !== undefined) { | |
| 177 | const moved = requeueCpuStop(app.db, claimed, outcome.cpuStop); | |
| 178 | if (moved) { | |
| 179 | write(CPU_STOP_SUMMARY); | |
| 180 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 181 | return; | |
| 182 | } | |
| 183 | } | |
| 184 | const seconds = Math.round((Date.now() - started) / 1000); | |
| 185 | const done = finishRun(app.db, run.id, { | |
| 186 | state: outcome.ok ? "ok" : "failed", | |
| 187 | summary: outcome.ok | |
| 188 | ? outcome.proven === undefined | |
| 189 | ? `check passed in ${seconds}s${outcome.cleared === undefined ? "" : `; ${outcome.cleared}`}` | |
| 190 | : `already proven green by ${outcome.proven}` | |
| 191 | : `check failed: ${outcome.error}`, | |
| 192 | ...(outcome.ok | |
| 193 | ? {} | |
| 194 | : { checkDetail: withDisk(outcome.detail, app.config.dataDir) }), | |
| 195 | }); | |
| 196 | write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`); | |
| 197 | // The pages the gate chose to picture are said on the pull (#13373), worked out by the | |
| 198 | // forge from the pull's commits, never read from the log the branch's code writes. | |
| 199 | if (done) void noteLookPages(app, repo, done).catch((error) => log.error("look pages note failed", { error: String(error) })); | |
| 200 | if (done) announceRun(app, done); | |
| 201 | if (done && outcome.ok) releaseOnGreen(app, done); | |
| 202 | if (!outcome.ok && run.issueId === null && run.env === repo.defaultBranch) | |
| 203 | proposeRevert(app, repo.id).catch((error) => | |
| 204 | log.error("revert proposal failed", { error: String(error) }), | |
| 205 | ); | |
| 206 | // A red verification after landing names its culprit (#7786). | |
| 207 | if (done && run.env === VERIFY_ENV) { | |
| 208 | reportRed(app, repo, done).forEach(write); | |
| 209 | // A file the run printed passing closes its open report issue (#9524). | |
| 210 | reportGreen(app, repo, done, outcome.passed ?? []).forEach(write); | |
| 211 | // A green that ran every file a culprit's issue names closes it (#11395). | |
| 212 | closeOnGreen(app, repo, done, outcome.passed ?? []).forEach(write); | |
| 213 | } | |
| 214 | if (done && inVerifyLane(run.env)) | |
| 215 | await afterVerify(app, repo, done, (r, env, oid, reason) => lane.request(r, env, oid, reason)) | |
| 216 | .then((lines) => lines.forEach(write)) | |
| 217 | .catch((error) => write(`culprit not named: ${String(error)}`)); | |
| 218 | } finally { | |
| 219 | release(); | |
| 220 | } | |
| 221 | } | |
| 222 | ||
| 223 | /** Describe a claimed check as data and hand it to the runner. */ | |
| 224 | async function check( | |
| 225 | lane: LaneState, | |
| 226 | run: Run, | |
| 227 | repo: Repo, | |
| 228 | write: (line: string) => void, | |
| 229 | signal: AbortSignal, | |
| 230 | ): Promise<CheckOutcome> { | |
| 231 | const { app } = lane; | |
| 232 | const issue = run.issueId === null ? undefined : getIssueById(app.db, run.issueId); | |
| 233 | const inLane = inVerifyLane(run.env); | |
| 234 | // A bisection step runs only the red's failed files, with the red's own command: an older | |
| 235 | // commit may not say how to verify (#7786). | |
| 236 | const origin = | |
| 237 | inLane && run.env !== VERIFY_ENV ? getRun(app.db, bisectOrigin(run.reason) ?? "") : undefined; | |
| 238 | const job = describeJob(run, { | |
| 239 | defaultBranch: repo.defaultBranch, | |
| 240 | verifyEnv: VERIFY_ENV, | |
| 241 | inVerifyLane: inLane, | |
| 242 | pulled: issue?.kind === "pull", | |
| 243 | origin: | |
| 244 | origin === undefined | |
| 245 | ? undefined | |
| 246 | : { files: origin.checkDetail?.files ?? [], commitOid: origin.commitOid }, | |
| 247 | }); | |
| 248 | if ("error" in job) return { ok: false, error: job.error }; | |
| 249 | return lane.runner.run(job, { write, signal }); | |
| 250 | } |
src/checks/gate_run.ts added +71
| @@ -0,0 +1,71 @@ | ||
| 1 | ||
| 2 | import { pullDir } from "../core/pulls.ts"; | |
| 3 | import type { Pull } from "../core/pulls.ts"; | |
| 4 | import type { Repo } from "../core/repos.ts"; | |
| 5 | import type { GateDone } from "./lane_state.ts"; | |
| 6 | import { beginGate, endGate } from "./gate_record.ts"; | |
| 7 | import { JOB_VANISHED } from "./job_step.ts"; | |
| 8 | import { gateRunId } from "./queue.ts"; | |
| 9 | import type { LaneState } from "./lane_state.ts"; | |
| 10 | import { runTier } from "./run_tier.ts"; | |
| 11 | ||
| 12 | /** A gate this forge would not start, for it is stopping. */ | |
| 13 | const INTERRUPTED = "this forge is stopping"; | |
| 14 | async function gateOnce( | |
| 15 | lane: LaneState, | |
| 16 | repo: Repo, | |
| 17 | pull: Pull, | |
| 18 | commitOid: string, | |
| 19 | controller: AbortController, | |
| 20 | signal: AbortSignal | undefined, | |
| 21 | ): Promise<GateDone> { | |
| 22 | const { app } = lane; | |
| 23 | const lines: string[] = []; | |
| 24 | const abandon = (): void => controller.abort(); | |
| 25 | signal?.addEventListener("abort", abandon, { once: true }); | |
| 26 | // The slot minutes it spends are the flow's gate minutes (#6426). | |
| 27 | const traced = beginGate(app.db, repo.id, pull.id, commitOid); | |
| 28 | try { | |
| 29 | const bare = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 30 | const done = await runTier(lane, { | |
| 31 | bare, | |
| 32 | repo, | |
| 33 | id: gateRunId(commitOid), | |
| 34 | commitOid, | |
| 35 | pull, | |
| 36 | tier: "full", | |
| 37 | write: (line) => lines.push(line), | |
| 38 | signal: controller.signal, | |
| 39 | }); | |
| 40 | if (done.ok) return { ok: true, error: "", lines }; | |
| 41 | const stage = done.stage ?? "setup"; | |
| 42 | // Cut off by this forge stopping, the Job lost under it, the check | |
| 43 | // killed by a signal from outside (the pod going, the OOM killer), or | |
| 44 | // an install the machine failed twice: the code was not judged, so | |
| 45 | // this is no red. A gate the drain lets finish is judged (#6611, #6722). | |
| 46 | if ( | |
| 47 | controller.signal.aborted || | |
| 48 | done.error === JOB_VANISHED || | |
| 49 | done.infrastructure === true || | |
| 50 | done.killed === true | |
| 51 | ) | |
| 52 | return { ok: false, error: done.error, lines, interrupted: true, stage }; | |
| 53 | return { ok: false, error: done.error, lines, stage }; | |
| 54 | } finally { | |
| 55 | endGate(app.db, traced); | |
| 56 | signal?.removeEventListener("abort", abandon); | |
| 57 | lane.gates.delete(controller); | |
| 58 | } | |
| 59 | } | |
| 60 | ||
| 61 | /** | |
| 62 | * A gate asked of a forge already stopping is not started: the forge that carries on asks it | |
| 63 | * again. | |
| 64 | */ | |
| 65 | export function gateRun(lane: LaneState, repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone> { | |
| 66 | if (lane.stopped) return Promise.resolve({ ok: false, error: INTERRUPTED, lines: [], interrupted: true }); | |
| 67 | const controller = new AbortController(); | |
| 68 | const done = gateOnce(lane, repo, pull, commitOid, controller, signal); | |
| 69 | lane.gates.set(controller, { commitOid, done }); | |
| 70 | return done; | |
| 71 | } |
src/checks/lane_state.ts added +218
| @@ -0,0 +1,218 @@ | ||
| 1 | ||
| 2 | import type { App } from "../app.ts"; | |
| 3 | import { holdClock } from "../core/one_core.ts"; | |
| 4 | import type { Pull } from "../core/pulls.ts"; | |
| 5 | import type { Repo } from "../core/repos.ts"; | |
| 6 | import type { Run } from "../core/runs.ts"; | |
| 7 | import { Gate } from "../git/env.ts"; | |
| 8 | import { runStep } from "../apps/process.ts"; | |
| 9 | import { capacityCount } from "../core/capacity.ts"; | |
| 10 | import type { Placement } from "./placement.ts"; | |
| 11 | import type { PodRoom } from "./pod_room.ts"; | |
| 12 | import type { GateStage } from "./gate_reason.ts"; | |
| 13 | import type { CheckOutcome, Runner } from "./runner.ts"; | |
| 14 | import type { JobOptions } from "../cluster/jobs.ts"; | |
| 15 | import type { Execution } from "./local_runner.ts"; | |
| 16 | ||
| 17 | export interface ChecksOptions { | |
| 18 | /** How many checks may run at once. @default 1 */ | |
| 19 | readonly concurrency?: number | undefined; | |
| 20 | /** How full this forge's memory is, 0 to 1: its cgroup's unless a test says otherwise. */ | |
| 21 | readonly pressure?: (() => number | undefined) | undefined; | |
| 22 | /** The forge's public URL, told to the check as `FORGE_URL`. */ | |
| 23 | readonly forgeUrl: string; | |
| 24 | /** The parent's gate, for a hosted child: its checks take their turn in the parent's lane. */ | |
| 25 | readonly gate?: Gate | undefined; | |
| 26 | /** Whose checks these are, written first in each run's log (a hosted child's name). */ | |
| 27 | readonly label?: string | undefined; | |
| 28 | /** How long a check may run before it is killed. @default BUDGETS.gate */ | |
| 29 | readonly budgetMs?: number | undefined; | |
| 30 | /** How long a verification of the default branch may run. @default BUDGETS.verify */ | |
| 31 | readonly verifyBudgetMs?: number | undefined; | |
| 32 | /** | |
| 33 | * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0, | |
| 34 | * nothing held; the forge says `RED_HOLD_MS`. | |
| 35 | */ | |
| 36 | readonly redHoldMs?: number | undefined; | |
| 37 | /** | |
| 38 | * How long main's check after a land waits before it may start, so the | |
| 39 | * steps of one candidate supersede each other while queued. @default 0, | |
| 40 | * none; the forge says `LAND_SETTLE_MS`. | |
| 41 | */ | |
| 42 | readonly landSettleMs?: number | undefined; | |
| 43 | /** | |
| 44 | * How long `stop` lets checks already running finish before it cuts them | |
| 45 | * off and queues them again (#1796). Nothing new starts meanwhile. | |
| 46 | * @default 0 | |
| 47 | */ | |
| 48 | readonly drainMs?: number | undefined; | |
| 49 | /** | |
| 50 | * Where a check's commands run (`checks/placement.ts`). @default from the | |
| 51 | * environment: `process`. | |
| 52 | */ | |
| 53 | readonly placement?: Placement | undefined; | |
| 54 | /** The kube client a `job` placement speaks through; tests stand one in. */ | |
| 55 | readonly job?: JobOptions | undefined; | |
| 56 | /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */ | |
| 57 | readonly installRetryMs?: number | undefined; | |
| 58 | /** | |
| 59 | * Runs the check step itself; tests stand one in for a stop the cell's watch makes | |
| 60 | * (#11617). @default runStep | |
| 61 | */ | |
| 62 | readonly checkStep?: typeof runStep | undefined; | |
| 63 | /** | |
| 64 | * Where the pod's memory.max is read and the forge's size | |
| 65 | * (`checks/pod_room.ts`); tests stand in a small pod. @default the real ones | |
| 66 | */ | |
| 67 | readonly podRoom?: Omit<PodRoom, "env"> | undefined; | |
| 68 | /** | |
| 69 | * How often the default branch is verified after landing at most | |
| 70 | * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds | |
| 71 | * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`. | |
| 72 | */ | |
| 73 | readonly verifyEveryMs?: number | undefined; | |
| 74 | /** Test seam: stands in for the tier the local runner runs, to see what reaches it. */ | |
| 75 | readonly runTier?: ((x: Execution) => Promise<CheckOutcome>) | undefined; | |
| 76 | /** Test seam: how often the queue is looked at. @default TICK_MS */ | |
| 77 | readonly tickMs?: number | undefined; | |
| 78 | /** The forge's CPU count; a 1-CPU forge holds pull checks for a crew run. @default forgeCpus */ | |
| 79 | readonly cpus?: (() => number) | undefined; | |
| 80 | /** Crew runs working now. @default the running agent runs */ | |
| 81 | readonly activeCrewRuns?: (() => number) | undefined; | |
| 82 | /** Test seams: how long a held pull check waits at most, and the clock it is judged by. */ | |
| 83 | readonly holdMaxMs?: number | undefined; | |
| 84 | readonly holdNow?: (() => number) | undefined; | |
| 85 | } | |
| 86 | /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */ | |
| 87 | export interface GateDone { | |
| 88 | ok: boolean; | |
| 89 | error: string; | |
| 90 | lines: string[]; | |
| 91 | interrupted?: boolean; | |
| 92 | stage?: GateStage; | |
| 93 | } | |
| 94 | ||
| 95 | export interface ChecksHandle { | |
| 96 | /** | |
| 97 | * Ask for a check of one commit; the answer is the run, or nothing when the commit declares no | |
| 98 | * check. | |
| 99 | */ | |
| 100 | request( | |
| 101 | repo: Repo, | |
| 102 | env: string, | |
| 103 | commitOid: string, | |
| 104 | reason: string, | |
| 105 | issueId?: string, | |
| 106 | actorId?: string, | |
| 107 | ): Promise<Run | undefined>; | |
| 108 | /** | |
| 109 | * The full tier on a merge, before it is made (#1198): `commitOid` is the | |
| 110 | * merge of the pull with its base, run in a clone of its own. Nothing is | |
| 111 | * queued and no run is written; the answer is the verdict. `interrupted` | |
| 112 | * is a gate that did not finish for a reason that is not the code — this | |
| 113 | * forge stopping, or the cluster losing its Job — and is no verdict (#3155). | |
| 114 | * `signal` aborting (the queue's deadline, #3430) cuts it off the same way. | |
| 115 | */ | |
| 116 | gate(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone>; | |
| 117 | /** | |
| 118 | * The gate of `commitOid` this forge is running now, to wait on rather | |
| 119 | * than start again (#6611); undefined when none is. | |
| 120 | */ | |
| 121 | adopt?(commitOid: string): Promise<GateDone> | undefined; | |
| 122 | /** | |
| 123 | * Stop checking. Checks under way get `drainMs` to finish unless `drain` | |
| 124 | * is false (a person's Ctrl-C): then they are cut off and queued again now. | |
| 125 | */ | |
| 126 | stop(options?: { drain?: boolean }): Promise<void>; | |
| 127 | /** | |
| 128 | * Check slots free now for a merge-queue gate beyond a line's first | |
| 129 | * (#4053): what neither a running check nor such a gate holds, less | |
| 130 | * `PULL_RESERVE` kept for pull checks. | |
| 131 | */ | |
| 132 | spareSlots(): number; | |
| 133 | /** The gate checks take their turn at; a hosted child's checks share it. */ | |
| 134 | readonly lane: Gate; | |
| 135 | /** | |
| 136 | * Ask for every verification after landing that is due now | |
| 137 | * (`checks/verify.ts`); the answer is the runs asked for. The forge's | |
| 138 | * own timer calls it when `verifyEveryMs` is set. | |
| 139 | */ | |
| 140 | verifyDue(now?: number): Promise<Run[]>; | |
| 141 | } | |
| 142 | ||
| 143 | /** | |
| 144 | * Check slots the merge queue's extra gates never take (#4053): pull checks | |
| 145 | * always have at least these, however long the queue. | |
| 146 | */ | |
| 147 | export const PULL_RESERVE = 2; | |
| 148 | ||
| 149 | /** | |
| 150 | * How long main's check after a land waits to start (#4061). The merge queue | |
| 151 | * lands a candidate's steps seconds apart, and only its tip was gated: a | |
| 152 | * step's check that started at once ran the full suite on a commit the next | |
| 153 | * step replaced, up to seven of them filling the lane for nine minutes. A | |
| 154 | * step's check that waits is superseded by the next while still queued, and | |
| 155 | * the tip's is proven green by the gate. | |
| 156 | */ | |
| 157 | export const LAND_SETTLE_MS = 30_000; | |
| 158 | ||
| 159 | /** | |
| 160 | * What one checks service holds, in one plain record the lane's functions are handed | |
| 161 | * (`startChecks` makes it): its queue's local state, and the two calls that reach back across | |
| 162 | * modules (a request schedules, a schedule's start asks again). | |
| 163 | */ | |
| 164 | export interface LaneState { | |
| 165 | readonly app: App; | |
| 166 | readonly options: ChecksOptions; | |
| 167 | /** The gate checks take their turn at. */ | |
| 168 | readonly gate: Gate; | |
| 169 | readonly placement: Placement; | |
| 170 | /** | |
| 171 | * Full gates on land in flight: not slots in the queue, but stopped with it — after the drain, | |
| 172 | * as a check is. By commit, so a merge queue that carries on adopts one rather than gate the | |
| 173 | * same tip again (#6611). | |
| 174 | */ | |
| 175 | readonly gates: Map<AbortController, { commitOid: string; done: Promise<GateDone> }>; | |
| 176 | readonly inFlight: Map<string, { promise: Promise<void>; abort: AbortController }>; | |
| 177 | /** Checks already told why they wait, so the log says it once. */ | |
| 178 | readonly waiting: Set<string>; | |
| 179 | /** | |
| 180 | * Queued checks a full disk holds that have said so once (#11799); apart from `waiting`, so | |
| 181 | * neither line hides the other. | |
| 182 | */ | |
| 183 | readonly diskWaiting: Set<string>; | |
| 184 | readonly memoryWaiting: Set<string>; | |
| 185 | /** The verification-lane run in flight, which holds no check slot. */ | |
| 186 | verifying: string | undefined; | |
| 187 | /** How long main's check after a land waits before it may start. */ | |
| 188 | readonly settle: number; | |
| 189 | stopped: boolean; | |
| 190 | /** While above 0, a batch of requests is being queued and `schedule` waits for its end. */ | |
| 191 | holding: number; | |
| 192 | readonly pullHold: ReturnType<typeof holdClock>; | |
| 193 | /** The runner every check goes to. */ | |
| 194 | readonly runner: Runner; | |
| 195 | /** Fill every free slot with the queued check that deserves it. */ | |
| 196 | schedule(): void; | |
| 197 | /** Ask for a check of one commit. */ | |
| 198 | request( | |
| 199 | repo: Repo, | |
| 200 | env: string, | |
| 201 | commitOid: string, | |
| 202 | reason: string, | |
| 203 | issueId?: string, | |
| 204 | actorId?: string, | |
| 205 | notBefore?: string, | |
| 206 | ): Promise<Run | undefined>; | |
| 207 | } | |
| 208 | ||
| 209 | /** Check slots in use: what runs, less the verification lane's, which holds none. */ | |
| 210 | export const slotted = (lane: LaneState): number => | |
| 211 | lane.inFlight.size - (lane.verifying === undefined ? 0 : 1); | |
| 212 | ||
| 213 | /** | |
| 214 | * How many checks may run at once, read on every pick (`core/capacity.ts`): an admin's change is | |
| 215 | * honoured by the next check, not the next boot. `concurrency` is the boot default. | |
| 216 | */ | |
| 217 | export const capacity = (lane: LaneState): number => | |
| 218 | capacityCount(lane.app.db, "checks.concurrency", lane.options.concurrency ?? 1); |
src/checks/local_runner.ts added +75
| @@ -0,0 +1,75 @@ | ||
| 1 | ||
| 2 | import { runDir } from "../core/pulls.ts"; | |
| 3 | import type { App } from "../app.ts"; | |
| 4 | import { getPull, type Pull } from "../core/pulls.ts"; | |
| 5 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 6 | import { getIssueById } from "../core/issues.ts"; | |
| 7 | import { LOCAL_RUNNER, type CheckOutcome, type Runner } from "./runner.ts"; | |
| 8 | ||
| 9 | ||
| 10 | /** What one tier run is given (`runTier`). */ | |
| 11 | export interface Execution { | |
| 12 | readonly bare: string; | |
| 13 | readonly repo: Repo; | |
| 14 | /** Names the checkout. */ | |
| 15 | readonly id: string; | |
| 16 | readonly runId?: string; | |
| 17 | readonly commitOid: string; | |
| 18 | readonly pull?: Pull | undefined; | |
| 19 | /** | |
| 20 | * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after | |
| 21 | * landing. | |
| 22 | */ | |
| 23 | readonly tier: "run" | "fast" | "full" | "verify"; | |
| 24 | /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */ | |
| 25 | readonly only?: readonly string[]; | |
| 26 | /** A bisection step: the commit whose `check.verify` it runs. */ | |
| 27 | readonly commandsFrom?: string; | |
| 28 | /** A check of the default branch's own push: its cache is kept. */ | |
| 29 | readonly keep?: boolean; | |
| 30 | /** A green run keeps the impact map it measured; `full` always does. */ | |
| 31 | readonly keepImpact?: boolean; | |
| 32 | /** The default branch's own run: only its log may open issues. */ | |
| 33 | readonly fileOrderDependent?: boolean; | |
| 34 | /** | |
| 35 | * The default branch's own run: a failure of one or two tests is re-run alone before it counts | |
| 36 | * as red. | |
| 37 | */ | |
| 38 | readonly isMain?: boolean; | |
| 39 | readonly write: (line: string) => void; | |
| 40 | readonly signal: AbortSignal; | |
| 41 | } | |
| 42 | ||
| 43 | /** The runner every check goes to: this process (`runner.ts`); `tier` runs one described check. */ | |
| 44 | export function makeLocalRunner(app: App, tier: (x: Execution) => Promise<CheckOutcome>): Runner { | |
| 45 | return { | |
| 46 | name: LOCAL_RUNNER, | |
| 47 | async run(job, io) { | |
| 48 | const repo = getRepoById(app.db, job.repoId); | |
| 49 | if (repo === undefined) return { ok: false, error: "the repository is gone" }; | |
| 50 | const bare = await runDir(app.db, app.reposRoot, repo, job.issueId); | |
| 51 | // Only a pull's own check is judged against the pull, resolved here and nowhere else; the | |
| 52 | // verification lane never is. | |
| 53 | const issue = | |
| 54 | job.tier === "fast" && job.issueId !== null ? getIssueById(app.db, job.issueId) : undefined; | |
| 55 | const pull = issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined; | |
| 56 | return tier({ | |
| 57 | bare, | |
| 58 | repo, | |
| 59 | id: job.runId, | |
| 60 | runId: job.runId, | |
| 61 | commitOid: job.commitOid, | |
| 62 | pull, | |
| 63 | tier: job.tier, | |
| 64 | ...(job.only === undefined ? {} : { only: job.only }), | |
| 65 | ...(job.commandsFrom === undefined ? {} : { commandsFrom: job.commandsFrom }), | |
| 66 | keep: job.keep, | |
| 67 | keepImpact: job.keepImpact, | |
| 68 | fileOrderDependent: job.fileOrderDependent, | |
| 69 | isMain: job.isMain, | |
| 70 | write: io.write, | |
| 71 | signal: io.signal, | |
| 72 | }); | |
| 73 | }, | |
| 74 | }; | |
| 75 | } |
src/checks/pick.ts added +55
| @@ -0,0 +1,55 @@ | ||
| 1 | ||
| 2 | import type { Run } from "../core/runs.ts"; | |
| 3 | import { ownersFirst } from "./outside_lane.ts"; | |
| 4 | import { STARVATION_MS } from "../kernel/budgets.ts"; | |
| 5 | ||
| 6 | ||
| 7 | /** | |
| 8 | * Whether a pull's check must pass over the free slots left (`free`): while | |
| 9 | * a merge-queue gate is about to start (`gateQueued`), the last one is the | |
| 10 | * gate's. | |
| 11 | */ | |
| 12 | export function holdsLastSlot(free: number, gateWaits: boolean): boolean { | |
| 13 | return gateWaits && free <= 1; | |
| 14 | } | |
| 15 | ||
| 16 | /** | |
| 17 | * Which queued check takes a slot when one frees: the one requested most | |
| 18 | * recently, so a push's own check is never stuck behind pulls that queued | |
| 19 | * hours earlier — unless something has already waited `STARVATION_MS` or | |
| 20 | * more, in which case the longest-waiting check goes first regardless of | |
| 21 | * how fresh the rest are. Checks of approved pulls (`approved`) go before | |
| 22 | * the rest, newest first among themselves (#6396): a check that is the last | |
| 23 | * thing between a pull and landing is not held up by a new pull's. Ties | |
| 24 | * (down to the same instant) fall back to id order, which is itself | |
| 25 | * creation order (`kernel/id.ts`). Among the rest, and among the approved, | |
| 26 | * the pull whose issue stands highest on the owner's ranked list (`rank`, | |
| 27 | * 1 at the top) goes first, the unranked last, newest first only as the | |
| 28 | * tie-break (#12230). | |
| 29 | */ | |
| 30 | export function pickNext( | |
| 31 | queued: readonly Run[], | |
| 32 | now: number = Date.now(), | |
| 33 | approved: (run: Run) => boolean = () => false, | |
| 34 | rank: (run: Run) => number | undefined = () => undefined, | |
| 35 | outside: (run: Run) => boolean = () => false, | |
| 36 | ): Run | undefined { | |
| 37 | // An unapproved outside contributor's check goes after everyone else's (`outside_lane.ts`). | |
| 38 | queued = ownersFirst(queued, outside); | |
| 39 | if (queued.length === 0) return undefined; | |
| 40 | const newer = (a: Run, b: Run): boolean => | |
| 41 | a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id); | |
| 42 | let oldest = queued[0]!; | |
| 43 | const place = (run: Run): number => rank(run) ?? Number.MAX_SAFE_INTEGER; | |
| 44 | const better = (a: Run, b: Run): boolean => place(a) < place(b) || (place(a) === place(b) && newer(a, b)); | |
| 45 | let newest: Run | undefined; | |
| 46 | let newestApproved: Run | undefined; | |
| 47 | for (const run of queued) { | |
| 48 | if (newer(oldest, run)) oldest = run; | |
| 49 | if (newest === undefined || better(run, newest)) newest = run; | |
| 50 | if (approved(run) && (newestApproved === undefined || better(run, newestApproved))) | |
| 51 | newestApproved = run; | |
| 52 | } | |
| 53 | if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest; | |
| 54 | return newestApproved ?? newest; | |
| 55 | } |
src/checks/recheck_base.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | ||
| 2 | import { pullDir } from "../core/pulls.ts"; | |
| 3 | import type { App } from "../app.ts"; | |
| 4 | import { getPull, listOpenPulls, type Pull } from "../core/pulls.ts"; | |
| 5 | import type { Repo } from "../core/repos.ts"; | |
| 6 | import { getRun, checksForHead, type Run } from "../core/runs.ts"; | |
| 7 | import { resolveOid } from "../git/merge.ts"; | |
| 8 | import { BASE_MOVED, dueGreen, newestBaseFirst } from "./base_moved.ts"; | |
| 9 | import { createLog } from "../kernel/log.ts"; | |
| 10 | import type { LaneState } from "./lane_state.ts"; | |
| 11 | ||
| 12 | const log = createLog("checks"); | |
| 13 | ||
| 14 | /** | |
| 15 | * The base moved (#12533): every open pull on it whose green was of an | |
| 16 | * older tip is checked again, once per move (`base_moved.ts`), newest base | |
| 17 | * first (`newestBaseFirst`). A queued check merges the newest tip when it | |
| 18 | * starts, so a later move supersedes nothing and queues nothing. Skipped | |
| 19 | * where the merge queue gates lands: its gate merges with the tip itself. | |
| 20 | */ | |
| 21 | export async function recheckBase(lane: LaneState, repo: Repo, branch: string, tip: string): Promise<void> { | |
| 22 | const { app } = lane; | |
| 23 | if (landGated(repo)) return; | |
| 24 | const due: { number: number; green: Run; pull: Pull }[] = []; | |
| 25 | for (const pull of listOpenPulls(app.db, repo.id)) { | |
| 26 | if (pull.baseBranch !== branch) continue; | |
| 27 | const green = dueOf(app, pull, tip); | |
| 28 | if (green !== undefined) due.push({ number: pull.number, green, pull }); | |
| 29 | } | |
| 30 | await askInOrder(lane, repo, newestBaseFirst(due).map((item) => item.pull)); | |
| 31 | } | |
| 32 | ||
| 33 | export const landGated = (repo: Repo): boolean => repo.autonomy === "autonomous" && repo.autonomyBy !== null; | |
| 34 | ||
| 35 | function dueOf(app: App, pull: Pull, tip: string): Run | undefined { | |
| 36 | if (pull.state !== "open" || pull.headOid === "" || pull.mergedAt !== null) return undefined; | |
| 37 | return dueGreen(checksForHead(app.db, pull.id, pull.headOid), `pull-${pull.number}`, tip, false); | |
| 38 | } | |
| 39 | ||
| 40 | /** | |
| 41 | * Asks a re-check for each pull, the first to take a slot first. The queue | |
| 42 | * starts the newest request first (`pickNext`), so they are asked last-first | |
| 43 | * with scheduling held until every one is queued: a free slot does not go | |
| 44 | * to whichever happened to be asked before the rest. | |
| 45 | */ | |
| 46 | async function askInOrder(lane: LaneState, repo: Repo, pulls: readonly Pull[]): Promise<void> { | |
| 47 | if (pulls.length === 0) return; | |
| 48 | lane.holding++; | |
| 49 | try { | |
| 50 | for (const pull of [...pulls].reverse()) { | |
| 51 | try { | |
| 52 | await lane.request(repo, `pull-${pull.number}`, pull.headOid, BASE_MOVED, pull.id); | |
| 53 | } catch (error) { | |
| 54 | log.error("check could not be asked again after the base moved", { pull: pull.number, error: String(error) }); | |
| 55 | } | |
| 56 | } | |
| 57 | } finally { | |
| 58 | lane.holding--; | |
| 59 | lane.schedule(); | |
| 60 | } | |
| 61 | } | |
| 62 | ||
| 63 | /** A check that finished green while its base moved is asked again, once, on the tip now. */ | |
| 64 | export async function recheckFinished(lane: LaneState, runId: string, repo: Repo): Promise<void> { | |
| 65 | const { app } = lane; | |
| 66 | // The common path, a red or a gated repository, reads no ref and no pull. | |
| 67 | if (landGated(repo)) return; | |
| 68 | const run = getRun(app.db, runId); | |
| 69 | const number = /^pull-(\d+)$/.exec(run?.env ?? "")?.[1]; | |
| 70 | if (run === undefined || number === undefined || run.state !== "ok" || run.baseOid === "") return; | |
| 71 | const pull = getPull(app.db, repo.id, Number(number)); | |
| 72 | if (!pull || pull.id !== run.issueId || pull.headOid !== run.commitOid) return; | |
| 73 | // A branch's tip is a file read (`resolveOid`), no process. | |
| 74 | const tip = await resolveOid(await pullDir(app.db, app.reposRoot, repo, pull), `refs/heads/${pull.baseBranch}`); | |
| 75 | if (tip === undefined || tip === run.baseOid) return; | |
| 76 | if (dueOf(app, pull, tip) !== undefined) await askInOrder(lane, repo, [pull]); | |
| 77 | } |
src/checks/request.ts added +99
| @@ -0,0 +1,99 @@ | ||
| 1 | ||
| 2 | import { runDir } from "../core/pulls.ts"; | |
| 3 | import { announceRun } from "../actions/runs.ts"; | |
| 4 | import { isContributionFork } from "../core/contribute.ts"; | |
| 5 | import { issueMayRun } from "../core/pull_trust.ts"; | |
| 6 | import type { Repo } from "../core/repos.ts"; | |
| 7 | import { cancelRun, createRun, listActive, type Run } from "../core/runs.ts"; | |
| 8 | import { ASKED } from "./red_main.ts"; | |
| 9 | import { VERIFY_ENV } from "./verify.ts"; | |
| 10 | import { transact } from "../kernel/db.ts"; | |
| 11 | import { createLog } from "../kernel/log.ts"; | |
| 12 | import { readFileAt } from "../apps/checkout.ts"; | |
| 13 | import { parseCheck } from "../apps/manifest.ts"; | |
| 14 | import { accountActive } from "../core/accounts.ts"; | |
| 15 | import type { LaneState } from "./lane_state.ts"; | |
| 16 | ||
| 17 | const log = createLog("checks"); | |
| 18 | ||
| 19 | export async function request( | |
| 20 | lane: LaneState, | |
| 21 | repo: Repo, | |
| 22 | env: string, | |
| 23 | commitOid: string, | |
| 24 | reason: string, | |
| 25 | issueId?: string, | |
| 26 | actorId?: string, | |
| 27 | notBefore?: string, | |
| 28 | ): Promise<Run | undefined> { | |
| 29 | const { app } = lane; | |
| 30 | // A suspended or removed owner's code is not run, not even to check it. | |
| 31 | if (lane.stopped || !accountActive(app.db, repo.ownerId)) return undefined; | |
| 32 | // A guest's fork runs nothing: its code runs only as a pull a person approved. | |
| 33 | if (isContributionFork(app.db, repo.id)) return undefined; | |
| 34 | // A pull from someone who cannot write runs nothing until a writer | |
| 35 | // approves this exact head (`core/pull_trust.ts`). | |
| 36 | if (!issueMayRun(app.db, repo, issueId, commitOid)) { | |
| 37 | log.info("check waits for a writer's approval", { | |
| 38 | repo: `${repo.ownerName}/${repo.name}`, | |
| 39 | env, | |
| 40 | commit: commitOid.slice(0, 7), | |
| 41 | }); | |
| 42 | return undefined; | |
| 43 | } | |
| 44 | // A sealed pull's commit is where its work is, never in the base (`pullDir`). | |
| 45 | const bare = await runDir(app.db, app.reposRoot, repo, issueId); | |
| 46 | const [forgeJson, packageJson] = await Promise.all([ | |
| 47 | readFileAt(bare, commitOid, "forge.json"), | |
| 48 | readFileAt(bare, commitOid, "package.json"), | |
| 49 | ]); | |
| 50 | const declared = parseCheck(forgeJson, packageJson); | |
| 51 | if (declared.ok && declared.value === undefined) return undefined; | |
| 52 | // Main is verified after landing only where it says how (#7786). | |
| 53 | if (env === VERIFY_ENV && (!declared.ok || declared.value?.verify == null)) return undefined; | |
| 54 | // One check per environment at a time: a newer commit supersedes what | |
| 55 | // is still waiting, and the same commit is not checked twice. Looked for | |
| 56 | // and written in one write transaction, so a second forge process over | |
| 57 | // this database (a roll) cannot queue the same check beside it. | |
| 58 | const superseded: Run[] = []; | |
| 59 | const { run, fresh } = transact(app.db, () => { | |
| 60 | for (const active of listActive(app.db, "check")) { | |
| 61 | if (active.repoId !== repo.id || active.env !== env) continue; | |
| 62 | // A person asking for a check a red main holds queues it as theirs, | |
| 63 | // which runs (`checks/red_main.ts`); any other repeat is the same check. | |
| 64 | if ( | |
| 65 | active.commitOid === commitOid && | |
| 66 | !(reason === ASKED && active.state === "queued" && active.reason !== ASKED) | |
| 67 | ) | |
| 68 | return { run: active, fresh: false }; | |
| 69 | // A pull's running check is for a head nobody will land either: | |
| 70 | // it is stopped too, below, once the row says so. | |
| 71 | if (active.state === "queued" || issueId !== undefined) { | |
| 72 | const gone = cancelRun( | |
| 73 | app.db, | |
| 74 | active.id, | |
| 75 | `superseded by ${commitOid.slice(0, 7)}`, | |
| 76 | ); | |
| 77 | if (gone && active.state === "running") superseded.push(gone); | |
| 78 | } | |
| 79 | } | |
| 80 | const created = createRun(app.db, { | |
| 81 | repoId: repo.id, | |
| 82 | kind: "check", | |
| 83 | env, | |
| 84 | commitOid, | |
| 85 | reason, | |
| 86 | ...(issueId === undefined ? {} : { issueId }), | |
| 87 | ...(actorId === undefined ? {} : { actorId }), | |
| 88 | ...(notBefore === undefined ? {} : { notBefore }), | |
| 89 | }); | |
| 90 | return { run: created, fresh: true }; | |
| 91 | }); | |
| 92 | // The row is cancelled, so the check's own end finishes nothing and | |
| 93 | // announces nothing; the abort is what frees its slot and its checkout. | |
| 94 | for (const old of superseded) lane.inFlight.get(old.id)?.abort.abort(); | |
| 95 | if (!fresh) return run; | |
| 96 | announceRun(app, run, actorId); | |
| 97 | lane.schedule(); | |
| 98 | return run; | |
| 99 | } |
src/checks/run_tier.ts added +376
| @@ -0,0 +1,376 @@ | ||
| 1 | ||
| 2 | import { cellHides } from "../spawn/hosted-mark.ts"; | |
| 3 | import { existsSync, mkdirSync } from "node:fs"; | |
| 4 | import { rm } from "node:fs/promises"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { noteRunProgress } from "../core/runs.ts"; | |
| 7 | import { runGit } from "../git/env.ts"; | |
| 8 | import { checkKey, provenBy, recordProven } from "./proven.ts"; | |
| 9 | import { giveHistory } from "./history.ts"; | |
| 10 | import { readLearned } from "./learned.ts"; | |
| 11 | import { keepCache, restoreCache } from "./cache.ts"; | |
| 12 | import { clearedIn } from "./base_red.ts"; | |
| 13 | import { scratchEnv } from "./disk_full.ts"; | |
| 14 | import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts"; | |
| 15 | import { KEPT_FILE, keepKept, seedKept } from "./kept-store.ts"; | |
| 16 | import { redDetail } from "./report.ts"; | |
| 17 | import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts"; | |
| 18 | import { failedFiles, onlyEnv, passedFiles } from "./verify.ts"; | |
| 19 | import { exportTree } from "../apps/checkout.ts"; | |
| 20 | import { workerCpus } from "../core/capacity.ts"; | |
| 21 | import { childEnv, killedStep, runStep } from "../apps/process.ts"; | |
| 22 | import { noteReached } from "../apps/egress_report.ts"; | |
| 23 | import { capacityEnv, cpuLimit, egressSetting } from "../core/capacity.ts"; | |
| 24 | import { cellsAvailable, origin, limitWarnings, resolveLimits, STEP_HOSTS, stoppedForCpu, type CellSpec } from "../apps/contain.ts"; | |
| 25 | import { summariseCheckFailure } from "./summary.ts"; | |
| 26 | import { err, type Result } from "../kernel/result.ts"; | |
| 27 | import { JOB_GATE_NOTE, jobGatePolicy, openJobGate } from "./job_gate.ts"; | |
| 28 | import type { Placement } from "./placement.ts"; | |
| 29 | import { refuseSmallPod } from "./pod_room.ts"; | |
| 30 | import { JOB_VANISHED, runJobStep } from "./job_step.ts"; | |
| 31 | import { checkStage } from "./gate_reason.ts"; | |
| 32 | import type { CheckOutcome } from "./runner.ts"; | |
| 33 | import type { LaneState } from "./lane_state.ts"; | |
| 34 | import type { Execution } from "./local_runner.ts"; | |
| 35 | import { resolveSource } from "./tier_source.ts"; | |
| 36 | import { resolveManifest } from "./tier_manifest.ts"; | |
| 37 | import { tryRunner } from "./tier_runner.ts"; | |
| 38 | import { runInstall } from "./tier_install.ts"; | |
| 39 | import { settleFlakes } from "./tier_flake.ts"; | |
| 40 | import { checksRoot, reclaimIfFull, removeRunDir } from "./sweep.ts"; | |
| 41 | ||
| 42 | /** | |
| 43 | * Run one tier of one check: resolve the commit and the manifest it runs by, give it to a runner | |
| 44 | * or run it here in a cell, and answer the verdict. The steps it is made of are `tier_*.ts`. | |
| 45 | */ | |
| 46 | export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutcome> { | |
| 47 | const { app, options } = lane; | |
| 48 | const { bare, repo, write, signal } = x; | |
| 49 | const sourced = await resolveSource(lane, x); | |
| 50 | if (!sourced.ok) return sourced; | |
| 51 | const { source, tip, builtOnBase, mergeLine } = sourced; | |
| 52 | const run = { id: x.id, commitOid: source }; | |
| 53 | const resolved = await resolveManifest(lane, x, source); | |
| 54 | if (!resolved.ok) return resolved; | |
| 55 | const { manifest, outside } = resolved; | |
| 56 | const command = | |
| 57 | x.tier === "full" | |
| 58 | ? manifest.full | |
| 59 | : x.tier === "fast" | |
| 60 | ? (manifest.fast ?? manifest.run) | |
| 61 | : x.tier === "verify" | |
| 62 | ? manifest.verify! | |
| 63 | : manifest.run; | |
| 64 | // The fast tier diffs what the pull changes against the commit it builds | |
| 65 | // on, which the clone is given as a repository of its own: `origin/main` | |
| 66 | // is not a ref there. | |
| 67 | const base = manifest.fast !== null ? builtOnBase : undefined; | |
| 68 | // The tree this runs on under this gate has been green before: that is | |
| 69 | // the verdict, and nothing is checked out (#1795). | |
| 70 | const learned = | |
| 71 | base === undefined ? "" : readLearned(app.config.dataDir, x.pull!.id).join(","); | |
| 72 | const tree = await runGit(["rev-parse", `${run.commitOid}^{tree}`], { cwd: bare }); | |
| 73 | // A bisection step runs a slice: its green proves nothing of the whole. | |
| 74 | const key = tree.ok && x.only === undefined | |
| 75 | ? checkKey({ | |
| 76 | repoId: repo.id, | |
| 77 | treeOid: tree.value.trim(), | |
| 78 | command, | |
| 79 | install: manifest.install, | |
| 80 | timeout: manifest.timeout, | |
| 81 | egress: manifest.egress, | |
| 82 | limits: manifest.limits, | |
| 83 | base, | |
| 84 | learned, | |
| 85 | }) | |
| 86 | : undefined; | |
| 87 | const earlier = key === undefined ? undefined : provenBy(app.db, key); | |
| 88 | if (earlier !== undefined) { | |
| 89 | write(`already proven green by ${earlier}`); | |
| 90 | return { ok: true, proven: earlier }; | |
| 91 | } | |
| 92 | const sent = await tryRunner(lane, x, { | |
| 93 | run, | |
| 94 | repo, | |
| 95 | bare, | |
| 96 | manifest, | |
| 97 | command, | |
| 98 | base, | |
| 99 | builtOnBase, | |
| 100 | learned, | |
| 101 | tree, | |
| 102 | key, | |
| 103 | outside, | |
| 104 | write, | |
| 105 | signal, | |
| 106 | }); | |
| 107 | if (sent !== undefined) return sent; | |
| 108 | if (await reclaimIfFull(app)) write("disk under its free floor: swept unused checkouts"); | |
| 109 | const dir = join(checksRoot(app.config.dataDir), run.id); | |
| 110 | mkdirSync(dir, { recursive: true }); | |
| 111 | write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine); | |
| 112 | const exported = await exportTree(bare, run.commitOid, dir); | |
| 113 | if (!exported.ok) return { ok: false, error: exported.error }; | |
| 114 | if (base !== undefined) { | |
| 115 | const given = await giveHistory(bare, dir, base, run.commitOid); | |
| 116 | if (!given.ok) { | |
| 117 | await rm(dir, { recursive: true, force: true }); | |
| 118 | return { ok: false, error: given.error }; | |
| 119 | } | |
| 120 | } | |
| 121 | // What the base's own check left (`checks/cache.ts`): tsc's build info | |
| 122 | // and the gate's per-file times, keyed by the commit this one builds on. | |
| 123 | const cached = restoreCache( | |
| 124 | app.config.dataDir, | |
| 125 | repo.id, | |
| 126 | builtOnBase === undefined ? [] : [builtOnBase], | |
| 127 | dir, | |
| 128 | ); | |
| 129 | if (cached !== undefined) write(`cache from ${cached.slice(0, 7)}`); | |
| 130 | // Every check starts from the impact map the last green default-branch | |
| 131 | // run measured; the fast tier selects by it. | |
| 132 | try { | |
| 133 | const seeded = seedImpact(app.config.dataDir, repo.id, dir); | |
| 134 | if (seeded > 0) write(`impact map: ${seeded} tests, from the last green ${repo.defaultBranch}`); | |
| 135 | } catch (error) { | |
| 136 | write(`impact map not seeded: ${String(error)}`); | |
| 137 | } | |
| 138 | // The fast tier skips a test file whose inputs a default-branch run already proved green. | |
| 139 | try { | |
| 140 | const seeded = seedKept(app.config.dataDir, repo.id, dir); | |
| 141 | if (seeded > 0) write(`kept results: ${seeded} tests, from ${repo.defaultBranch}'s measuring runs`); | |
| 142 | } catch (error) { | |
| 143 | write(`kept results not seeded: ${String(error)}`); | |
| 144 | } | |
| 145 | // The check is repository code: it runs in a cell of its own | |
| 146 | // (`apps/contain.ts`), its checkout and a home its only places to write, | |
| 147 | // the forge, the registry and what it declared its only way out. | |
| 148 | // A Job holds egress only through a gate the forge opens for it | |
| 149 | // (`checks/job_gate.ts`, #13251): a cluster that names none cannot | |
| 150 | // honour forge.json, so the check stays in a cell, never unrestricted. | |
| 151 | const fellBack = lane.placement.mode === "job" && lane.placement.gate === undefined; | |
| 152 | // A check a runner did not take runs here as `process` does. | |
| 153 | const placed: Placement = fellBack || lane.placement.mode === "runner" ? { mode: "process" } : lane.placement; | |
| 154 | const job = placed.mode === "job" ? placed : undefined; | |
| 155 | if (x.runId !== undefined) | |
| 156 | noteRunProgress(app.db, x.runId, { placement: placed.mode }); | |
| 157 | if (fellBack) | |
| 158 | write("placement: job skipped, this cluster names no egress gate for a Job (FORGE_CHECKS_EGRESS_GATE)"); | |
| 159 | // A Job is its own boundary: no cell here, the cluster and its gate hold it. | |
| 160 | const cells = job === undefined ? await cellsAvailable() : undefined; | |
| 161 | if (cells !== undefined) { | |
| 162 | if (x.runId !== undefined) | |
| 163 | noteRunProgress(app.db, x.runId, { | |
| 164 | sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason }, | |
| 165 | }); | |
| 166 | if (!cells.ok) | |
| 167 | write(`uncontained here: ${cells.reason}; the check runs unrestricted`); | |
| 168 | } else { | |
| 169 | if (x.runId !== undefined) | |
| 170 | noteRunProgress(app.db, x.runId, { sandbox: { enforced: true, note: JOB_GATE_NOTE } }); | |
| 171 | write("placement: job"); | |
| 172 | } | |
| 173 | const home = `${dir}.home`; | |
| 174 | const cell: CellSpec | undefined = cells?.ok | |
| 175 | ? { | |
| 176 | dir, | |
| 177 | writable: [], | |
| 178 | home, | |
| 179 | hide: cellHides(app), | |
| 180 | egress: [...STEP_HOSTS, ...manifest.egress], | |
| 181 | reach: [options.forgeUrl], | |
| 182 | limits: resolveLimits("step", manifest.limits), | |
| 183 | onUndeclared: (host, port) => | |
| 184 | noteReached(app.config.dataDir, repo.id, host, port), | |
| 185 | // What an outside contributor's check may reach is held to, whatever the forge's setting. | |
| 186 | egressMode: outside ? "enforce" : egressSetting(app.db), | |
| 187 | label: `check:${run.id}`, | |
| 188 | } | |
| 189 | : undefined; | |
| 190 | try { | |
| 191 | // The pod's memory less the forge's own leaves too little for a cell | |
| 192 | // (#12519): refused before a step starts, so the forge is never the one | |
| 193 | // the OOM killer takes. The machine's, not the pull's: no red. A Job's | |
| 194 | // memory is the cluster's, not this pod's. | |
| 195 | const small = job === undefined | |
| 196 | ? refuseSmallPod({ env: process.env["FORGE_CELL_MEMORY_MAX_MIB"], ...options.podRoom }, write) | |
| 197 | : null; | |
| 198 | if (small !== null) return small; | |
| 199 | for (const line of limitWarnings(manifest.limits)) write(line); | |
| 200 | const cellCpus = resolveLimits("step", manifest.limits).cpus; | |
| 201 | const env = childEnv({ | |
| 202 | // The shard count an admin set (`core/capacity.ts`), else the | |
| 203 | // default sized to what this check's cell holds, never the pod's. | |
| 204 | ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)), | |
| 205 | CI: "1", | |
| 206 | FORGE_URL: options.forgeUrl, | |
| 207 | FORGE_CHECK: run.id, | |
| 208 | FORGE_COMMIT: x.commitOid, | |
| 209 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 210 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 211 | ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }), | |
| 212 | // The cell's CPU limit, which `availableParallelism()` may not | |
| 213 | // report inside it; the shard count follows this (#2257). | |
| 214 | FORGE_CELL_CPUS: String(cellCpus), | |
| 215 | // The one place besides the checkout the cell can write on disk; its | |
| 216 | // own /tmp is a 256 MiB tmpfs the shards would fill (#8679). | |
| 217 | ...scratchEnv(cell === undefined ? undefined : home), | |
| 218 | ...(base === undefined | |
| 219 | ? {} | |
| 220 | : { | |
| 221 | FORGE_CHECK_BASE: base, | |
| 222 | FORGE_CHECK_LEARNED: learned, | |
| 223 | }), | |
| 224 | // A cell inside a cell is not a cell. This check's own cell is the | |
| 225 | // boundary; a forge the tests boot inside it would wrap its apps in | |
| 226 | // nested cells, and those never answer in time. Its children run as | |
| 227 | // plain processes here — as they do on a runner without bwrap. | |
| 228 | FORGE_CELLS: "off", | |
| 229 | }); | |
| 230 | let ran: Result<void, string>; | |
| 231 | const checkLines: string[] = []; | |
| 232 | // A bisection step runs only some files: what it shows green says nothing of the rest. | |
| 233 | const wholeVerify = x.tier === "verify" && x.only === undefined; | |
| 234 | // The gate is held to what a cell holds: the forge, the registry and | |
| 235 | // what forge.json declares, an outside contributor's always enforced. | |
| 236 | const gate = job?.gate === undefined | |
| 237 | ? undefined | |
| 238 | : openJobGate( | |
| 239 | job.gate, | |
| 240 | jobGatePolicy({ | |
| 241 | hosts: [...STEP_HOSTS, ...manifest.egress], | |
| 242 | origins: [options.forgeUrl].map(origin).filter((o) => o !== undefined), | |
| 243 | label: `check:${run.id}`, | |
| 244 | outside, | |
| 245 | setting: egressSetting(app.db), | |
| 246 | note: (host, port) => noteReached(app.config.dataDir, repo.id, host, port), | |
| 247 | }), | |
| 248 | write, | |
| 249 | ); | |
| 250 | if (gate !== undefined && !gate.ok) { | |
| 251 | ran = err(gate.error); | |
| 252 | } else if (job !== undefined) { | |
| 253 | try { | |
| 254 | ran = await runJobStep(job, { | |
| 255 | runId: run.id, | |
| 256 | dir, | |
| 257 | commands: manifest.install === null ? [command] : [manifest.install, command], | |
| 258 | env, | |
| 259 | log: write, | |
| 260 | output: checkLines, | |
| 261 | signal, | |
| 262 | timeoutMs: manifest.timeout * 1000, | |
| 263 | limits: resolveLimits("step", manifest.limits), | |
| 264 | onNode: (node) => { | |
| 265 | if (x.runId !== undefined) noteRunProgress(app.db, x.runId, { node }); | |
| 266 | }, | |
| 267 | ...(gate === undefined ? {} : { proxy: gate.value.env }), | |
| 268 | ...(options.job === undefined ? {} : { options: options.job }), | |
| 269 | }); | |
| 270 | } finally { | |
| 271 | gate?.value.close(); | |
| 272 | } | |
| 273 | } else { | |
| 274 | const installed = await runInstall(lane, { dir, env, cell, repo, runId: run.id, manifest, write, signal }); | |
| 275 | if (installed !== undefined) return installed; | |
| 276 | // The check step's own lines are kept beside what `write` already | |
| 277 | // logs, so a failure can be summarised once here — never by reading | |
| 278 | // the log back, on this run or any later render of it. `runStep` | |
| 279 | // echoes the command as its first line before any output of the | |
| 280 | // command's own; that echo is not output to summarise from. | |
| 281 | let sawEcho = false; | |
| 282 | ran = await (options.checkStep ?? runStep)("check", command, { | |
| 283 | cwd: dir, | |
| 284 | env, | |
| 285 | log: (line) => { | |
| 286 | if (sawEcho) checkLines.push(line); | |
| 287 | sawEcho = true; | |
| 288 | write(line); | |
| 289 | }, | |
| 290 | signal, | |
| 291 | timeoutMs: manifest.timeout * 1000, | |
| 292 | cell, | |
| 293 | }); | |
| 294 | } | |
| 295 | if (!ran.ok && ran.error === JOB_VANISHED) return { ok: false, error: JOB_VANISHED }; | |
| 296 | ran = await settleFlakes( | |
| 297 | lane, | |
| 298 | x, | |
| 299 | { run, repo, bare, dir, env, cell, manifest, tip, inJob: job !== undefined, checkLines, write, signal }, | |
| 300 | ran, | |
| 301 | ); | |
| 302 | if (!ran.ok) { | |
| 303 | const { summary, detail } = summariseCheckFailure( | |
| 304 | checkLines, | |
| 305 | ran.error, | |
| 306 | ); | |
| 307 | const killed = killedStep(ran.error); | |
| 308 | // A red verification keeps the map it measured all the same, and | |
| 309 | // names the files it failed in for the bisection (#7786). | |
| 310 | const files = x.tier === "verify" ? failedFiles(checkLines) : []; | |
| 311 | // Each failing file once more on its own (#12296): green alone is | |
| 312 | // flaky under load, which a builder cannot fix; red alone stays red. | |
| 313 | const flaky = await rerunRed( | |
| 314 | { files, inJob: job !== undefined, killed, aborted: signal.aborted }, | |
| 315 | async (file, timeoutMs) => { | |
| 316 | const again = await runStep("rerun", rerunFileCommand(file), { | |
| 317 | cwd: dir, | |
| 318 | env, | |
| 319 | log: write, | |
| 320 | signal, | |
| 321 | timeoutMs: Math.min(timeoutMs, manifest.timeout * 1000), | |
| 322 | cell, | |
| 323 | }); | |
| 324 | return again.ok; | |
| 325 | }, | |
| 326 | write, | |
| 327 | ); | |
| 328 | if (x.tier === "verify" && x.keepImpact === true) { | |
| 329 | try { | |
| 330 | const kept = keepImpact(app.config.dataDir, repo.id, dir); | |
| 331 | if (kept !== undefined) write(kept); | |
| 332 | const results = keepKept(app.config.dataDir, repo.id, dir, true); | |
| 333 | if (results !== undefined) write(results); | |
| 334 | } catch (error) { | |
| 335 | write(`impact map not kept: ${String(error)}`); | |
| 336 | } | |
| 337 | } | |
| 338 | return { | |
| 339 | ok: false, | |
| 340 | error: killed ? ran.error : summary, | |
| 341 | detail: withFlaky(redDetail(detail, files, checkLines), flaky), | |
| 342 | ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}), | |
| 343 | stage: checkStage(ran.error), | |
| 344 | ...(killed ? { killed } : {}), | |
| 345 | ...(stoppedForCpu(ran.error) ? { cpuStop: ran.error } : {}), | |
| 346 | }; | |
| 347 | } | |
| 348 | // Only a green check of the default branch is kept for the next. | |
| 349 | if ( | |
| 350 | x.keep === true && | |
| 351 | keepCache(app.config.dataDir, repo.id, run.commitOid, dir) | |
| 352 | ) | |
| 353 | write(`cache kept for ${run.commitOid.slice(0, 7)}`); | |
| 354 | if (x.tier === "full" || x.keepImpact === true) { | |
| 355 | try { | |
| 356 | const kept = keepImpact(app.config.dataDir, repo.id, dir); | |
| 357 | if (kept !== undefined) write(kept); | |
| 358 | // Only the default branch's own run hands results back: a pull's tree could write any. | |
| 359 | const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true); | |
| 360 | if (results !== undefined) write(results); | |
| 361 | } catch (error) { | |
| 362 | write(`impact map not kept: ${String(error)}`); | |
| 363 | } | |
| 364 | } | |
| 365 | if (key !== undefined) recordProven(app.db, key, x.id); | |
| 366 | const cleared = clearedIn(checkLines); | |
| 367 | return { | |
| 368 | ok: true, | |
| 369 | ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}), | |
| 370 | ...(cleared === undefined ? {} : { cleared }), | |
| 371 | }; | |
| 372 | } finally { | |
| 373 | if (existsSync(dir)) await removeRunDir(dir); | |
| 374 | await removeRunDir(home); | |
| 375 | } | |
| 376 | } |
src/checks/scheduler.ts added +131
| @@ -0,0 +1,131 @@ | ||
| 1 | ||
| 2 | import { activeCrewRuns, crewHoldsCore, forgeCpus } from "../core/one_core.ts"; | |
| 3 | import { getRepoById } from "../core/repos.ts"; | |
| 4 | import { listActive, type Run } from "../core/runs.ts"; | |
| 5 | import { approvedOnce } from "./approved_first.ts"; | |
| 6 | import { outsideOnce } from "./outside_lane.ts"; | |
| 7 | import { redMain, unheld } from "./red_main.ts"; | |
| 8 | import { rebasesFirst } from "./finishing_first.ts"; | |
| 9 | import { inVerifyLane } from "./verify.ts"; | |
| 10 | import { appendRunLog } from "../runs/log.ts"; | |
| 11 | import { memoryHold } from "../runs/memory_hold.ts"; | |
| 12 | import { runningLimitSentence, runningRuns } from "../settings/quotas.ts"; | |
| 13 | import { usedRefusal } from "../apps/disk_guard.ts"; | |
| 14 | import { rankOnce } from "./ranked_first.ts"; | |
| 15 | import { gateQueued, gatesInFlight } from "./queue.ts"; | |
| 16 | import type { LaneState } from "./lane_state.ts"; | |
| 17 | import { slotted, capacity } from "./lane_state.ts"; | |
| 18 | import { pickNext, holdsLastSlot } from "./pick.ts"; | |
| 19 | import { start } from "./start.ts"; | |
| 20 | import { startVerify } from "./verify_lane.ts"; | |
| 21 | ||
| 22 | /** | |
| 23 | * An account at its concurrent-checks limit (`settings/quotas.ts`) keeps | |
| 24 | * the check queued, the count in its log once, until one of its own ends. | |
| 25 | */ | |
| 26 | function atLimit(lane: LaneState, run: Run): boolean { | |
| 27 | const { app } = lane; | |
| 28 | const owner = getRepoById(app.db, run.repoId)?.ownerId; | |
| 29 | if (owner === undefined) return false; | |
| 30 | const full = runningLimitSentence(app.db, "checks", "concurrent checks", owner, () => | |
| 31 | runningRuns(app.db, "check", owner, lane.inFlight.keys()), | |
| 32 | ); | |
| 33 | if (full === undefined) return false; | |
| 34 | if (!lane.waiting.has(run.id)) { | |
| 35 | lane.waiting.add(run.id); | |
| 36 | appendRunLog(app.config.dataDir, run.id, `queued, not started: ${full}`); | |
| 37 | } | |
| 38 | return true; | |
| 39 | } | |
| 40 | ||
| 41 | /** `pickNext` among `queued`, passing over any whose account is at its limit. */ | |
| 42 | function startable(lane: LaneState, queued: readonly Run[]): Run | undefined { | |
| 43 | const { app, options } = lane; | |
| 44 | // At 80% used no check starts (#11799): they stay queued, one line each, and the tick asks | |
| 45 | // again. | |
| 46 | const full = usedRefusal(app.config.dataDir); | |
| 47 | if (full !== undefined) { | |
| 48 | for (const run of queued) | |
| 49 | if (!lane.diskWaiting.has(run.id)) { | |
| 50 | lane.diskWaiting.add(run.id); | |
| 51 | appendRunLog(app.config.dataDir, run.id, `queued, ${full}`); | |
| 52 | } | |
| 53 | return undefined; | |
| 54 | } | |
| 55 | // A child near its memory limit runs one check at a time (`runs/memory_hold.ts`). | |
| 56 | const tight = memoryHold(slotted(lane), options.pressure); | |
| 57 | if (tight !== undefined) { | |
| 58 | for (const run of queued) | |
| 59 | if (!lane.memoryWaiting.has(run.id)) { | |
| 60 | lane.memoryWaiting.add(run.id); | |
| 61 | appendRunLog(app.config.dataDir, run.id, `queued, ${tight}`); | |
| 62 | } | |
| 63 | return undefined; | |
| 64 | } | |
| 65 | let rest = queued; | |
| 66 | const isApproved = approvedOnce(app.db); | |
| 67 | const rankOf = rankOnce(app.db); | |
| 68 | const isOutside = outsideOnce(app.db); | |
| 69 | let next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside); | |
| 70 | while (next !== undefined && atLimit(lane, next)) { | |
| 71 | const held = next; | |
| 72 | rest = rest.filter((run) => run !== held); | |
| 73 | next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside); | |
| 74 | } | |
| 75 | return next; | |
| 76 | } | |
| 77 | ||
| 78 | /** | |
| 79 | * Fill every free slot with the queued check `pickNext` says deserves it — | |
| 80 | * one the forge before this one never reached, one a forge stopping | |
| 81 | * beside this one handed back, or simply the newest push once a slot | |
| 82 | * opens up. Claiming is what decides between two forges, so calling this | |
| 83 | * from both never runs the same check twice. | |
| 84 | */ | |
| 85 | export function schedule(lane: LaneState): void { | |
| 86 | const { app, options } = lane; | |
| 87 | if (lane.stopped || lane.holding > 0) return; | |
| 88 | const limit = capacity(lane); | |
| 89 | if (options.gate === undefined) lane.gate.resize(limit); | |
| 90 | // On one CPU a pull's check waits while a crew run works, for at most | |
| 91 | // `holdMaxMs`; the verify lane, a rebase's check and a main check are the | |
| 92 | // pipeline and never wait (#11599). | |
| 93 | const crewHolds = crewHoldsCore((options.cpus ?? forgeCpus)(), options.activeCrewRuns ?? (() => activeCrewRuns(app.db))); | |
| 94 | // The clock counts one continuous hold: it starts again once no crew run | |
| 95 | // holds, and forgets checks that are no longer queued. | |
| 96 | if (!crewHolds) lane.pullHold.reset(); | |
| 97 | else lane.pullHold.keep(new Set(listActive(app.db, "check").filter((run) => run.state === "queued").map((run) => run.id))); | |
| 98 | startVerify(lane); | |
| 99 | // Every merge-queue gate in flight holds a slot, a line's first too (#6376). | |
| 100 | while (slotted(lane) < limit - gatesInFlight()) { | |
| 101 | // A pull's check waits while its main is red (`checks/red_main.ts`). | |
| 102 | const queued = unheld( | |
| 103 | listActive(app.db, "check").filter( | |
| 104 | (run) => | |
| 105 | run.state === "queued" && | |
| 106 | !lane.inFlight.has(run.id) && | |
| 107 | !inVerifyLane(run.env) && | |
| 108 | (run.notBefore === null || Date.parse(run.notBefore) <= Date.now()), | |
| 109 | ), | |
| 110 | (repoId) => redMain(app.db, repoId), | |
| 111 | options.redHoldMs ?? 0, | |
| 112 | ); | |
| 113 | // The limit is asked at the point of starting, one pick at a time: | |
| 114 | // a started check is in flight before the next is asked about, so a | |
| 115 | // burst of one account's checks cannot pass together. A rebase's | |
| 116 | // check goes before any other (`finishing_first.ts`, #6140). | |
| 117 | const { rebases, rest: others } = rebasesFirst(app.db, queued); | |
| 118 | // While a gate is about to start, the last free slot is its: a pull's | |
| 119 | // check waits for it, a rebase's and a main check do not. | |
| 120 | const held = holdsLastSlot(limit - gatesInFlight() - slotted(lane), gateQueued()); | |
| 121 | const waiting = crewHolds | |
| 122 | ? others.filter((run) => run.issueId === null || !lane.pullHold.held(run.id, (options.holdNow ?? Date.now)())) | |
| 123 | : others; | |
| 124 | const next = startable(lane, rebases) ?? startable(lane, held ? waiting.filter((run) => run.issueId === null) : waiting); | |
| 125 | if (!next) return; | |
| 126 | const repo = getRepoById(app.db, next.repoId); | |
| 127 | if (!repo) return; | |
| 128 | lane.pullHold.release(next.id); | |
| 129 | start(lane, next, repo); | |
| 130 | } | |
| 131 | } |
src/checks/service.ts +66 −2124
| @@ -11,371 +11,39 @@ | ||
| 11 | 11 | * is that command's business. |
| 12 | 12 | */ |
| 13 | 13 | |
| 14 | import { pullDir, runDir } from "../core/pulls.ts"; | |
| 15 | import { cellHides } from "../spawn/hosted-mark.ts"; | |
| 16 | import { childLabel } from "../perf/child_time.ts"; | |
| 17 | import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync } from "node:fs"; | |
| 18 | import { chmod, lstat, readdir, rm } from "node:fs/promises"; | |
| 19 | import { join } from "node:path"; | |
| 14 | import { holdClock } from "../core/one_core.ts"; | |
| 20 | 15 | import type { App } from "../app.ts"; |
| 21 | import { releaseOnGreen } from "../actions/holds.ts"; | |
| 22 | import { activeCrewRuns, crewHoldsCore, forgeCpus, holdClock } from "../core/one_core.ts"; | |
| 23 | import { announceRun } from "../actions/runs.ts"; | |
| 24 | import { announceShard } from "./shard_event.ts"; | |
| 25 | import type { ForgeEvent } from "../core/events.ts"; | |
| 26 | import { getPull, listOpenPulls, listPulls, type Pull } from "../core/pulls.ts"; | |
| 27 | import { isContributionFork } from "../core/contribute.ts"; | |
| 28 | import { issueMayRun } from "../core/pull_trust.ts"; | |
| 29 | import { outsideAgentWaits } from "../origins/guest.ts"; | |
| 30 | import { ASKS_NEXT, asksLine, heldPull } from "./held.ts"; | |
| 31 | import { OUTSIDE_CHECK_LINE, OUTSIDE_NO_GATE, OUTSIDE_UNCONTAINED } from "./outside.ts"; | |
| 32 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 33 | import { | |
| 34 | cancelRun, | |
| 35 | getRun, | |
| 36 | leaseLapsed, | |
| 37 | cancelRunsForPull, | |
| 38 | checksForHead, | |
| 39 | claimRun, | |
| 40 | createRun, | |
| 41 | finishRun, | |
| 42 | listActive, | |
| 43 | noteRunProgress, | |
| 44 | overranBefore, | |
| 45 | setRunBase, | |
| 46 | setRunCombined, | |
| 47 | vanishedBefore, | |
| 48 | VANISHED_SUMMARY, | |
| 49 | infraBefore, | |
| 50 | INFRA_SUMMARY, | |
| 51 | CPU_STOP_SUMMARY, | |
| 52 | DISK_USED_SUMMARY, | |
| 53 | requeueCheck, | |
| 54 | chainCutOffs, | |
| 55 | CHECK_RELEASE_LIMIT, | |
| 56 | RELEASE_SUMMARY, | |
| 57 | REQUEUED_SUMMARY, | |
| 58 | sweepAbandonedChecks, | |
| 59 | type Run, | |
| 60 | } from "../core/runs.ts"; | |
| 61 | import { Gate, runGit } from "../git/env.ts"; | |
| 62 | import { resolveOid } from "../git/merge.ts"; | |
| 63 | import { mergeForCheck } from "./merged.ts"; | |
| 64 | import { readCombined } from "../git/combined.ts"; | |
| 65 | import { approvedOnce } from "./approved_first.ts"; | |
| 66 | import { outsideOnce, ownersFirst } from "./outside_lane.ts"; | |
| 67 | import { checkOnApproval } from "./approval.ts"; | |
| 68 | import { BASE_MOVED, dueGreen, newestBaseFirst } from "./base_moved.ts"; | |
| 69 | import { checkKey, provenBy, recordProven } from "./proven.ts"; | |
| 70 | import { beginGate, endGate, gateHolding } from "./gate_record.ts"; | |
| 71 | import { giveHistory, headDepth } from "./history.ts"; | |
| 72 | import { readLearned } from "./learned.ts"; | |
| 73 | import { keepCache, restoreCache } from "./cache.ts"; | |
| 74 | import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts"; | |
| 75 | import { parseBunTestFailure } from "./bun-test.ts"; | |
| 76 | import { clearedIn } from "./base_red.ts"; | |
| 77 | import { recordCrashFlakes, passedAlone, recordFlakes, rerunCommand, rerunnable, pullOwnsRed } from "./flake.ts"; | |
| 78 | import { diskRecovered, withDisk } from "./disk_red.ts"; | |
| 79 | import { scratchEnv } from "./disk_full.ts"; | |
| 80 | import { ASKED, redMain, unheld } from "./red_main.ts"; | |
| 81 | import { proposeRevert } from "./revert_red.ts"; | |
| 82 | import { rebasesFirst } from "./finishing_first.ts"; | |
| 83 | import { IMPACT_DIR, IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts"; | |
| 84 | import { KEPT_DIR, KEPT_FILE, keepKept, seedKept } from "./kept-store.ts"; | |
| 85 | import { afterVerify, closeOnGreen } from "./culprit.ts"; | |
| 86 | import { noteLookPages } from "./look_pages.ts"; | |
| 87 | import { redDetail, reportGreen, reportRed } from "./report.ts"; | |
| 88 | import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts"; | |
| 89 | import { | |
| 90 | bisectOrigin, | |
| 91 | failedFiles, | |
| 92 | inVerifyLane, | |
| 93 | onlyEnv, | |
| 94 | passedFiles, | |
| 95 | VERIFY_ENV, | |
| 96 | VERIFY_EVERY_MS, | |
| 97 | VERIFY_REASON, | |
| 98 | VERIFY_SWEEP_MS, | |
| 99 | verifyDue, | |
| 100 | } from "./verify.ts"; | |
| 101 | import { getIssueById } from "../core/issues.ts"; | |
| 102 | import { BUDGETS, budgetSummary, checkLimitMs, STARVATION_MS } from "../kernel/budgets.ts"; | |
| 103 | import { transact } from "../kernel/db.ts"; | |
| 104 | import { createLog } from "../kernel/log.ts"; | |
| 105 | import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts"; | |
| 106 | import { appendRunLog } from "../runs/log.ts"; | |
| 107 | import { memoryHold } from "../runs/memory_hold.ts"; | |
| 108 | import { cancelledLine } from "../runs/cancelled.ts"; | |
| 109 | import { trackRun, untrackRun } from "../runs/stoppable.ts"; | |
| 110 | import { workBegins, workEnds } from "../runs/ending.ts"; | |
| 111 | import { runningLimitSentence, runningRuns } from "../settings/quotas.ts"; | |
| 112 | import { exportTree, readFileAt } from "../apps/checkout.ts"; | |
| 113 | import { CHECK_TIMEOUT_MAX, parseCheck } from "../apps/manifest.ts"; | |
| 114 | import { workerCpus } from "../core/capacity.ts"; | |
| 115 | import { accountActive } from "../core/accounts.ts"; | |
| 116 | import { childEnv, killedStep, runStep } from "../apps/process.ts"; | |
| 117 | import { requeueCpuStop } from "./cpu_stop.ts"; | |
| 118 | import { isUsedRefusal, usedRefusal } from "../apps/disk_guard.ts"; | |
| 119 | import { noteReached } from "../apps/egress_report.ts"; | |
| 120 | import { | |
| 121 | capacityCount, | |
| 122 | noteBooted, | |
| 123 | capacityEnv, | |
| 124 | cpuLimit, | |
| 125 | egressSetting, | |
| 126 | } from "../core/capacity.ts"; | |
| 127 | import { | |
| 128 | cellsAvailable, | |
| 129 | origin, | |
| 130 | limitWarnings, | |
| 131 | resolveLimits, | |
| 132 | STEP_HOSTS, | |
| 133 | stoppedForCpu, | |
| 134 | type CellSpec, | |
| 135 | } from "../apps/contain.ts"; | |
| 136 | import { summariseCheckFailure } from "./summary.ts"; | |
| 137 | import { err, type Result } from "../kernel/result.ts"; | |
| 138 | import { JOB_GATE_NOTE, jobGatePolicy, openJobGate } from "./job_gate.ts"; | |
| 139 | import { placementFromEnv, type Placement } from "./placement.ts"; | |
| 140 | import { refuseSmallPod, type PodRoom } from "./pod_room.ts"; | |
| 141 | import { JOB_VANISHED, runJobStep } from "./job_step.ts"; | |
| 142 | import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts"; | |
| 143 | 16 | import { sweepJobs } from "./runner_jobs.ts"; |
| 144 | import { | |
| 145 | cellInstall, | |
| 146 | checkInstallCache, | |
| 147 | meteredInstall, | |
| 148 | installWithRetry, | |
| 149 | prefillInstallCache, | |
| 150 | withLock, | |
| 151 | } from "./install.ts"; | |
| 152 | import { evictSoon, lend } from "./install_evict.ts"; | |
| 153 | import { checkStage, type GateStage } from "./gate_reason.ts"; | |
| 154 | import { rankOnce } from "./ranked_first.ts"; | |
| 155 | import { describeJob, LOCAL_RUNNER, type CheckOutcome, type Runner } from "./runner.ts"; | |
| 156 | import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts"; | |
| 157 | import type { JobOptions } from "../cluster/jobs.ts"; | |
| 158 | import type { CheckDetail } from "../core/runs.ts"; | |
| 159 | import { settleLandedRuns } from "../crew/interrupted.ts"; | |
| 17 | import { watchGates, extraGates } from "./queue.ts"; | |
| 18 | import { Gate } from "../git/env.ts"; | |
| 19 | import { capacityCount, noteBooted } from "../core/capacity.ts"; | |
| 20 | import { childLabel } from "../perf/child_time.ts"; | |
| 21 | import { VERIFY_SWEEP_MS } from "./verify.ts"; | |
| 22 | import { createLog } from "../kernel/log.ts"; | |
| 160 | 23 | import { during } from "../perf/jobs.ts"; |
| 24 | import type { ForgeEvent } from "../core/events.ts"; | |
| 25 | import { placementFromEnv } from "./placement.ts"; | |
| 26 | import { capacity, PULL_RESERVE, slotted, type ChecksHandle, type ChecksOptions, type LaneState } from "./lane_state.ts"; | |
| 27 | import { makeLocalRunner } from "./local_runner.ts"; | |
| 28 | import { sweepSoon } from "./sweep.ts"; | |
| 29 | import { request } from "./request.ts"; | |
| 30 | import { schedule } from "./scheduler.ts"; | |
| 31 | import { askVerifications, catchUp } from "./verify_lane.ts"; | |
| 32 | import { onEvent } from "./event_handler.ts"; | |
| 33 | import { gateRun } from "./gate_run.ts"; | |
| 34 | import { runTier } from "./run_tier.ts"; | |
| 161 | 35 | |
| 162 | 36 | const log = createLog("checks"); |
| 163 | 37 | |
| 164 | /** A gate this forge would not start, for it is stopping. */ | |
| 165 | const INTERRUPTED = "this forge is stopping"; | |
| 166 | ||
| 167 | export interface ChecksOptions { | |
| 168 | /** How many checks may run at once. @default 1 */ | |
| 169 | readonly concurrency?: number | undefined; | |
| 170 | /** How full this forge's memory is, 0 to 1: its cgroup's unless a test says otherwise. */ | |
| 171 | readonly pressure?: (() => number | undefined) | undefined; | |
| 172 | /** The forge's public URL, told to the check as `FORGE_URL`. */ | |
| 173 | readonly forgeUrl: string; | |
| 174 | /** The parent's gate, for a hosted child: its checks take their turn in the parent's lane. */ | |
| 175 | readonly gate?: Gate | undefined; | |
| 176 | /** Whose checks these are, written first in each run's log (a hosted child's name). */ | |
| 177 | readonly label?: string | undefined; | |
| 178 | /** How long a check may run before it is killed. @default BUDGETS.gate */ | |
| 179 | readonly budgetMs?: number | undefined; | |
| 180 | /** How long a verification of the default branch may run. @default BUDGETS.verify */ | |
| 181 | readonly verifyBudgetMs?: number | undefined; | |
| 182 | /** | |
| 183 | * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0, | |
| 184 | * nothing held; the forge says `RED_HOLD_MS`. | |
| 185 | */ | |
| 186 | readonly redHoldMs?: number | undefined; | |
| 187 | /** | |
| 188 | * How long main's check after a land waits before it may start, so the | |
| 189 | * steps of one candidate supersede each other while queued. @default 0, | |
| 190 | * none; the forge says `LAND_SETTLE_MS`. | |
| 191 | */ | |
| 192 | readonly landSettleMs?: number | undefined; | |
| 193 | /** | |
| 194 | * How long `stop` lets checks already running finish before it cuts them | |
| 195 | * off and queues them again (#1796). Nothing new starts meanwhile. | |
| 196 | * @default 0 | |
| 197 | */ | |
| 198 | readonly drainMs?: number | undefined; | |
| 199 | /** | |
| 200 | * Where a check's commands run (`checks/placement.ts`). @default from the | |
| 201 | * environment: `process`. | |
| 202 | */ | |
| 203 | readonly placement?: Placement | undefined; | |
| 204 | /** The kube client a `job` placement speaks through; tests stand one in. */ | |
| 205 | readonly job?: JobOptions | undefined; | |
| 206 | /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */ | |
| 207 | readonly installRetryMs?: number | undefined; | |
| 208 | /** | |
| 209 | * Runs the check step itself; tests stand one in for a stop the cell's watch makes | |
| 210 | * (#11617). @default runStep | |
| 211 | */ | |
| 212 | readonly checkStep?: typeof runStep | undefined; | |
| 213 | /** | |
| 214 | * Where the pod's memory.max is read and the forge's size | |
| 215 | * (`checks/pod_room.ts`); tests stand in a small pod. @default the real ones | |
| 216 | */ | |
| 217 | readonly podRoom?: Omit<PodRoom, "env"> | undefined; | |
| 218 | /** | |
| 219 | * How often the default branch is verified after landing at most | |
| 220 | * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds | |
| 221 | * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`. | |
| 222 | */ | |
| 223 | readonly verifyEveryMs?: number | undefined; | |
| 224 | /** Test seam: stands in for the tier the local runner runs, to see what reaches it. */ | |
| 225 | readonly runTier?: ((x: Execution) => Promise<CheckOutcome>) | undefined; | |
| 226 | /** Test seam: how often the queue is looked at. @default TICK_MS */ | |
| 227 | readonly tickMs?: number | undefined; | |
| 228 | /** The forge's CPU count; a 1-CPU forge holds pull checks for a crew run. @default forgeCpus */ | |
| 229 | readonly cpus?: (() => number) | undefined; | |
| 230 | /** Crew runs working now. @default the running agent runs */ | |
| 231 | readonly activeCrewRuns?: (() => number) | undefined; | |
| 232 | /** Test seams: how long a held pull check waits at most, and the clock it is judged by. */ | |
| 233 | readonly holdMaxMs?: number | undefined; | |
| 234 | readonly holdNow?: (() => number) | undefined; | |
| 235 | } | |
| 236 | ||
| 237 | /** What one tier run is given (`runTier`). */ | |
| 238 | export interface Execution { | |
| 239 | readonly bare: string; | |
| 240 | readonly repo: Repo; | |
| 241 | /** Names the checkout. */ | |
| 242 | readonly id: string; | |
| 243 | readonly runId?: string; | |
| 244 | readonly commitOid: string; | |
| 245 | readonly pull?: Pull | undefined; | |
| 246 | /** | |
| 247 | * `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after | |
| 248 | * landing. | |
| 249 | */ | |
| 250 | readonly tier: "run" | "fast" | "full" | "verify"; | |
| 251 | /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */ | |
| 252 | readonly only?: readonly string[]; | |
| 253 | /** A bisection step: the commit whose `check.verify` it runs. */ | |
| 254 | readonly commandsFrom?: string; | |
| 255 | /** A check of the default branch's own push: its cache is kept. */ | |
| 256 | readonly keep?: boolean; | |
| 257 | /** A green run keeps the impact map it measured; `full` always does. */ | |
| 258 | readonly keepImpact?: boolean; | |
| 259 | /** The default branch's own run: only its log may open issues. */ | |
| 260 | readonly fileOrderDependent?: boolean; | |
| 261 | /** | |
| 262 | * The default branch's own run: a failure of one or two tests is re-run alone before it counts | |
| 263 | * as red. | |
| 264 | */ | |
| 265 | readonly isMain?: boolean; | |
| 266 | readonly write: (line: string) => void; | |
| 267 | readonly signal: AbortSignal; | |
| 268 | } | |
| 269 | ||
| 270 | /** The runner every check goes to: this process (`runner.ts`); `tier` runs one described check. */ | |
| 271 | export function makeLocalRunner(app: App, tier: (x: Execution) => Promise<CheckOutcome>): Runner { | |
| 272 | return { | |
| 273 | name: LOCAL_RUNNER, | |
| 274 | async run(job, io) { | |
| 275 | const repo = getRepoById(app.db, job.repoId); | |
| 276 | if (repo === undefined) return { ok: false, error: "the repository is gone" }; | |
| 277 | const bare = await runDir(app.db, app.reposRoot, repo, job.issueId); | |
| 278 | // Only a pull's own check is judged against the pull, resolved here and nowhere else; the | |
| 279 | // verification lane never is. | |
| 280 | const issue = | |
| 281 | job.tier === "fast" && job.issueId !== null ? getIssueById(app.db, job.issueId) : undefined; | |
| 282 | const pull = issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined; | |
| 283 | return tier({ | |
| 284 | bare, | |
| 285 | repo, | |
| 286 | id: job.runId, | |
| 287 | runId: job.runId, | |
| 288 | commitOid: job.commitOid, | |
| 289 | pull, | |
| 290 | tier: job.tier, | |
| 291 | ...(job.only === undefined ? {} : { only: job.only }), | |
| 292 | ...(job.commandsFrom === undefined ? {} : { commandsFrom: job.commandsFrom }), | |
| 293 | keep: job.keep, | |
| 294 | keepImpact: job.keepImpact, | |
| 295 | fileOrderDependent: job.fileOrderDependent, | |
| 296 | isMain: job.isMain, | |
| 297 | write: io.write, | |
| 298 | signal: io.signal, | |
| 299 | }); | |
| 300 | }, | |
| 301 | }; | |
| 302 | } | |
| 303 | ||
| 304 | /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */ | |
| 305 | export interface GateDone { | |
| 306 | ok: boolean; | |
| 307 | error: string; | |
| 308 | lines: string[]; | |
| 309 | interrupted?: boolean; | |
| 310 | stage?: GateStage; | |
| 311 | } | |
| 312 | ||
| 313 | export interface ChecksHandle { | |
| 314 | /** | |
| 315 | * Ask for a check of one commit; the answer is the run, or nothing when the commit declares no | |
| 316 | * check. | |
| 317 | */ | |
| 318 | request( | |
| 319 | repo: Repo, | |
| 320 | env: string, | |
| 321 | commitOid: string, | |
| 322 | reason: string, | |
| 323 | issueId?: string, | |
| 324 | actorId?: string, | |
| 325 | ): Promise<Run | undefined>; | |
| 326 | /** | |
| 327 | * The full tier on a merge, before it is made (#1198): `commitOid` is the | |
| 328 | * merge of the pull with its base, run in a clone of its own. Nothing is | |
| 329 | * queued and no run is written; the answer is the verdict. `interrupted` | |
| 330 | * is a gate that did not finish for a reason that is not the code — this | |
| 331 | * forge stopping, or the cluster losing its Job — and is no verdict (#3155). | |
| 332 | * `signal` aborting (the queue's deadline, #3430) cuts it off the same way. | |
| 333 | */ | |
| 334 | gate(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone>; | |
| 335 | /** | |
| 336 | * The gate of `commitOid` this forge is running now, to wait on rather | |
| 337 | * than start again (#6611); undefined when none is. | |
| 338 | */ | |
| 339 | adopt?(commitOid: string): Promise<GateDone> | undefined; | |
| 340 | /** | |
| 341 | * Stop checking. Checks under way get `drainMs` to finish unless `drain` | |
| 342 | * is false (a person's Ctrl-C): then they are cut off and queued again now. | |
| 343 | */ | |
| 344 | stop(options?: { drain?: boolean }): Promise<void>; | |
| 345 | /** | |
| 346 | * Check slots free now for a merge-queue gate beyond a line's first | |
| 347 | * (#4053): what neither a running check nor such a gate holds, less | |
| 348 | * `PULL_RESERVE` kept for pull checks. | |
| 349 | */ | |
| 350 | spareSlots(): number; | |
| 351 | /** The gate checks take their turn at; a hosted child's checks share it. */ | |
| 352 | readonly lane: Gate; | |
| 353 | /** | |
| 354 | * Ask for every verification after landing that is due now | |
| 355 | * (`checks/verify.ts`); the answer is the runs asked for. The forge's | |
| 356 | * own timer calls it when `verifyEveryMs` is set. | |
| 357 | */ | |
| 358 | verifyDue(now?: number): Promise<Run[]>; | |
| 359 | } | |
| 360 | ||
| 361 | /** | |
| 362 | * Check slots the merge queue's extra gates never take (#4053): pull checks | |
| 363 | * always have at least these, however long the queue. | |
| 364 | */ | |
| 365 | export const PULL_RESERVE = 2; | |
| 366 | ||
| 367 | /** | |
| 368 | * How long main's check after a land waits to start (#4061). The merge queue | |
| 369 | * lands a candidate's steps seconds apart, and only its tip was gated: a | |
| 370 | * step's check that started at once ran the full suite on a commit the next | |
| 371 | * step replaced, up to seven of them filling the lane for nine minutes. A | |
| 372 | * step's check that waits is superseded by the next while still queued, and | |
| 373 | * the tip's is proven green by the gate. | |
| 374 | */ | |
| 375 | export const LAND_SETTLE_MS = 30_000; | |
| 376 | ||
| 377 | /** The longest a failed test is given to pass alone, in seconds. */ | |
| 378 | const RERUN_TIMEOUT_S = 300; | |
| 38 | // The service is cut along its seams, one concept to a module, and these are the names it has | |
| 39 | // always been imported by. | |
| 40 | export type { ChecksHandle, ChecksOptions, GateDone } from "./lane_state.ts"; | |
| 41 | export { LAND_SETTLE_MS, PULL_RESERVE } from "./lane_state.ts"; | |
| 42 | export type { Execution } from "./local_runner.ts"; | |
| 43 | export { makeLocalRunner } from "./local_runner.ts"; | |
| 44 | export { holdsLastSlot, pickNext } from "./pick.ts"; | |
| 45 | export { requeueAbandonedChecks } from "./abandoned.ts"; | |
| 46 | export { reclaimIfFull, removeRunDir, sweepCheckouts, sweepSoon } from "./sweep.ts"; | |
| 379 | 47 | |
| 380 | 48 | /** How often queued checks another forge left are looked for. */ |
| 381 | 49 | const TICK_MS = 5_000; |
| @@ -392,202 +60,8 @@ const TICK_MS = 5_000; | ||
| 392 | 60 | export { STARVATION_MS } from "../kernel/budgets.ts"; |
| 393 | 61 | |
| 394 | 62 | /** |
| 395 | * Whether a pull's check must pass over the free slots left (`free`): while | |
| 396 | * a merge-queue gate is about to start (`gateQueued`), the last one is the | |
| 397 | * gate's. | |
| 63 | * The one place a checks service is kept per forge: `checksFor` reads it, `startChecks` writes it. | |
| 398 | 64 | */ |
| 399 | export function holdsLastSlot(free: number, gateWaits: boolean): boolean { | |
| 400 | return gateWaits && free <= 1; | |
| 401 | } | |
| 402 | ||
| 403 | /** | |
| 404 | * Which queued check takes a slot when one frees: the one requested most | |
| 405 | * recently, so a push's own check is never stuck behind pulls that queued | |
| 406 | * hours earlier — unless something has already waited `STARVATION_MS` or | |
| 407 | * more, in which case the longest-waiting check goes first regardless of | |
| 408 | * how fresh the rest are. Checks of approved pulls (`approved`) go before | |
| 409 | * the rest, newest first among themselves (#6396): a check that is the last | |
| 410 | * thing between a pull and landing is not held up by a new pull's. Ties | |
| 411 | * (down to the same instant) fall back to id order, which is itself | |
| 412 | * creation order (`kernel/id.ts`). Among the rest, and among the approved, | |
| 413 | * the pull whose issue stands highest on the owner's ranked list (`rank`, | |
| 414 | * 1 at the top) goes first, the unranked last, newest first only as the | |
| 415 | * tie-break (#12230). | |
| 416 | */ | |
| 417 | export function pickNext( | |
| 418 | queued: readonly Run[], | |
| 419 | now: number = Date.now(), | |
| 420 | approved: (run: Run) => boolean = () => false, | |
| 421 | rank: (run: Run) => number | undefined = () => undefined, | |
| 422 | outside: (run: Run) => boolean = () => false, | |
| 423 | ): Run | undefined { | |
| 424 | // An unapproved outside contributor's check goes after everyone else's (`outside_lane.ts`). | |
| 425 | queued = ownersFirst(queued, outside); | |
| 426 | if (queued.length === 0) return undefined; | |
| 427 | const newer = (a: Run, b: Run): boolean => | |
| 428 | a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id); | |
| 429 | let oldest = queued[0]!; | |
| 430 | const place = (run: Run): number => rank(run) ?? Number.MAX_SAFE_INTEGER; | |
| 431 | const better = (a: Run, b: Run): boolean => place(a) < place(b) || (place(a) === place(b) && newer(a, b)); | |
| 432 | let newest: Run | undefined; | |
| 433 | let newestApproved: Run | undefined; | |
| 434 | for (const run of queued) { | |
| 435 | if (newer(oldest, run)) oldest = run; | |
| 436 | if (newest === undefined || better(run, newest)) newest = run; | |
| 437 | if (approved(run) && (newestApproved === undefined || better(run, newestApproved))) | |
| 438 | newestApproved = run; | |
| 439 | } | |
| 440 | if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest; | |
| 441 | return newestApproved ?? newest; | |
| 442 | } | |
| 443 | ||
| 444 | /** | |
| 445 | * Settle the checks a dead forge left running, and say so on the bus: a | |
| 446 | * check whose commit still heads an open pull is queued again for whichever | |
| 447 | * checks service is up to run, so the change keeps reading *working*. | |
| 448 | * Called on boot and on the way by whichever forge is alive. A cancelled | |
| 449 | * check is not announced, as a superseded one is not: `run.finished` reads | |
| 450 | * as a verdict on the change, and the forge dying is none. | |
| 451 | */ | |
| 452 | export function requeueAbandonedChecks(app: App, staleMs: number): number { | |
| 453 | const swept = sweepAbandonedChecks(app.db, staleMs); | |
| 454 | for (const { cancelled, again } of swept.requeued) { | |
| 455 | appendRunLog(app.config.dataDir, cancelled.id, REQUEUED_SUMMARY); | |
| 456 | announceRun(app, again, again.actorId ?? undefined); | |
| 457 | } | |
| 458 | for (const run of swept.cancelled) | |
| 459 | appendRunLog(app.config.dataDir, run.id, run.summary); | |
| 460 | for (const run of swept.failed) { | |
| 461 | appendRunLog(app.config.dataDir, run.id, run.summary); | |
| 462 | announceRun(app, run); | |
| 463 | } | |
| 464 | // On the same timer: what a dead or lapsed run left on the disk. Removing | |
| 465 | // a checkout is thousands of unlinks: off the loop, never in front of /healthz (#7688). | |
| 466 | sweepSoon(app); | |
| 467 | return swept.requeued.length + swept.failed.length + swept.cancelled.length; | |
| 468 | } | |
| 469 | ||
| 470 | /** | |
| 471 | * Remove a run's directory even when the cell left read-only files in it (a | |
| 472 | * bun cache is): `rmSync` cannot unlink inside a directory without write | |
| 473 | * permission, so permissions are restored and the removal tried again. A | |
| 474 | * removal that still fails is logged, never swallowed. | |
| 475 | */ | |
| 476 | export async function removeRunDir(path: string): Promise<void> { | |
| 477 | try { | |
| 478 | await rm(path, { recursive: true, force: true }); | |
| 479 | return; | |
| 480 | } catch { | |
| 481 | /* fall through to restore permissions */ | |
| 482 | } | |
| 483 | const open = async (p: string): Promise<void> => { | |
| 484 | try { | |
| 485 | if ((await lstat(p)).isSymbolicLink()) return; | |
| 486 | if ((await lstat(p)).isDirectory()) { | |
| 487 | await chmod(p, 0o700); | |
| 488 | for (const n of await readdir(p)) await open(join(p, n)); | |
| 489 | } | |
| 490 | } catch { | |
| 491 | /* gone, or not ours: the retry says */ | |
| 492 | } | |
| 493 | }; | |
| 494 | await open(path); | |
| 495 | try { | |
| 496 | await rm(path, { recursive: true, force: true }); | |
| 497 | } catch (error) { | |
| 498 | log.error("could not remove a run directory", { path, error: String(error) }); | |
| 499 | } | |
| 500 | } | |
| 501 | ||
| 502 | /** The sweep of the checkouts under way, if any: one at a time. */ | |
| 503 | let sweeping: Promise<void> | undefined; | |
| 504 | ||
| 505 | /** | |
| 506 | * Sweep the checkouts in the background and answer when done. A sweep | |
| 507 | * already running is the one answered, not doubled; a failure is logged. | |
| 508 | */ | |
| 509 | export function sweepSoon(app: App, now: number = Date.now()): Promise<void> { | |
| 510 | sweeping ??= sweepCheckouts(app, now) | |
| 511 | .catch((error) => log.error("could not sweep the checkouts", { error: String(error) })) | |
| 512 | .finally(() => { | |
| 513 | sweeping = undefined; | |
| 514 | }); | |
| 515 | return sweeping; | |
| 516 | } | |
| 517 | ||
| 518 | /** Where a check's checkout lives while it runs. */ | |
| 519 | const checksRoot = (dataDir: string): string => join(dataDir, "checks"); | |
| 520 | ||
| 521 | /** | |
| 522 | * What is kept under the checks' root that is no checkout: every | |
| 523 | * repository's impact map (`impact-store.ts`) and kept results | |
| 524 | * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`), | |
| 525 | * the gates' logs (`gate_log.ts`) and the last gate's results | |
| 526 | * (`gate_last.ts`). The sweep took each for a stranger's checkout and | |
| 527 | * removed it at every boot and every 30 seconds after: a map lived half a | |
| 528 | * minute, so no check was ever seeded with one (#12178). | |
| 529 | */ | |
| 530 | const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]); | |
| 531 | ||
| 532 | /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */ | |
| 533 | const GATE_DIR = /^gate-([0-9a-f]{12})$/; | |
| 534 | ||
| 535 | /** | |
| 536 | * Whether a gate's checkout, this forge's or a peer's during a roll, is younger than a gate may | |
| 537 | * run. | |
| 538 | */ | |
| 539 | function gateMayBeRunning(path: string, now: number): boolean { | |
| 540 | try { | |
| 541 | return now - lstatSync(path).mtimeMs < GATE_DEADLINE_MS; | |
| 542 | } catch { | |
| 543 | return false; | |
| 544 | } | |
| 545 | } | |
| 546 | ||
| 547 | /** | |
| 548 | * Remove the checkouts nobody is using: one whose run is not running with a | |
| 549 | * live lease, or has no run at all. A forge booting during a roll shares this | |
| 550 | * directory and database with the old one, whose checks are still running in | |
| 551 | * theirs; those are left alone. A gate writes no run: its checkout is left | |
| 552 | * while it is younger than a gate's deadline, and past that while its | |
| 553 | * `check_gate` row is open, for as long as a check may run. What the checks | |
| 554 | * keep between runs ({@link STORES}) is no checkout and is never swept. | |
| 555 | */ | |
| 556 | export async function sweepCheckouts(app: App, now: number = Date.now()): Promise<void> { | |
| 557 | const root = checksRoot(app.config.dataDir); | |
| 558 | if (!existsSync(root)) return; | |
| 559 | const since = new Date(now - CHECK_TIMEOUT_MAX * 1000); | |
| 560 | for (const name of readdirSync(root)) { | |
| 561 | if (STORES.has(name)) continue; | |
| 562 | // A gate has no run row: its checkout is spared until no gate could still be using it. | |
| 563 | if (name.startsWith("gate-") && gateMayBeRunning(join(root, name), now)) continue; | |
| 564 | // `<id>.home` is the home of run `<id>`: it lives and dies with it. | |
| 565 | const id = name.endsWith(".home") ? name.slice(0, -5) : name; | |
| 566 | const gate = GATE_DIR.exec(id); | |
| 567 | if (gate !== null) { | |
| 568 | if (!gateHolding(app.db, gate[1]!, since)) await removeRunDir(join(root, name)); | |
| 569 | continue; | |
| 570 | } | |
| 571 | const run = getRun(app.db, id); | |
| 572 | const live = | |
| 573 | run !== undefined && | |
| 574 | run.state === "running" && | |
| 575 | !leaseLapsed(run, STALE_AFTER_MS, now); | |
| 576 | if (!live) await removeRunDir(join(root, name)); | |
| 577 | } | |
| 578 | } | |
| 579 | ||
| 580 | /** | |
| 581 | * Under the free floor, give back what no check is using before taking more | |
| 582 | * (#6945): checkouts leaked by a crash sat on the disk until the next boot | |
| 583 | * while every check failed red on it. Answers whether it swept. | |
| 584 | */ | |
| 585 | export async function reclaimIfFull(app: App, now: number = Date.now()): Promise<boolean> { | |
| 586 | if (diskRecovered(app.config.dataDir)) return false; | |
| 587 | await sweepCheckouts(app, now); | |
| 588 | return true; | |
| 589 | } | |
| 590 | ||
| 591 | 65 | const handles = new WeakMap<App, ChecksHandle>(); |
| 592 | 66 | |
| 593 | 67 | /** The checks service of this forge, if one was started. */ |
| @@ -604,1273 +78,36 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 604 | 78 | // by the next check, not the next boot. `concurrency` is the boot default. |
| 605 | 79 | const booted = options.concurrency ?? 1; |
| 606 | 80 | noteBooted(app.db, "checks.concurrency", String(booted)); |
| 607 | const capacity = (): number => | |
| 608 | capacityCount(app.db, "checks.concurrency", booted); | |
| 609 | const gate = options.gate ?? new Gate(capacity()); | |
| 610 | 81 | const placement = options.placement ?? placementFromEnv(); |
| 611 | 82 | // Jobs a forge that died left on the runners' desk: their checks are queued again elsewhere. |
| 612 | 83 | if (placement.mode === "runner") sweepJobs(app.db); |
| 613 | /** | |
| 614 | * Full gates on land in flight: not slots in the queue, but stopped with | |
| 615 | * it — after the drain, as a check is. By commit, so a merge queue that | |
| 616 | * carries on adopts one rather than gate the same tip again (#6611). | |
| 617 | */ | |
| 618 | const gates = new Map<AbortController, { commitOid: string; done: Promise<GateDone> }>(); | |
| 619 | const inFlight = new Map< | |
| 620 | string, | |
| 621 | { promise: Promise<void>; abort: AbortController } | |
| 622 | >(); | |
| 623 | /** Checks already told why they wait, so the log says it once. */ | |
| 624 | const waiting = new Set<string>(); | |
| 625 | /** | |
| 626 | * Queued checks a full disk holds that have said so once (#11799); apart from `waiting`, so | |
| 627 | * neither line hides the other. | |
| 628 | */ | |
| 629 | const diskWaiting = new Set<string>(); | |
| 630 | const memoryWaiting = new Set<string>(); | |
| 631 | /** The verification-lane run in flight, which holds no check slot. */ | |
| 632 | let verifying: string | undefined; | |
| 633 | const slotted = (): number => inFlight.size - (verifying === undefined ? 0 : 1); | |
| 634 | const settle = options.landSettleMs ?? 0; | |
| 635 | let stopped = false; | |
| 636 | /** While above 0, a batch of requests is being queued and `schedule` waits for its end. */ | |
| 637 | let holding = 0; | |
| 84 | const lane: LaneState = { | |
| 85 | app, | |
| 86 | options, | |
| 87 | gate: options.gate ?? new Gate(capacityCount(app.db, "checks.concurrency", booted)), | |
| 88 | placement, | |
| 89 | gates: new Map(), | |
| 90 | inFlight: new Map(), | |
| 91 | waiting: new Set(), | |
| 92 | diskWaiting: new Set(), | |
| 93 | memoryWaiting: new Set(), | |
| 94 | verifying: undefined, | |
| 95 | settle: options.landSettleMs ?? 0, | |
| 96 | stopped: false, | |
| 97 | holding: 0, | |
| 98 | pullHold: holdClock(options.holdMaxMs), | |
| 99 | runner: makeLocalRunner(app, (x) => (options.runTier ?? ((y) => runTier(lane, y)))(x)), | |
| 100 | schedule: () => schedule(lane), | |
| 101 | request: (...ask) => request(lane, ...ask), | |
| 102 | }; | |
| 638 | 103 | |
| 639 | 104 | sweepSoon(app); |
| 640 | 105 | |
| 641 | async function request( | |
| 642 | repo: Repo, | |
| 643 | env: string, | |
| 644 | commitOid: string, | |
| 645 | reason: string, | |
| 646 | issueId?: string, | |
| 647 | actorId?: string, | |
| 648 | notBefore?: string, | |
| 649 | ): Promise<Run | undefined> { | |
| 650 | // A suspended or removed owner's code is not run, not even to check it. | |
| 651 | if (stopped || !accountActive(app.db, repo.ownerId)) return undefined; | |
| 652 | // A guest's fork runs nothing: its code runs only as a pull a person approved. | |
| 653 | if (isContributionFork(app.db, repo.id)) return undefined; | |
| 654 | // A pull from someone who cannot write runs nothing until a writer | |
| 655 | // approves this exact head (`core/pull_trust.ts`). | |
| 656 | if (!issueMayRun(app.db, repo, issueId, commitOid)) { | |
| 657 | log.info("check waits for a writer's approval", { | |
| 658 | repo: `${repo.ownerName}/${repo.name}`, | |
| 659 | env, | |
| 660 | commit: commitOid.slice(0, 7), | |
| 661 | }); | |
| 662 | return undefined; | |
| 663 | } | |
| 664 | // A sealed pull's commit is where its work is, never in the base (`pullDir`). | |
| 665 | const bare = await runDir(app.db, app.reposRoot, repo, issueId); | |
| 666 | const [forgeJson, packageJson] = await Promise.all([ | |
| 667 | readFileAt(bare, commitOid, "forge.json"), | |
| 668 | readFileAt(bare, commitOid, "package.json"), | |
| 669 | ]); | |
| 670 | const declared = parseCheck(forgeJson, packageJson); | |
| 671 | if (declared.ok && declared.value === undefined) return undefined; | |
| 672 | // Main is verified after landing only where it says how (#7786). | |
| 673 | if (env === VERIFY_ENV && (!declared.ok || declared.value?.verify == null)) return undefined; | |
| 674 | // One check per environment at a time: a newer commit supersedes what | |
| 675 | // is still waiting, and the same commit is not checked twice. Looked for | |
| 676 | // and written in one write transaction, so a second forge process over | |
| 677 | // this database (a roll) cannot queue the same check beside it. | |
| 678 | const superseded: Run[] = []; | |
| 679 | const { run, fresh } = transact(app.db, () => { | |
| 680 | for (const active of listActive(app.db, "check")) { | |
| 681 | if (active.repoId !== repo.id || active.env !== env) continue; | |
| 682 | // A person asking for a check a red main holds queues it as theirs, | |
| 683 | // which runs (`checks/red_main.ts`); any other repeat is the same check. | |
| 684 | if ( | |
| 685 | active.commitOid === commitOid && | |
| 686 | !(reason === ASKED && active.state === "queued" && active.reason !== ASKED) | |
| 687 | ) | |
| 688 | return { run: active, fresh: false }; | |
| 689 | // A pull's running check is for a head nobody will land either: | |
| 690 | // it is stopped too, below, once the row says so. | |
| 691 | if (active.state === "queued" || issueId !== undefined) { | |
| 692 | const gone = cancelRun( | |
| 693 | app.db, | |
| 694 | active.id, | |
| 695 | `superseded by ${commitOid.slice(0, 7)}`, | |
| 696 | ); | |
| 697 | if (gone && active.state === "running") superseded.push(gone); | |
| 698 | } | |
| 699 | } | |
| 700 | const created = createRun(app.db, { | |
| 701 | repoId: repo.id, | |
| 702 | kind: "check", | |
| 703 | env, | |
| 704 | commitOid, | |
| 705 | reason, | |
| 706 | ...(issueId === undefined ? {} : { issueId }), | |
| 707 | ...(actorId === undefined ? {} : { actorId }), | |
| 708 | ...(notBefore === undefined ? {} : { notBefore }), | |
| 709 | }); | |
| 710 | return { run: created, fresh: true }; | |
| 711 | }); | |
| 712 | // The row is cancelled, so the check's own end finishes nothing and | |
| 713 | // announces nothing; the abort is what frees its slot and its checkout. | |
| 714 | for (const old of superseded) inFlight.get(old.id)?.abort.abort(); | |
| 715 | if (!fresh) return run; | |
| 716 | announceRun(app, run, actorId); | |
| 717 | schedule(); | |
| 718 | return run; | |
| 719 | } | |
| 720 | ||
| 721 | /** Run a queued check here, whichever forge queued it. */ | |
| 722 | function start(run: Run, repo: Repo): void { | |
| 723 | if (inFlight.has(run.id)) return; | |
| 724 | waiting.delete(run.id); | |
| 725 | diskWaiting.delete(run.id); | |
| 726 | memoryWaiting.delete(run.id); | |
| 727 | const abort = new AbortController(); | |
| 728 | const low = inVerifyLane(run.env); | |
| 729 | if (low) verifying = run.id; | |
| 730 | // A person's cancel (`cancelRepoRun`) reaches it through this. | |
| 731 | trackRun(run.id, abort); | |
| 732 | workBegins(app.db, run.id); | |
| 733 | const promise = execute(run, repo, abort.signal, !low) | |
| 734 | .catch((error) => | |
| 735 | log.error("check failed to execute", { | |
| 736 | run: run.id, | |
| 737 | error: String(error), | |
| 738 | }), | |
| 739 | ) | |
| 740 | .finally(() => { | |
| 741 | untrackRun(run.id, abort); | |
| 742 | // Every process the check started has ended: a cancel waiting on | |
| 743 | // that is heard now (#13291). | |
| 744 | workEnds(app.db, run.id); | |
| 745 | inFlight.delete(run.id); | |
| 746 | if (verifying === run.id) verifying = undefined; | |
| 747 | // A pull's green that ran while its base moved is of the old tip: asked again once. | |
| 748 | if (run.issueId !== null && !stopped) | |
| 749 | void recheckFinished(run.id, repo).catch((error) => | |
| 750 | log.error("check could not be asked again after the base moved", { run: run.id, error: String(error) }), | |
| 751 | ); | |
| 752 | // The slot this run held is free again: give it to whichever queued | |
| 753 | // check now deserves it most, not whoever asked first. | |
| 754 | schedule(); | |
| 755 | }); | |
| 756 | inFlight.set(run.id, { promise, abort }); | |
| 757 | } | |
| 758 | ||
| 759 | /** | |
| 760 | * An account at its concurrent-checks limit (`settings/quotas.ts`) keeps | |
| 761 | * the check queued, the count in its log once, until one of its own ends. | |
| 762 | */ | |
| 763 | function atLimit(run: Run): boolean { | |
| 764 | const owner = getRepoById(app.db, run.repoId)?.ownerId; | |
| 765 | if (owner === undefined) return false; | |
| 766 | const full = runningLimitSentence(app.db, "checks", "concurrent checks", owner, () => | |
| 767 | runningRuns(app.db, "check", owner, inFlight.keys()), | |
| 768 | ); | |
| 769 | if (full === undefined) return false; | |
| 770 | if (!waiting.has(run.id)) { | |
| 771 | waiting.add(run.id); | |
| 772 | appendRunLog(app.config.dataDir, run.id, `queued, not started: ${full}`); | |
| 773 | } | |
| 774 | return true; | |
| 775 | } | |
| 776 | ||
| 777 | /** | |
| 778 | * Fill every free slot with the queued check `pickNext` says deserves it — | |
| 779 | * one the forge before this one never reached, one a forge stopping | |
| 780 | * beside this one handed back, or simply the newest push once a slot | |
| 781 | * opens up. Claiming is what decides between two forges, so calling this | |
| 782 | * from both never runs the same check twice. | |
| 783 | */ | |
| 784 | /** `pickNext` among `queued`, passing over any whose account is at its limit. */ | |
| 785 | function startable(queued: readonly Run[]): Run | undefined { | |
| 786 | // At 80% used no check starts (#11799): they stay queued, one line each, and the tick asks | |
| 787 | // again. | |
| 788 | const full = usedRefusal(app.config.dataDir); | |
| 789 | if (full !== undefined) { | |
| 790 | for (const run of queued) | |
| 791 | if (!diskWaiting.has(run.id)) { | |
| 792 | diskWaiting.add(run.id); | |
| 793 | appendRunLog(app.config.dataDir, run.id, `queued, ${full}`); | |
| 794 | } | |
| 795 | return undefined; | |
| 796 | } | |
| 797 | // A child near its memory limit runs one check at a time (`runs/memory_hold.ts`). | |
| 798 | const tight = memoryHold(slotted(), options.pressure); | |
| 799 | if (tight !== undefined) { | |
| 800 | for (const run of queued) | |
| 801 | if (!memoryWaiting.has(run.id)) { | |
| 802 | memoryWaiting.add(run.id); | |
| 803 | appendRunLog(app.config.dataDir, run.id, `queued, ${tight}`); | |
| 804 | } | |
| 805 | return undefined; | |
| 806 | } | |
| 807 | let rest = queued; | |
| 808 | const isApproved = approvedOnce(app.db); | |
| 809 | const rankOf = rankOnce(app.db); | |
| 810 | const isOutside = outsideOnce(app.db); | |
| 811 | let next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside); | |
| 812 | while (next !== undefined && atLimit(next)) { | |
| 813 | const held = next; | |
| 814 | rest = rest.filter((run) => run !== held); | |
| 815 | next = pickNext(rest, Date.now(), isApproved, rankOf, isOutside); | |
| 816 | } | |
| 817 | return next; | |
| 818 | } | |
| 819 | ||
| 820 | const pullHold = holdClock(options.holdMaxMs); | |
| 821 | ||
| 822 | function schedule(): void { | |
| 823 | if (stopped || holding > 0) return; | |
| 824 | const limit = capacity(); | |
| 825 | if (options.gate === undefined) gate.resize(limit); | |
| 826 | // On one CPU a pull's check waits while a crew run works, for at most | |
| 827 | // `holdMaxMs`; the verify lane, a rebase's check and a main check are the | |
| 828 | // pipeline and never wait (#11599). | |
| 829 | const crewHolds = crewHoldsCore((options.cpus ?? forgeCpus)(), options.activeCrewRuns ?? (() => activeCrewRuns(app.db))); | |
| 830 | // The clock counts one continuous hold: it starts again once no crew run | |
| 831 | // holds, and forgets checks that are no longer queued. | |
| 832 | if (!crewHolds) pullHold.reset(); | |
| 833 | else pullHold.keep(new Set(listActive(app.db, "check").filter((run) => run.state === "queued").map((run) => run.id))); | |
| 834 | startVerify(); | |
| 835 | // Every merge-queue gate in flight holds a slot, a line's first too (#6376). | |
| 836 | while (slotted() < limit - gatesInFlight()) { | |
| 837 | // A pull's check waits while its main is red (`checks/red_main.ts`). | |
| 838 | const queued = unheld( | |
| 839 | listActive(app.db, "check").filter( | |
| 840 | (run) => | |
| 841 | run.state === "queued" && | |
| 842 | !inFlight.has(run.id) && | |
| 843 | !inVerifyLane(run.env) && | |
| 844 | (run.notBefore === null || Date.parse(run.notBefore) <= Date.now()), | |
| 845 | ), | |
| 846 | (repoId) => redMain(app.db, repoId), | |
| 847 | options.redHoldMs ?? 0, | |
| 848 | ); | |
| 849 | // The limit is asked at the point of starting, one pick at a time: | |
| 850 | // a started check is in flight before the next is asked about, so a | |
| 851 | // burst of one account's checks cannot pass together. A rebase's | |
| 852 | // check goes before any other (`finishing_first.ts`, #6140). | |
| 853 | const { rebases, rest: others } = rebasesFirst(app.db, queued); | |
| 854 | // While a gate is about to start, the last free slot is its: a pull's | |
| 855 | // check waits for it, a rebase's and a main check do not. | |
| 856 | const held = holdsLastSlot(limit - gatesInFlight() - slotted(), gateQueued()); | |
| 857 | const waiting = crewHolds | |
| 858 | ? others.filter((run) => run.issueId === null || !pullHold.held(run.id, (options.holdNow ?? Date.now)())) | |
| 859 | : others; | |
| 860 | const next = startable(rebases) ?? startable(held ? waiting.filter((run) => run.issueId === null) : waiting); | |
| 861 | if (!next) return; | |
| 862 | const repo = getRepoById(app.db, next.repoId); | |
| 863 | if (!repo) return; | |
| 864 | pullHold.release(next.id); | |
| 865 | start(next, repo); | |
| 866 | } | |
| 867 | } | |
| 868 | ||
| 869 | /** | |
| 870 | * The verification lane (#7786): one run at a time, forge-wide, beside | |
| 871 | * the check slots rather than in one, so no landing and no pull's check | |
| 872 | * ever waits for it. A bisection step goes before a new verification. | |
| 873 | */ | |
| 874 | function startVerify(): void { | |
| 875 | if (verifying !== undefined) return; | |
| 876 | const queued = listActive(app.db, "check").filter( | |
| 877 | (run) => run.state === "queued" && inVerifyLane(run.env) && !inFlight.has(run.id), | |
| 878 | ); | |
| 879 | const next = | |
| 880 | queued.find((run) => bisectOrigin(run.reason) !== undefined) ?? queued[0]; | |
| 881 | if (next === undefined) return; | |
| 882 | const repo = getRepoById(app.db, next.repoId); | |
| 883 | if (repo !== undefined) start(next, repo); | |
| 884 | } | |
| 885 | ||
| 886 | /** Every verification due now: main moved since the last, and that one is old enough. */ | |
| 887 | async function askVerifications(now = Date.now()): Promise<Run[]> { | |
| 888 | const everyMs = options.verifyEveryMs ?? VERIFY_EVERY_MS; | |
| 889 | const since = new Date(now - 24 * 3_600_000).toISOString(); | |
| 890 | // The newest default-branch check of each repository that had one today. | |
| 891 | const tips = app.db | |
| 892 | .query<{ repo_id: string; commit_oid: string }, [string]>( | |
| 893 | `SELECT r.repo_id, r.commit_oid FROM run r JOIN repo ON repo.id = r.repo_id | |
| 894 | WHERE r.id IN (SELECT MAX(x.id) FROM run x JOIN repo y ON y.id = x.repo_id | |
| 895 | WHERE x.kind = 'check' AND x.issue_id IS NULL | |
| 896 | AND x.env = y.default_branch AND x.created_at >= ?1 | |
| 897 | GROUP BY x.repo_id)`, | |
| 898 | ) | |
| 899 | .all(since); | |
| 900 | const asked: Run[] = []; | |
| 901 | for (const tip of tips) { | |
| 902 | const last = app.db | |
| 903 | .query<{ commit_oid: string; created_at: string }, [string]>( | |
| 904 | `SELECT commit_oid, created_at FROM run | |
| 905 | WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}' | |
| 906 | ORDER BY id DESC LIMIT 1`, | |
| 907 | ) | |
| 908 | .get(tip.repo_id); | |
| 909 | const active = listActive(app.db, "check").some( | |
| 910 | (run) => run.repoId === tip.repo_id && inVerifyLane(run.env), | |
| 911 | ); | |
| 912 | const due = verifyDue({ | |
| 913 | tip: tip.commit_oid, | |
| 914 | last: last === null ? undefined : { commitOid: last.commit_oid, createdAt: last.created_at }, | |
| 915 | active, | |
| 916 | now, | |
| 917 | everyMs, | |
| 918 | }); | |
| 919 | if (!due) continue; | |
| 920 | const repo = getRepoById(app.db, tip.repo_id); | |
| 921 | if (repo === undefined) continue; | |
| 922 | const run = await request(repo, VERIFY_ENV, tip.commit_oid, VERIFY_REASON); | |
| 923 | if (run !== undefined) asked.push(run); | |
| 924 | } | |
| 925 | return asked; | |
| 926 | } | |
| 927 | ||
| 928 | /** | |
| 929 | * An open pull whose head was never checked — the event that should have | |
| 930 | * asked landed while a forge was on its way out, or before this forge | |
| 931 | * existed — is asked for on boot. A pull that stays unchecked would never | |
| 932 | * land on green, however green it is. | |
| 933 | */ | |
| 934 | async function catchUp(): Promise<void> { | |
| 935 | const heads = app.db | |
| 936 | .query< | |
| 937 | { repo_id: string; issue_id: string; number: number; head_oid: string }, | |
| 938 | [] | |
| 939 | >( | |
| 940 | `SELECT issue.repo_id, issue.id AS issue_id, issue.number, pull.head_oid | |
| 941 | FROM pull JOIN issue ON issue.id = pull.issue_id | |
| 942 | WHERE issue.state = 'open' AND pull.merged_at IS NULL | |
| 943 | ORDER BY issue.created_at DESC LIMIT 200`, | |
| 944 | ) | |
| 945 | .all(); | |
| 946 | for (const head of heads) { | |
| 947 | if (head.head_oid === "") continue; | |
| 948 | if (checksForHead(app.db, head.issue_id, head.head_oid).length > 0) | |
| 949 | continue; | |
| 950 | const repo = getRepoById(app.db, head.repo_id); | |
| 951 | if (!repo) continue; | |
| 952 | try { | |
| 953 | await request( | |
| 954 | repo, | |
| 955 | `pull-${head.number}`, | |
| 956 | head.head_oid, | |
| 957 | "boot", | |
| 958 | head.issue_id, | |
| 959 | ); | |
| 960 | } catch (error) { | |
| 961 | log.error("check could not be requested on boot", { | |
| 962 | pull: head.number, | |
| 963 | error: String(error), | |
| 964 | }); | |
| 965 | } | |
| 966 | } | |
| 967 | } | |
| 968 | ||
| 969 | async function execute( | |
| 970 | run: Run, | |
| 971 | repo: Repo, | |
| 972 | signal: AbortSignal, | |
| 973 | slot = true, | |
| 974 | ): Promise<void> { | |
| 975 | // The verification lane takes no slot of the gate (`startVerify`). | |
| 976 | const release = slot ? await gate.acquire() : () => {}; | |
| 977 | try { | |
| 978 | // A slot that opens while this forge is stopping is not taken: the run | |
| 979 | // stays queued for the forge that carries on. | |
| 980 | if (stopped) return; | |
| 981 | // Asked again as it starts: a check queued before a rule changed, or | |
| 982 | // handed back by a forge that stopped, runs only what may run now. | |
| 983 | if (!issueMayRun(app.db, repo, run.issueId, run.commitOid)) { | |
| 984 | cancelRun(app.db, run.id, "waiting for a writer to approve this head"); | |
| 985 | return; | |
| 986 | } | |
| 987 | const claimed = claimRun(app.db, run.id); | |
| 988 | if (!claimed) return; | |
| 989 | const lease = holdLease(app.db, run.id); | |
| 990 | announceRun(app, claimed); | |
| 991 | const write = (line: string): void => { | |
| 992 | appendRunLog(app.config.dataDir, run.id, line); | |
| 993 | announceShard(app, run, line); | |
| 994 | }; | |
| 995 | if (options.label !== undefined) | |
| 996 | write(`forge ${options.label} — checked in its parent's lane`); | |
| 997 | const started = Date.now(); | |
| 998 | // The budget: past it the check is aborted, whatever it is doing. | |
| 999 | const limit = checkLimitMs(run.env, { | |
| 1000 | gate: options.budgetMs ?? BUDGETS.gate, | |
| 1001 | verify: options.verifyBudgetMs ?? BUDGETS.verify, | |
| 1002 | review: BUDGETS.review, | |
| 1003 | }); | |
| 1004 | const budget = new AbortController(); | |
| 1005 | let overran = false; | |
| 1006 | const forward = (): void => budget.abort(); | |
| 1007 | signal.addEventListener("abort", forward, { once: true }); | |
| 1008 | const clock = setTimeout(() => { | |
| 1009 | overran = true; | |
| 1010 | budget.abort(); | |
| 1011 | }, limit); | |
| 1012 | const outcome = await check(claimed, repo, write, budget.signal).finally( | |
| 1013 | () => { | |
| 1014 | clearTimeout(clock); | |
| 1015 | signal.removeEventListener("abort", forward); | |
| 1016 | lease(); | |
| 1017 | }, | |
| 1018 | ); | |
| 1019 | // Where a check cut off here is asked again. A verification is of the | |
| 1020 | // default branch as it stands: it goes back at the branch's newest | |
| 1021 | // checked commit, the one `askVerifications` would ask of. Left at the | |
| 1022 | // commit it began on, each restart and each overrun began it there | |
| 1023 | // again, no other was asked while it was active, and main went | |
| 1024 | // unverified from 01:48Z to 03:37Z on 2026-11-14 (#12178). | |
| 1025 | const againAt = (): string | undefined => | |
| 1026 | run.env === VERIFY_ENV | |
| 1027 | ? app.db | |
| 1028 | .query<{ commit_oid: string }, [string, string]>( | |
| 1029 | `SELECT commit_oid FROM run WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL | |
| 1030 | AND env = ?2 ORDER BY id DESC LIMIT 1`, | |
| 1031 | ) | |
| 1032 | .get(repo.id, repo.defaultBranch)?.commit_oid || undefined | |
| 1033 | : undefined; | |
| 1034 | if (overran) { | |
| 1035 | const summary = budgetSummary( | |
| 1036 | "a full gate", | |
| 1037 | Date.now() - started, | |
| 1038 | limit, | |
| 1039 | ); | |
| 1040 | write(summary); | |
| 1041 | if (!overranBefore(app.db, claimed)) { | |
| 1042 | const moved = requeueCheck(app.db, run.id, undefined, summary, againAt()); | |
| 1043 | if (moved) { | |
| 1044 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1045 | return; | |
| 1046 | } | |
| 1047 | } | |
| 1048 | // The second overrun: a failure, and the card the vitals carry. | |
| 1049 | const failed = finishRun(app.db, run.id, { | |
| 1050 | state: "failed", | |
| 1051 | summary, | |
| 1052 | }); | |
| 1053 | if (failed) announceRun(app, failed); | |
| 1054 | return; | |
| 1055 | } | |
| 1056 | // Ended from outside while it ran: its pull landed or closed, and | |
| 1057 | // the row says so. That is not this check failing. | |
| 1058 | // A person's cancel ends the log with who cancelled it (#11229). | |
| 1059 | const row = getRun(app.db, run.id); | |
| 1060 | if (!outcome.ok && row?.state === "cancelled") { | |
| 1061 | write(cancelledLine(row) ?? `superseded: ${row.summary}`); | |
| 1062 | return; | |
| 1063 | } | |
| 1064 | if (!outcome.ok && stopped && signal.aborted) { | |
| 1065 | // Cut off by this forge stopping, not by the check: the same check | |
| 1066 | // goes back on the queue, and the forge that carries on runs it. | |
| 1067 | // Past the release ceiling the shutdown is what this check hangs or | |
| 1068 | // slows: it fails, saying so, instead of riding every release. | |
| 1069 | if (chainCutOffs(app.db, row ?? claimed, RELEASE_SUMMARY) >= CHECK_RELEASE_LIMIT) { | |
| 1070 | const summary = `the forge was stopped under this check ${CHECK_RELEASE_LIMIT} times; not trying again`; | |
| 1071 | write(summary); | |
| 1072 | const failed = finishRun(app.db, run.id, { state: "failed", summary }); | |
| 1073 | if (failed) announceRun(app, failed); | |
| 1074 | return; | |
| 1075 | } | |
| 1076 | const moved = requeueCheck(app.db, run.id, undefined, RELEASE_SUMMARY, againAt()); | |
| 1077 | if (!moved) return; | |
| 1078 | write(RELEASE_SUMMARY); | |
| 1079 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1080 | return; | |
| 1081 | } | |
| 1082 | // The cluster lost the Job (evicted, node gone): interrupted, not | |
| 1083 | // failed, and queued once more as after a restart; the second loss fails. | |
| 1084 | if (!outcome.ok && outcome.error === JOB_VANISHED && !vanishedBefore(app.db, claimed)) { | |
| 1085 | const moved = requeueCheck(app.db, run.id, undefined, VANISHED_SUMMARY); | |
| 1086 | if (moved) { | |
| 1087 | write(VANISHED_SUMMARY); | |
| 1088 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1089 | return; | |
| 1090 | } | |
| 1091 | } | |
| 1092 | // The install failed twice on what the machine did, not the code: the | |
| 1093 | // check is queued once more and nothing reads it as red (#5093). | |
| 1094 | if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) { | |
| 1095 | const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY); | |
| 1096 | if (moved) { | |
| 1097 | write(INFRA_SUMMARY); | |
| 1098 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1099 | return; | |
| 1100 | } | |
| 1101 | } | |
| 1102 | // The disk was 80% used when the step would have started (#11799): | |
| 1103 | // the machine's wait. Queued again, not red; the pick holds it until there is room. | |
| 1104 | if (!outcome.ok && isUsedRefusal(outcome.error)) { | |
| 1105 | const moved = requeueCheck(app.db, run.id, undefined, DISK_USED_SUMMARY); | |
| 1106 | if (moved) { | |
| 1107 | write(DISK_USED_SUMMARY); | |
| 1108 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1109 | return; | |
| 1110 | } | |
| 1111 | } | |
| 1112 | // Stopped for CPU by the cell's watch: the machine's, not the pull's. | |
| 1113 | // Queued once more on its own, and no builder hears check_failed for | |
| 1114 | // it; a second stop in a row is reported as any red is (#11617). | |
| 1115 | if (!outcome.ok && outcome.cpuStop !== undefined) { | |
| 1116 | const moved = requeueCpuStop(app.db, claimed, outcome.cpuStop); | |
| 1117 | if (moved) { | |
| 1118 | write(CPU_STOP_SUMMARY); | |
| 1119 | announceRun(app, moved.again, run.actorId ?? undefined); | |
| 1120 | return; | |
| 1121 | } | |
| 1122 | } | |
| 1123 | const seconds = Math.round((Date.now() - started) / 1000); | |
| 1124 | const done = finishRun(app.db, run.id, { | |
| 1125 | state: outcome.ok ? "ok" : "failed", | |
| 1126 | summary: outcome.ok | |
| 1127 | ? outcome.proven === undefined | |
| 1128 | ? `check passed in ${seconds}s${outcome.cleared === undefined ? "" : `; ${outcome.cleared}`}` | |
| 1129 | : `already proven green by ${outcome.proven}` | |
| 1130 | : `check failed: ${outcome.error}`, | |
| 1131 | ...(outcome.ok | |
| 1132 | ? {} | |
| 1133 | : { checkDetail: withDisk(outcome.detail, app.config.dataDir) }), | |
| 1134 | }); | |
| 1135 | write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`); | |
| 1136 | // The pages the gate chose to picture are said on the pull (#13373), worked out by the | |
| 1137 | // forge from the pull's commits, never read from the log the branch's code writes. | |
| 1138 | if (done) void noteLookPages(app, repo, done).catch((error) => log.error("look pages note failed", { error: String(error) })); | |
| 1139 | if (done) announceRun(app, done); | |
| 1140 | if (done && outcome.ok) releaseOnGreen(app, done); | |
| 1141 | if (!outcome.ok && run.issueId === null && run.env === repo.defaultBranch) | |
| 1142 | proposeRevert(app, repo.id).catch((error) => | |
| 1143 | log.error("revert proposal failed", { error: String(error) }), | |
| 1144 | ); | |
| 1145 | // A red verification after landing names its culprit (#7786). | |
| 1146 | if (done && run.env === VERIFY_ENV) { | |
| 1147 | reportRed(app, repo, done).forEach(write); | |
| 1148 | // A file the run printed passing closes its open report issue (#9524). | |
| 1149 | reportGreen(app, repo, done, outcome.passed ?? []).forEach(write); | |
| 1150 | // A green that ran every file a culprit's issue names closes it (#11395). | |
| 1151 | closeOnGreen(app, repo, done, outcome.passed ?? []).forEach(write); | |
| 1152 | } | |
| 1153 | if (done && inVerifyLane(run.env)) | |
| 1154 | await afterVerify(app, repo, done, (r, env, oid, reason) => request(r, env, oid, reason)) | |
| 1155 | .then((lines) => lines.forEach(write)) | |
| 1156 | .catch((error) => write(`culprit not named: ${String(error)}`)); | |
| 1157 | } finally { | |
| 1158 | release(); | |
| 1159 | } | |
| 1160 | } | |
| 1161 | ||
| 1162 | const localRunner = makeLocalRunner(app, (x) => (options.runTier ?? runTier)(x)); | |
| 1163 | ||
| 1164 | /** Describe a claimed check as data and hand it to the runner. */ | |
| 1165 | async function check( | |
| 1166 | run: Run, | |
| 1167 | repo: Repo, | |
| 1168 | write: (line: string) => void, | |
| 1169 | signal: AbortSignal, | |
| 1170 | ): Promise<CheckOutcome> { | |
| 1171 | const issue = run.issueId === null ? undefined : getIssueById(app.db, run.issueId); | |
| 1172 | const lane = inVerifyLane(run.env); | |
| 1173 | // A bisection step runs only the red's failed files, with the red's own command: an older | |
| 1174 | // commit may not say how to verify (#7786). | |
| 1175 | const origin = | |
| 1176 | lane && run.env !== VERIFY_ENV ? getRun(app.db, bisectOrigin(run.reason) ?? "") : undefined; | |
| 1177 | const job = describeJob(run, { | |
| 1178 | defaultBranch: repo.defaultBranch, | |
| 1179 | verifyEnv: VERIFY_ENV, | |
| 1180 | inVerifyLane: lane, | |
| 1181 | pulled: issue?.kind === "pull", | |
| 1182 | origin: | |
| 1183 | origin === undefined | |
| 1184 | ? undefined | |
| 1185 | : { files: origin.checkDetail?.files ?? [], commitOid: origin.commitOid }, | |
| 1186 | }); | |
| 1187 | if ("error" in job) return { ok: false, error: job.error }; | |
| 1188 | return localRunner.run(job, { write, signal }); | |
| 1189 | } | |
| 1190 | ||
| 1191 | async function runTier( | |
| 1192 | x: Execution, | |
| 1193 | ): Promise< | |
| 1194 | | { | |
| 1195 | ok: true; | |
| 1196 | proven?: string; | |
| 1197 | passed?: readonly string[]; | |
| 1198 | /** A red it cleared as main's (#10304): `base_red.ts`'s words, kept in the summary. */ | |
| 1199 | cleared?: string; | |
| 1200 | } | |
| 1201 | | { | |
| 1202 | ok: false; | |
| 1203 | error: string; | |
| 1204 | detail?: CheckDetail | undefined; | |
| 1205 | /** A whole verification: the files its log ran to the end green (#9524). */ | |
| 1206 | passed?: readonly string[]; | |
| 1207 | infrastructure?: boolean; | |
| 1208 | /** The check step's error when the cell's watch stopped it for CPU (#11617). */ | |
| 1209 | cpuStop?: string; | |
| 1210 | /** Where it stopped red (#5276); none is before any step ran. */ | |
| 1211 | stage?: GateStage; | |
| 1212 | /** The check step was ended by a signal from outside: no verdict (#6722). */ | |
| 1213 | killed?: boolean; | |
| 1214 | } | |
| 1215 | > { | |
| 1216 | const { bare, repo, write, signal } = x; | |
| 1217 | // A pull's fast check is of the pull as it would land: its head merged | |
| 1218 | // into the base's tip. The gate comes from that tree, so a pull forked | |
| 1219 | // before the base learned something is judged with it. | |
| 1220 | let source = x.commitOid; | |
| 1221 | let tip: string | undefined; | |
| 1222 | // The commit the checked tree builds on: the base tip for a merge made | |
| 1223 | // now, the base a combined commit names for a kept fork's pull. The fast | |
| 1224 | // tier diffs against it, so a combined built before main moved is not | |
| 1225 | // charged with main's later changes. | |
| 1226 | let builtOnBase: string | undefined; | |
| 1227 | let mergeLine = ""; | |
| 1228 | if (x.tier === "fast" && x.pull !== undefined) { | |
| 1229 | tip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); | |
| 1230 | if (tip === undefined) | |
| 1231 | return { ok: false, error: `the base ${x.pull.baseBranch} could not be found` }; | |
| 1232 | // A kept fork's pull is checked on its combined commit (#9932): the | |
| 1233 | // tree that lands, built on the base tip it names (`combined/N`). | |
| 1234 | const combined = x.pull.crossRepo | |
| 1235 | ? await readCombined(bare, x.pull.number, x.commitOid) | |
| 1236 | : undefined; | |
| 1237 | if (combined !== undefined && combined.oid !== x.commitOid) { | |
| 1238 | source = combined.oid; | |
| 1239 | const builtOn = combined.baseOid === "" ? tip : combined.baseOid; | |
| 1240 | if (x.runId !== undefined) { | |
| 1241 | setRunBase(app.db, x.runId, builtOn); | |
| 1242 | setRunCombined(app.db, x.runId, combined.oid); | |
| 1243 | } | |
| 1244 | builtOnBase = builtOn; | |
| 1245 | mergeLine = `checked as combined with ${x.pull.baseBranch} at ${builtOn.slice(0, 7)} (${combined.oid.slice(0, 7)})`; | |
| 1246 | } else { | |
| 1247 | const merged = await mergeForCheck(bare, x.pull.baseBranch, tip, x.commitOid); | |
| 1248 | if (!merged.ok) return { ok: false, error: merged.error }; | |
| 1249 | source = merged.commit; | |
| 1250 | if (x.runId !== undefined) setRunBase(app.db, x.runId, tip); | |
| 1251 | builtOnBase = tip; | |
| 1252 | mergeLine = merged.line; | |
| 1253 | } | |
| 1254 | } | |
| 1255 | const run = { id: x.id, commitOid: source }; | |
| 1256 | const [forgeJson, packageJson] = await Promise.all([ | |
| 1257 | readFileAt(bare, run.commitOid, "forge.json"), | |
| 1258 | readFileAt(bare, run.commitOid, "package.json"), | |
| 1259 | ]); | |
| 1260 | const declared = parseCheck(forgeJson, packageJson); | |
| 1261 | if (!declared.ok) | |
| 1262 | return { | |
| 1263 | ok: false, | |
| 1264 | error: | |
| 1265 | declared.error.kind === "invalid" | |
| 1266 | ? declared.error.message | |
| 1267 | : "this commit declares no check", | |
| 1268 | }; | |
| 1269 | if (declared.value === undefined) | |
| 1270 | return { ok: false, error: "this commit declares no check" }; | |
| 1271 | // A pull does not choose its own gate: the commands and the timeout come | |
| 1272 | // from the base branch's `forge.json` where the pull left it, and so do | |
| 1273 | // the hosts its steps may reach and the limits its cell is given | |
| 1274 | // (`checks/held.ts`). The branch's own supplies only its install step. | |
| 1275 | let manifest = declared.value; | |
| 1276 | let baseTip: string | undefined; | |
| 1277 | /** | |
| 1278 | * An outside agent's pull nobody has approved: its code is checked, inside a cell or not at | |
| 1279 | * all. | |
| 1280 | */ | |
| 1281 | let outside = false; | |
| 1282 | if (x.pull !== undefined) { | |
| 1283 | baseTip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); | |
| 1284 | if (baseTip === undefined) | |
| 1285 | return { ok: false, error: "the pull's base branch could not be found" }; | |
| 1286 | const [baseJson, basePackage] = await Promise.all([ | |
| 1287 | readFileAt(bare, baseTip, "forge.json"), | |
| 1288 | readFileAt(bare, baseTip, "package.json"), | |
| 1289 | ]); | |
| 1290 | const gate = parseCheck(baseJson, basePackage); | |
| 1291 | if (!gate.ok) | |
| 1292 | return { | |
| 1293 | ok: false, | |
| 1294 | error: | |
| 1295 | gate.error.kind === "invalid" | |
| 1296 | ? `${x.pull.baseBranch}: ${gate.error.message}` | |
| 1297 | : "the base branch declares no check", | |
| 1298 | }; | |
| 1299 | const held = heldPull(manifest, baseJson, "check"); | |
| 1300 | manifest = held.manifest; | |
| 1301 | if (gate.value !== undefined) { | |
| 1302 | manifest = { | |
| 1303 | ...manifest, | |
| 1304 | run: gate.value.run, | |
| 1305 | fast: gate.value.fast, | |
| 1306 | full: gate.value.full, | |
| 1307 | timeout: gate.value.timeout, | |
| 1308 | }; | |
| 1309 | write( | |
| 1310 | `tier ${x.tier} from ${x.pull.baseBranch}@${baseTip.slice(0, 7)}`, | |
| 1311 | ); | |
| 1312 | // An outside agent's pull nobody has approved is checked at once, with nobody's | |
| 1313 | // permission: in a cell, held to that list whatever the forge's egress setting, until a | |
| 1314 | // writer approves the pull (`origins/guest.ts`). | |
| 1315 | if (outsideAgentWaits(app.db, x.pull)) { | |
| 1316 | // Uncontained, a check is a plain process beside the forge's own data: a writer's code | |
| 1317 | // may be; an outside contributor's waits for a writer here, and says how to go on. | |
| 1318 | if (!(await cellsAvailable()).ok) return { ok: false, error: OUTSIDE_UNCONTAINED }; | |
| 1319 | outside = true; | |
| 1320 | write(OUTSIDE_CHECK_LINE(x.pull.baseBranch)); | |
| 1321 | } | |
| 1322 | } else if (outsideAgentWaits(app.db, x.pull)) { | |
| 1323 | // No gate on the base branch to hold it to: an outside contributor's pull does not bring | |
| 1324 | // its own. | |
| 1325 | return { ok: false, error: OUTSIDE_NO_GATE(x.pull.baseBranch) }; | |
| 1326 | } | |
| 1327 | // What it asked for and does not have yet, said where a refused request is looked for. | |
| 1328 | if (held.asks !== undefined) write(`${asksLine(held.asks, x.pull.baseBranch)} ${ASKS_NEXT}`); | |
| 1329 | } | |
| 1330 | if (x.commandsFrom !== undefined) { | |
| 1331 | const from = parseCheck(await readFileAt(bare, x.commandsFrom, "forge.json"), undefined); | |
| 1332 | if (from.ok && from.value !== undefined) manifest = { ...manifest, verify: from.value.verify, timeout: from.value.timeout }; | |
| 1333 | } | |
| 1334 | if (x.tier === "verify" && manifest.verify === null) | |
| 1335 | return { ok: false, error: "this commit declares no check.verify" }; | |
| 1336 | const command = | |
| 1337 | x.tier === "full" | |
| 1338 | ? manifest.full | |
| 1339 | : x.tier === "fast" | |
| 1340 | ? (manifest.fast ?? manifest.run) | |
| 1341 | : x.tier === "verify" | |
| 1342 | ? manifest.verify! | |
| 1343 | : manifest.run; | |
| 1344 | // The fast tier diffs what the pull changes against the commit it builds | |
| 1345 | // on, which the clone is given as a repository of its own: `origin/main` | |
| 1346 | // is not a ref there. | |
| 1347 | const base = manifest.fast !== null ? builtOnBase : undefined; | |
| 1348 | // The tree this runs on under this gate has been green before: that is | |
| 1349 | // the verdict, and nothing is checked out (#1795). | |
| 1350 | const learned = | |
| 1351 | base === undefined ? "" : readLearned(app.config.dataDir, x.pull!.id).join(","); | |
| 1352 | const tree = await runGit(["rev-parse", `${run.commitOid}^{tree}`], { cwd: bare }); | |
| 1353 | // A bisection step runs a slice: its green proves nothing of the whole. | |
| 1354 | const key = tree.ok && x.only === undefined | |
| 1355 | ? checkKey({ | |
| 1356 | repoId: repo.id, | |
| 1357 | treeOid: tree.value.trim(), | |
| 1358 | command, | |
| 1359 | install: manifest.install, | |
| 1360 | timeout: manifest.timeout, | |
| 1361 | egress: manifest.egress, | |
| 1362 | limits: manifest.limits, | |
| 1363 | base, | |
| 1364 | learned, | |
| 1365 | }) | |
| 1366 | : undefined; | |
| 1367 | const earlier = key === undefined ? undefined : provenBy(app.db, key); | |
| 1368 | if (earlier !== undefined) { | |
| 1369 | write(`already proven green by ${earlier}`); | |
| 1370 | return { ok: true, proven: earlier }; | |
| 1371 | } | |
| 1372 | // A runner that joined takes the steps (`runner_step.ts`, #13652): only a check that hands | |
| 1373 | // back nothing but its log and its verdict, and only while a runner is connected. Not taken, | |
| 1374 | // or an answer that is not the check's own verdict, and it runs here as it always has. | |
| 1375 | if (placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) { | |
| 1376 | const lines: string[] = []; | |
| 1377 | const sent = await runRunnerStep(app, { | |
| 1378 | runId: x.runId, | |
| 1379 | repoId: repo.id, | |
| 1380 | dir: bare, | |
| 1381 | job: { | |
| 1382 | run: x.runId, | |
| 1383 | repo: `${repo.ownerName}/${repo.name}`, | |
| 1384 | commit: run.commitOid, | |
| 1385 | ...(base === undefined ? {} : { base }), | |
| 1386 | depth: base === undefined ? 1 : await headDepth(bare, base, run.commitOid), | |
| 1387 | steps: [ | |
| 1388 | ...(manifest.install === null ? [] : [{ name: "install" as const, command: manifest.install }]), | |
| 1389 | { name: "check" as const, command }, | |
| 1390 | ], | |
| 1391 | // Only the check's own variables cross, never the forge's; the runner adds its cell's. | |
| 1392 | env: { | |
| 1393 | CI: "1", | |
| 1394 | FORGE_URL: options.forgeUrl, | |
| 1395 | FORGE_CHECK: run.id, | |
| 1396 | FORGE_COMMIT: x.commitOid, | |
| 1397 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 1398 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 1399 | ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }), | |
| 1400 | FORGE_CELLS: "off", | |
| 1401 | }, | |
| 1402 | limits: { | |
| 1403 | ...(manifest.limits.cpus === undefined ? {} : { cpus: manifest.limits.cpus }), | |
| 1404 | ...(manifest.limits.memoryMiB === undefined ? {} : { memoryMiB: manifest.limits.memoryMiB }), | |
| 1405 | ...(manifest.limits.diskMiB === undefined ? {} : { diskMiB: manifest.limits.diskMiB }), | |
| 1406 | }, | |
| 1407 | timeoutS: manifest.timeout, | |
| 1408 | egress: [...STEP_HOSTS, ...manifest.egress], | |
| 1409 | egressMode: outside ? "enforce" : egressSetting(app.db), | |
| 1410 | // An outside contributor's code runs in a cell or not at all, there as here. | |
| 1411 | cell: outside ? "required" : "preferred", | |
| 1412 | cache: { repo: repo.id, tree: tree.ok ? tree.value.trim() : "", base: builtOnBase ?? "" }, | |
| 1413 | }, | |
| 1414 | log: write, | |
| 1415 | output: lines, | |
| 1416 | signal, | |
| 1417 | waitMs: placement.waitMs, | |
| 1418 | }); | |
| 1419 | const verdict = sent.taken ? runnerVerdict(sent, lines, signal.aborted) : sent.why; | |
| 1420 | if (typeof verdict === "string") write(`placement: runner; ${verdict}; running here`); | |
| 1421 | else { | |
| 1422 | if (sent.taken && sent.runner !== "") | |
| 1423 | noteRunProgress(app.db, x.runId, { | |
| 1424 | placement: "runner", | |
| 1425 | node: sent.runner, | |
| 1426 | sandbox: { enforced: sent.contained, note: sent.note }, | |
| 1427 | }); | |
| 1428 | try { | |
| 1429 | recordCrashFlakes(app.db, app.bus, repo, lines, { runId: run.id, commit: run.commitOid, pull: true }, verdict.ok); | |
| 1430 | } catch (error) { | |
| 1431 | write(`crash flakes not recorded: ${String(error)}`); | |
| 1432 | } | |
| 1433 | if (!verdict.ok) return verdict; | |
| 1434 | if (key !== undefined) recordProven(app.db, key, x.id); | |
| 1435 | const cleared = clearedIn(lines); | |
| 1436 | return { ok: true, ...(cleared === undefined ? {} : { cleared }) }; | |
| 1437 | } | |
| 1438 | } | |
| 1439 | if (await reclaimIfFull(app)) write("disk under its free floor: swept unused checkouts"); | |
| 1440 | const dir = join(checksRoot(app.config.dataDir), run.id); | |
| 1441 | mkdirSync(dir, { recursive: true }); | |
| 1442 | write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine); | |
| 1443 | const exported = await exportTree(bare, run.commitOid, dir); | |
| 1444 | if (!exported.ok) return { ok: false, error: exported.error }; | |
| 1445 | if (base !== undefined) { | |
| 1446 | const given = await giveHistory(bare, dir, base, run.commitOid); | |
| 1447 | if (!given.ok) { | |
| 1448 | await rm(dir, { recursive: true, force: true }); | |
| 1449 | return { ok: false, error: given.error }; | |
| 1450 | } | |
| 1451 | } | |
| 1452 | // What the base's own check left (`checks/cache.ts`): tsc's build info | |
| 1453 | // and the gate's per-file times, keyed by the commit this one builds on. | |
| 1454 | const cached = restoreCache( | |
| 1455 | app.config.dataDir, | |
| 1456 | repo.id, | |
| 1457 | builtOnBase === undefined ? [] : [builtOnBase], | |
| 1458 | dir, | |
| 1459 | ); | |
| 1460 | if (cached !== undefined) write(`cache from ${cached.slice(0, 7)}`); | |
| 1461 | // Every check starts from the impact map the last green default-branch | |
| 1462 | // run measured; the fast tier selects by it. | |
| 1463 | try { | |
| 1464 | const seeded = seedImpact(app.config.dataDir, repo.id, dir); | |
| 1465 | if (seeded > 0) write(`impact map: ${seeded} tests, from the last green ${repo.defaultBranch}`); | |
| 1466 | } catch (error) { | |
| 1467 | write(`impact map not seeded: ${String(error)}`); | |
| 1468 | } | |
| 1469 | // The fast tier skips a test file whose inputs a default-branch run already proved green. | |
| 1470 | try { | |
| 1471 | const seeded = seedKept(app.config.dataDir, repo.id, dir); | |
| 1472 | if (seeded > 0) write(`kept results: ${seeded} tests, from ${repo.defaultBranch}'s measuring runs`); | |
| 1473 | } catch (error) { | |
| 1474 | write(`kept results not seeded: ${String(error)}`); | |
| 1475 | } | |
| 1476 | // The check is repository code: it runs in a cell of its own | |
| 1477 | // (`apps/contain.ts`), its checkout and a home its only places to write, | |
| 1478 | // the forge, the registry and what it declared its only way out. | |
| 1479 | // A Job holds egress only through a gate the forge opens for it | |
| 1480 | // (`checks/job_gate.ts`, #13251): a cluster that names none cannot | |
| 1481 | // honour forge.json, so the check stays in a cell, never unrestricted. | |
| 1482 | const fellBack = placement.mode === "job" && placement.gate === undefined; | |
| 1483 | // A check a runner did not take runs here as `process` does. | |
| 1484 | const placed: Placement = fellBack || placement.mode === "runner" ? { mode: "process" } : placement; | |
| 1485 | const job = placed.mode === "job" ? placed : undefined; | |
| 1486 | if (x.runId !== undefined) | |
| 1487 | noteRunProgress(app.db, x.runId, { placement: placed.mode }); | |
| 1488 | if (fellBack) | |
| 1489 | write("placement: job skipped, this cluster names no egress gate for a Job (FORGE_CHECKS_EGRESS_GATE)"); | |
| 1490 | // A Job is its own boundary: no cell here, the cluster and its gate hold it. | |
| 1491 | const cells = job === undefined ? await cellsAvailable() : undefined; | |
| 1492 | if (cells !== undefined) { | |
| 1493 | if (x.runId !== undefined) | |
| 1494 | noteRunProgress(app.db, x.runId, { | |
| 1495 | sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason }, | |
| 1496 | }); | |
| 1497 | if (!cells.ok) | |
| 1498 | write(`uncontained here: ${cells.reason}; the check runs unrestricted`); | |
| 1499 | } else { | |
| 1500 | if (x.runId !== undefined) | |
| 1501 | noteRunProgress(app.db, x.runId, { sandbox: { enforced: true, note: JOB_GATE_NOTE } }); | |
| 1502 | write("placement: job"); | |
| 1503 | } | |
| 1504 | const home = `${dir}.home`; | |
| 1505 | const cell: CellSpec | undefined = cells?.ok | |
| 1506 | ? { | |
| 1507 | dir, | |
| 1508 | writable: [], | |
| 1509 | home, | |
| 1510 | hide: cellHides(app), | |
| 1511 | egress: [...STEP_HOSTS, ...manifest.egress], | |
| 1512 | reach: [options.forgeUrl], | |
| 1513 | limits: resolveLimits("step", manifest.limits), | |
| 1514 | onUndeclared: (host, port) => | |
| 1515 | noteReached(app.config.dataDir, repo.id, host, port), | |
| 1516 | // What an outside contributor's check may reach is held to, whatever the forge's setting. | |
| 1517 | egressMode: outside ? "enforce" : egressSetting(app.db), | |
| 1518 | label: `check:${run.id}`, | |
| 1519 | } | |
| 1520 | : undefined; | |
| 1521 | try { | |
| 1522 | // The pod's memory less the forge's own leaves too little for a cell | |
| 1523 | // (#12519): refused before a step starts, so the forge is never the one | |
| 1524 | // the OOM killer takes. The machine's, not the pull's: no red. A Job's | |
| 1525 | // memory is the cluster's, not this pod's. | |
| 1526 | const small = job === undefined | |
| 1527 | ? refuseSmallPod({ env: process.env["FORGE_CELL_MEMORY_MAX_MIB"], ...options.podRoom }, write) | |
| 1528 | : null; | |
| 1529 | if (small !== null) return small; | |
| 1530 | for (const line of limitWarnings(manifest.limits)) write(line); | |
| 1531 | const cellCpus = resolveLimits("step", manifest.limits).cpus; | |
| 1532 | const env = childEnv({ | |
| 1533 | // The shard count an admin set (`core/capacity.ts`), else the | |
| 1534 | // default sized to what this check's cell holds, never the pod's. | |
| 1535 | ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)), | |
| 1536 | CI: "1", | |
| 1537 | FORGE_URL: options.forgeUrl, | |
| 1538 | FORGE_CHECK: run.id, | |
| 1539 | FORGE_COMMIT: x.commitOid, | |
| 1540 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 1541 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 1542 | ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }), | |
| 1543 | // The cell's CPU limit, which `availableParallelism()` may not | |
| 1544 | // report inside it; the shard count follows this (#2257). | |
| 1545 | FORGE_CELL_CPUS: String(cellCpus), | |
| 1546 | // The one place besides the checkout the cell can write on disk; its | |
| 1547 | // own /tmp is a 256 MiB tmpfs the shards would fill (#8679). | |
| 1548 | ...scratchEnv(cell === undefined ? undefined : home), | |
| 1549 | ...(base === undefined | |
| 1550 | ? {} | |
| 1551 | : { | |
| 1552 | FORGE_CHECK_BASE: base, | |
| 1553 | FORGE_CHECK_LEARNED: learned, | |
| 1554 | }), | |
| 1555 | // A cell inside a cell is not a cell. This check's own cell is the | |
| 1556 | // boundary; a forge the tests boot inside it would wrap its apps in | |
| 1557 | // nested cells, and those never answer in time. Its children run as | |
| 1558 | // plain processes here — as they do on a runner without bwrap. | |
| 1559 | FORGE_CELLS: "off", | |
| 1560 | }); | |
| 1561 | let ran: Result<void, string>; | |
| 1562 | const checkLines: string[] = []; | |
| 1563 | // A bisection step runs only some files: what it shows green says nothing of the rest. | |
| 1564 | const wholeVerify = x.tier === "verify" && x.only === undefined; | |
| 1565 | // The gate is held to what a cell holds: the forge, the registry and | |
| 1566 | // what forge.json declares, an outside contributor's always enforced. | |
| 1567 | const gate = job?.gate === undefined | |
| 1568 | ? undefined | |
| 1569 | : openJobGate( | |
| 1570 | job.gate, | |
| 1571 | jobGatePolicy({ | |
| 1572 | hosts: [...STEP_HOSTS, ...manifest.egress], | |
| 1573 | origins: [options.forgeUrl].map(origin).filter((o) => o !== undefined), | |
| 1574 | label: `check:${run.id}`, | |
| 1575 | outside, | |
| 1576 | setting: egressSetting(app.db), | |
| 1577 | note: (host, port) => noteReached(app.config.dataDir, repo.id, host, port), | |
| 1578 | }), | |
| 1579 | write, | |
| 1580 | ); | |
| 1581 | if (gate !== undefined && !gate.ok) { | |
| 1582 | ran = err(gate.error); | |
| 1583 | } else if (job !== undefined) { | |
| 1584 | try { | |
| 1585 | ran = await runJobStep(job, { | |
| 1586 | runId: run.id, | |
| 1587 | dir, | |
| 1588 | commands: manifest.install === null ? [command] : [manifest.install, command], | |
| 1589 | env, | |
| 1590 | log: write, | |
| 1591 | output: checkLines, | |
| 1592 | signal, | |
| 1593 | timeoutMs: manifest.timeout * 1000, | |
| 1594 | limits: resolveLimits("step", manifest.limits), | |
| 1595 | onNode: (node) => { | |
| 1596 | if (x.runId !== undefined) noteRunProgress(app.db, x.runId, { node }); | |
| 1597 | }, | |
| 1598 | ...(gate === undefined ? {} : { proxy: gate.value.env }), | |
| 1599 | ...(options.job === undefined ? {} : { options: options.job }), | |
| 1600 | }); | |
| 1601 | } finally { | |
| 1602 | gate?.value.close(); | |
| 1603 | } | |
| 1604 | } else { | |
| 1605 | if (manifest.install !== null) { | |
| 1606 | const install = manifest.install; | |
| 1607 | // A cell's home is its own cache. Uncontained, the repository's | |
| 1608 | // cache is shared, so installs into it take turns (`checks/install.ts`). | |
| 1609 | const shared = | |
| 1610 | cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined; | |
| 1611 | // A cell installs from that same cache, bound read-only (#7543): | |
| 1612 | // the forge fills it first, on the host, so no pull writes what | |
| 1613 | // the gate on main installs. Cluster jobs (`runJobStep`) are | |
| 1614 | // unchanged: a Job has its own cache and never takes this path. | |
| 1615 | const warm = cell === undefined ? undefined : checkInstallCache(app.config.dataDir, repo.id); | |
| 1616 | // Set once the fill has run: a cell installs from the shared cache | |
| 1617 | // only when the forge filled it; otherwise it keeps a cache of its own. | |
| 1618 | let filled = false; | |
| 1619 | const once = (log: (line: string) => void): Promise<Result<void, string>> => { | |
| 1620 | const base = shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared }; | |
| 1621 | const from = cellInstall(install, base, filled ? warm : undefined); | |
| 1622 | const revealed = cell !== undefined && filled && warm !== undefined ? { ...cell, reveal: [warm] } : cell; | |
| 1623 | return runStep("install", from.command, { | |
| 1624 | cwd: dir, | |
| 1625 | env: from.env, | |
| 1626 | log, | |
| 1627 | signal, | |
| 1628 | timeoutMs: manifest.timeout * 1000, | |
| 1629 | cell: revealed, | |
| 1630 | }); | |
| 1631 | }; | |
| 1632 | // What the cell's proxy received, for the one line that shows the cache | |
| 1633 | // working (#7546). Uncontained, nothing is logged: the shared cache | |
| 1634 | // is not metered, and its growth is wrong under concurrent checks. | |
| 1635 | // Stopped in `finally`, so a throw never leaves the label metered; | |
| 1636 | // over a MiB from the npm registry adds the miss line (#8382). | |
| 1637 | const installed = await meteredInstall( | |
| 1638 | cell === undefined ? undefined : `check:${run.id}`, | |
| 1639 | repo.id, | |
| 1640 | () => | |
| 1641 | installWithRetry( | |
| 1642 | async (log) => { | |
| 1643 | if (shared === undefined) { | |
| 1644 | if (warm !== undefined) { | |
| 1645 | const fill = await prefillInstallCache({ | |
| 1646 | cache: warm, | |
| 1647 | dir, | |
| 1648 | env, | |
| 1649 | log, | |
| 1650 | signal, | |
| 1651 | timeoutMs: manifest.timeout * 1000, | |
| 1652 | }); | |
| 1653 | if (!fill.ok) return fill; | |
| 1654 | filled = fill.value; | |
| 1655 | void evictSoon(warm, process.env); | |
| 1656 | } | |
| 1657 | return once(log); | |
| 1658 | } | |
| 1659 | mkdirSync(shared, { recursive: true }); | |
| 1660 | const held = await withLock( | |
| 1661 | `${shared}.lock`, | |
| 1662 | async () => { | |
| 1663 | const done = await once(log); | |
| 1664 | if (done.ok) lend(shared, dir); | |
| 1665 | return done; | |
| 1666 | }, | |
| 1667 | { signal }, | |
| 1668 | ); | |
| 1669 | if (held.ok && held.value.ok) void evictSoon(shared, process.env); | |
| 1670 | return held.ok ? held.value : held; | |
| 1671 | }, | |
| 1672 | { log: write, signal, retryMs: options.installRetryMs }, | |
| 1673 | ), | |
| 1674 | write, | |
| 1675 | { fetched: true }, | |
| 1676 | ); | |
| 1677 | if (!installed.ok) | |
| 1678 | return { | |
| 1679 | ok: false, | |
| 1680 | error: installed.error, | |
| 1681 | infrastructure: installed.infrastructure, | |
| 1682 | stage: "install", | |
| 1683 | }; | |
| 1684 | } | |
| 1685 | // The check step's own lines are kept beside what `write` already | |
| 1686 | // logs, so a failure can be summarised once here — never by reading | |
| 1687 | // the log back, on this run or any later render of it. `runStep` | |
| 1688 | // echoes the command as its first line before any output of the | |
| 1689 | // command's own; that echo is not output to summarise from. | |
| 1690 | let sawEcho = false; | |
| 1691 | ran = await (options.checkStep ?? runStep)("check", command, { | |
| 1692 | cwd: dir, | |
| 1693 | env, | |
| 1694 | log: (line) => { | |
| 1695 | if (sawEcho) checkLines.push(line); | |
| 1696 | sawEcho = true; | |
| 1697 | write(line); | |
| 1698 | }, | |
| 1699 | signal, | |
| 1700 | timeoutMs: manifest.timeout * 1000, | |
| 1701 | cell, | |
| 1702 | }); | |
| 1703 | } | |
| 1704 | if (!ran.ok && ran.error === JOB_VANISHED) return { ok: false, error: JOB_VANISHED }; | |
| 1705 | // A file the gate cleared because it passed alone is still a failure | |
| 1706 | // to track: the cell cannot reach the forge, so it is filed from here. | |
| 1707 | try { | |
| 1708 | // Only the default branch's own run files: a pull branch controls | |
| 1709 | // its check's output, and the issues are authored as the owner. | |
| 1710 | const all = | |
| 1711 | x.fileOrderDependent === true | |
| 1712 | ? clearedFiles(checkLines) | |
| 1713 | : []; | |
| 1714 | const cleared = all.slice(0, MAX_FILED); | |
| 1715 | if (cleared.length > 0) { | |
| 1716 | fileOrderDependent(app.db, app.bus, repo, cleared, { | |
| 1717 | runId: run.id, | |
| 1718 | commit: run.commitOid, | |
| 1719 | }); | |
| 1720 | write(`filed: ${cleared.length} test file(s) that fail only beside others`); | |
| 1721 | if (all.length > cleared.length) | |
| 1722 | write(`not filed: ${all.length - cleared.length} more (at most ${MAX_FILED} per run)`); | |
| 1723 | } | |
| 1724 | } catch (error) { | |
| 1725 | write(`order-dependent tests not filed: ${String(error)}`); | |
| 1726 | } | |
| 1727 | // Main's red costs the forge every pull's check (#6643): one or two | |
| 1728 | // failed tests are re-run alone, on this commit in this cell, before | |
| 1729 | // main counts as red. Passing alone is a flake: green, and recorded. | |
| 1730 | // A pull's red is the same unless the pull touched the test (#7426). | |
| 1731 | if (!ran.ok && job === undefined && (x.isMain === true || tip !== undefined)) { | |
| 1732 | try { | |
| 1733 | let tests = rerunnable(checkLines, parseBunTestFailure(checkLines)); | |
| 1734 | if (tests !== undefined && x.isMain !== true) { | |
| 1735 | const diff = await runGit( | |
| 1736 | ["diff", "--name-only", "-z", `${tip}...${x.commitOid}`], | |
| 1737 | { cwd: bare }, | |
| 1738 | ); | |
| 1739 | const changed = diff.ok ? diff.value.split("\0").filter((p) => p !== "") : undefined; | |
| 1740 | const read = (path: string): string | undefined => { | |
| 1741 | try { | |
| 1742 | return readFileSync(join(dir, path), "utf8"); | |
| 1743 | } catch { | |
| 1744 | return undefined; | |
| 1745 | } | |
| 1746 | }; | |
| 1747 | if (pullOwnsRed(tests, changed, read)) { | |
| 1748 | write("not re-run alone: the pull changed a failed test or what it imports"); | |
| 1749 | tests = undefined; | |
| 1750 | } | |
| 1751 | } | |
| 1752 | if (tests !== undefined) { | |
| 1753 | write(`re-running ${tests.length} failed test(s) alone`); | |
| 1754 | let alone = true; | |
| 1755 | for (const test of tests) { | |
| 1756 | const out: string[] = []; | |
| 1757 | const again = await runStep("flake", rerunCommand(test), { | |
| 1758 | cwd: dir, | |
| 1759 | env, | |
| 1760 | log: (line) => { | |
| 1761 | out.push(line); | |
| 1762 | write(line); | |
| 1763 | }, | |
| 1764 | signal, | |
| 1765 | timeoutMs: Math.min(manifest.timeout, RERUN_TIMEOUT_S) * 1000, | |
| 1766 | cell, | |
| 1767 | }); | |
| 1768 | if (!again.ok || !passedAlone(out)) { | |
| 1769 | alone = false; | |
| 1770 | write(`"${test.name}" fails alone too: ${x.isMain === true ? "main" : "the check"} is red`); | |
| 1771 | break; | |
| 1772 | } | |
| 1773 | } | |
| 1774 | if (alone) { | |
| 1775 | recordFlakes(app.db, app.bus, repo, tests, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }); | |
| 1776 | for (const t of tests) write(`flake: ${t.file} > ${t.name} passed alone`); | |
| 1777 | write(`flaky: ${tests.map((t) => `"${t.name}"`).join(", ")} failed in the full run and passed alone; recorded, ${x.isMain === true ? "main" : "the check"} stays green`); | |
| 1778 | ran = { ok: true, value: undefined }; | |
| 1779 | } | |
| 1780 | } | |
| 1781 | } catch (error) { | |
| 1782 | write(`flake re-run failed: ${String(error)}`); | |
| 1783 | } | |
| 1784 | } | |
| 1785 | // A shard that crashed outside any test and ran green once more (#7781). | |
| 1786 | { | |
| 1787 | try { | |
| 1788 | recordCrashFlakes(app.db, app.bus, repo, checkLines, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }, ran.ok); | |
| 1789 | } catch (error) { | |
| 1790 | write(`crash flakes not recorded: ${String(error)}`); | |
| 1791 | } | |
| 1792 | } | |
| 1793 | if (!ran.ok) { | |
| 1794 | const { summary, detail } = summariseCheckFailure( | |
| 1795 | checkLines, | |
| 1796 | ran.error, | |
| 1797 | ); | |
| 1798 | const killed = killedStep(ran.error); | |
| 1799 | // A red verification keeps the map it measured all the same, and | |
| 1800 | // names the files it failed in for the bisection (#7786). | |
| 1801 | const files = x.tier === "verify" ? failedFiles(checkLines) : []; | |
| 1802 | // Each failing file once more on its own (#12296): green alone is | |
| 1803 | // flaky under load, which a builder cannot fix; red alone stays red. | |
| 1804 | const flaky = await rerunRed( | |
| 1805 | { files, inJob: job !== undefined, killed, aborted: signal.aborted }, | |
| 1806 | async (file, timeoutMs) => { | |
| 1807 | const again = await runStep("rerun", rerunFileCommand(file), { | |
| 1808 | cwd: dir, | |
| 1809 | env, | |
| 1810 | log: write, | |
| 1811 | signal, | |
| 1812 | timeoutMs: Math.min(timeoutMs, manifest.timeout * 1000), | |
| 1813 | cell, | |
| 1814 | }); | |
| 1815 | return again.ok; | |
| 1816 | }, | |
| 1817 | write, | |
| 1818 | ); | |
| 1819 | if (x.tier === "verify" && x.keepImpact === true) { | |
| 1820 | try { | |
| 1821 | const kept = keepImpact(app.config.dataDir, repo.id, dir); | |
| 1822 | if (kept !== undefined) write(kept); | |
| 1823 | const results = keepKept(app.config.dataDir, repo.id, dir, true); | |
| 1824 | if (results !== undefined) write(results); | |
| 1825 | } catch (error) { | |
| 1826 | write(`impact map not kept: ${String(error)}`); | |
| 1827 | } | |
| 1828 | } | |
| 1829 | return { | |
| 1830 | ok: false, | |
| 1831 | error: killed ? ran.error : summary, | |
| 1832 | detail: withFlaky(redDetail(detail, files, checkLines), flaky), | |
| 1833 | ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}), | |
| 1834 | stage: checkStage(ran.error), | |
| 1835 | ...(killed ? { killed } : {}), | |
| 1836 | ...(stoppedForCpu(ran.error) ? { cpuStop: ran.error } : {}), | |
| 1837 | }; | |
| 1838 | } | |
| 1839 | // Only a green check of the default branch is kept for the next. | |
| 1840 | if ( | |
| 1841 | x.keep === true && | |
| 1842 | keepCache(app.config.dataDir, repo.id, run.commitOid, dir) | |
| 1843 | ) | |
| 1844 | write(`cache kept for ${run.commitOid.slice(0, 7)}`); | |
| 1845 | if (x.tier === "full" || x.keepImpact === true) { | |
| 1846 | try { | |
| 1847 | const kept = keepImpact(app.config.dataDir, repo.id, dir); | |
| 1848 | if (kept !== undefined) write(kept); | |
| 1849 | // Only the default branch's own run hands results back: a pull's tree could write any. | |
| 1850 | const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true); | |
| 1851 | if (results !== undefined) write(results); | |
| 1852 | } catch (error) { | |
| 1853 | write(`impact map not kept: ${String(error)}`); | |
| 1854 | } | |
| 1855 | } | |
| 1856 | if (key !== undefined) recordProven(app.db, key, x.id); | |
| 1857 | const cleared = clearedIn(checkLines); | |
| 1858 | return { | |
| 1859 | ok: true, | |
| 1860 | ...(wholeVerify ? { passed: passedFiles(checkLines) } : {}), | |
| 1861 | ...(cleared === undefined ? {} : { cleared }), | |
| 1862 | }; | |
| 1863 | } finally { | |
| 1864 | if (existsSync(dir)) await removeRunDir(dir); | |
| 1865 | await removeRunDir(home); | |
| 1866 | } | |
| 1867 | } | |
| 1868 | ||
| 1869 | 106 | // What starts a check: a branch moving, a pull opening, a fork's push |
| 1870 | 107 | // moving a pull's head. The commit checked is always the one the pull |
| 1871 | 108 | // proposes now. |
| 1872 | 109 | const unsubscribe = app.bus.subscribe((event: ForgeEvent) => { |
| 1873 | onEvent(event).catch((error) => | |
| 110 | onEvent(lane, event).catch((error) => | |
| 1874 | 111 | log.error("checks event handler failed", { |
| 1875 | 112 | kind: event.kind, |
| 1876 | 113 | error: String(error), |
| @@ -1878,335 +115,40 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 1878 | 115 | ); |
| 1879 | 116 | }); |
| 1880 | 117 | |
| 1881 | async function onEvent(event: ForgeEvent): Promise<void> { | |
| 1882 | if (stopped || event.repoId === null) return; | |
| 1883 | const repo = getRepoById(app.db, event.repoId); | |
| 1884 | if (!repo) return; | |
| 1885 | const actor = event.actorId ?? undefined; | |
| 1886 | switch (event.kind) { | |
| 1887 | case "push": { | |
| 1888 | // A fork that silently followed holds its parent's commit, byte for | |
| 1889 | // byte: checking the copy proves nothing the parent's own check did | |
| 1890 | // not, once per fork, on every landed pull. A merge of the parent | |
| 1891 | // into the fork's own work is a new commit, and is checked. | |
| 1892 | if ( | |
| 1893 | typeof event.payload["followed"] === "string" && | |
| 1894 | event.payload["merged"] !== true | |
| 1895 | ) | |
| 1896 | return; | |
| 1897 | const updates = event.payload["updates"]; | |
| 1898 | if (!Array.isArray(updates)) return; | |
| 1899 | for (const update of updates as { ref?: unknown; newOid?: unknown }[]) { | |
| 1900 | if ( | |
| 1901 | typeof update.ref !== "string" || | |
| 1902 | typeof update.newOid !== "string" | |
| 1903 | ) | |
| 1904 | continue; | |
| 1905 | if ( | |
| 1906 | !update.ref.startsWith("refs/heads/") || | |
| 1907 | /^0+$/.test(update.newOid) | |
| 1908 | ) | |
| 1909 | continue; | |
| 1910 | const branch = update.ref.slice("refs/heads/".length); | |
| 1911 | if (branch === repo.defaultBranch) | |
| 1912 | await request( | |
| 1913 | repo, | |
| 1914 | branch, | |
| 1915 | update.newOid, | |
| 1916 | "push", | |
| 1917 | undefined, | |
| 1918 | actor, | |
| 1919 | ); | |
| 1920 | await recheckBase(repo, branch, update.newOid); | |
| 1921 | for (const pull of listPulls(app.db, repo.id, { | |
| 1922 | sealed: true, | |
| 1923 | state: "open", | |
| 1924 | limit: 200, | |
| 1925 | })) { | |
| 1926 | if (pull.crossRepo || pull.headBranch !== branch) continue; | |
| 1927 | await request( | |
| 1928 | repo, | |
| 1929 | `pull-${pull.number}`, | |
| 1930 | update.newOid, | |
| 1931 | "push", | |
| 1932 | pull.id, | |
| 1933 | actor, | |
| 1934 | ); | |
| 1935 | } | |
| 1936 | } | |
| 1937 | return; | |
| 1938 | } | |
| 1939 | case "pull.opened": | |
| 1940 | case "pull.reopened": { | |
| 1941 | const number = event.payload["number"]; | |
| 1942 | if (typeof number !== "number") return; | |
| 1943 | const pull = getPull(app.db, repo.id, number); | |
| 1944 | if (!pull) return; | |
| 1945 | const head = await headOf(repo, pull); | |
| 1946 | if (head !== undefined) | |
| 1947 | await request( | |
| 1948 | repo, | |
| 1949 | `pull-${pull.number}`, | |
| 1950 | head, | |
| 1951 | "pull", | |
| 1952 | pull.id, | |
| 1953 | actor, | |
| 1954 | ); | |
| 1955 | return; | |
| 1956 | } | |
| 1957 | case "pull.pushed": { | |
| 1958 | const number = event.payload["number"]; | |
| 1959 | const headOid = event.payload["headOid"]; | |
| 1960 | if (typeof number !== "number" || typeof headOid !== "string") return; | |
| 1961 | const pull = getPull(app.db, repo.id, number); | |
| 1962 | if (!pull) return; | |
| 1963 | await request( | |
| 1964 | repo, | |
| 1965 | `pull-${pull.number}`, | |
| 1966 | headOid, | |
| 1967 | "push", | |
| 1968 | pull.id, | |
| 1969 | actor, | |
| 1970 | ); | |
| 1971 | return; | |
| 1972 | } | |
| 1973 | // An approval is what lets a reader's pull run: check its head now. | |
| 1974 | case "pull.run_granted": | |
| 1975 | case "pull.reviewed": { | |
| 1976 | const number = event.payload["number"]; | |
| 1977 | if (typeof number !== "number") return; | |
| 1978 | if ( | |
| 1979 | event.kind === "pull.reviewed" && | |
| 1980 | event.payload["state"] !== "approved" | |
| 1981 | ) | |
| 1982 | return; | |
| 1983 | const pull = getPull(app.db, repo.id, number); | |
| 1984 | if (!pull || pull.state !== "open") return; | |
| 1985 | const head = await headOf(repo, pull); | |
| 1986 | if (head === undefined) return; | |
| 1987 | // A head already green is not checked again for nothing (#4391). | |
| 1988 | const env = `pull-${pull.number}`; | |
| 1989 | const dir = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 1990 | const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`); | |
| 1991 | // The merge queue gates only an autonomous repository's lands (`crew/autonomy.ts` admit). | |
| 1992 | const landGated = repo.autonomy === "autonomous" && repo.autonomyBy !== null; | |
| 1993 | if (!checkOnApproval(checksForHead(app.db, pull.id, head), env, tip, landGated)) | |
| 1994 | return; | |
| 1995 | await request(repo, env, head, "approved", pull.id, actor); | |
| 1996 | return; | |
| 1997 | } | |
| 1998 | // Landed or closed: nothing left to check, so nothing waits in the lane. | |
| 1999 | case "pull.merged": | |
| 2000 | case "pull.closed": { | |
| 2001 | const number = event.payload["number"]; | |
| 2002 | if (typeof number !== "number") return; | |
| 2003 | const pull = getPull(app.db, repo.id, number); | |
| 2004 | if (!pull) return; | |
| 2005 | const cancelled = cancelRunsForPull( | |
| 2006 | app.db, | |
| 2007 | pull.id, | |
| 2008 | event.kind === "pull.merged" ? "pull landed" : "pull closed", | |
| 2009 | ); | |
| 2010 | for (const run of cancelled) { | |
| 2011 | inFlight.get(run.id)?.abort.abort(); | |
| 2012 | announceRun(app, run); | |
| 2013 | } | |
| 2014 | // A builder run still in flight when its pull lands reads as done now (#12049). | |
| 2015 | if (event.kind === "pull.merged") | |
| 2016 | for (const id of settleLandedRuns(app.db, { issueId: pull.id, headBranch: pull.headBranch, repoId: repo.id, number })) { | |
| 2017 | const done = getRun(app.db, id); | |
| 2018 | if (done) announceRun(app, done); | |
| 2019 | } | |
| 2020 | // A land moves the default branch without a push event: check its | |
| 2021 | // new head, or a red main stays red until someone pushes to it. The | |
| 2022 | // check settles first: a candidate's next step supersedes it. | |
| 2023 | const mergeOid = event.payload["mergeOid"]; | |
| 2024 | if ( | |
| 2025 | event.kind === "pull.merged" && | |
| 2026 | pull.baseBranch === repo.defaultBranch && | |
| 2027 | typeof mergeOid === "string" && | |
| 2028 | /^[0-9a-f]{40}$/.test(mergeOid) | |
| 2029 | ) | |
| 2030 | await request( | |
| 2031 | repo, | |
| 2032 | repo.defaultBranch, | |
| 2033 | mergeOid, | |
| 2034 | "push", | |
| 2035 | undefined, | |
| 2036 | actor, | |
| 2037 | settle > 0 ? new Date(Date.now() + settle).toISOString() : undefined, | |
| 2038 | ); | |
| 2039 | if (event.kind === "pull.merged" && typeof mergeOid === "string" && /^[0-9a-f]{40}$/.test(mergeOid)) | |
| 2040 | await recheckBase(repo, pull.baseBranch, mergeOid); | |
| 2041 | return; | |
| 2042 | } | |
| 2043 | default: | |
| 2044 | return; | |
| 2045 | } | |
| 2046 | } | |
| 2047 | ||
| 2048 | /** | |
| 2049 | * The base moved (#12533): every open pull on it whose green was of an | |
| 2050 | * older tip is checked again, once per move (`base_moved.ts`), newest base | |
| 2051 | * first (`newestBaseFirst`). A queued check merges the newest tip when it | |
| 2052 | * starts, so a later move supersedes nothing and queues nothing. Skipped | |
| 2053 | * where the merge queue gates lands: its gate merges with the tip itself. | |
| 2054 | */ | |
| 2055 | async function recheckBase(repo: Repo, branch: string, tip: string): Promise<void> { | |
| 2056 | if (landGated(repo)) return; | |
| 2057 | const due: { number: number; green: Run; pull: Pull }[] = []; | |
| 2058 | for (const pull of listOpenPulls(app.db, repo.id)) { | |
| 2059 | if (pull.baseBranch !== branch) continue; | |
| 2060 | const green = dueOf(pull, tip); | |
| 2061 | if (green !== undefined) due.push({ number: pull.number, green, pull }); | |
| 2062 | } | |
| 2063 | await askInOrder(repo, newestBaseFirst(due).map((item) => item.pull)); | |
| 2064 | } | |
| 2065 | ||
| 2066 | const landGated = (repo: Repo): boolean => repo.autonomy === "autonomous" && repo.autonomyBy !== null; | |
| 2067 | ||
| 2068 | function dueOf(pull: Pull, tip: string): Run | undefined { | |
| 2069 | if (pull.state !== "open" || pull.headOid === "" || pull.mergedAt !== null) return undefined; | |
| 2070 | return dueGreen(checksForHead(app.db, pull.id, pull.headOid), `pull-${pull.number}`, tip, false); | |
| 2071 | } | |
| 2072 | ||
| 2073 | /** | |
| 2074 | * Asks a re-check for each pull, the first to take a slot first. The queue | |
| 2075 | * starts the newest request first (`pickNext`), so they are asked last-first | |
| 2076 | * with scheduling held until every one is queued: a free slot does not go | |
| 2077 | * to whichever happened to be asked before the rest. | |
| 2078 | */ | |
| 2079 | async function askInOrder(repo: Repo, pulls: readonly Pull[]): Promise<void> { | |
| 2080 | if (pulls.length === 0) return; | |
| 2081 | holding++; | |
| 2082 | try { | |
| 2083 | for (const pull of [...pulls].reverse()) { | |
| 2084 | try { | |
| 2085 | await request(repo, `pull-${pull.number}`, pull.headOid, BASE_MOVED, pull.id); | |
| 2086 | } catch (error) { | |
| 2087 | log.error("check could not be asked again after the base moved", { pull: pull.number, error: String(error) }); | |
| 2088 | } | |
| 2089 | } | |
| 2090 | } finally { | |
| 2091 | holding--; | |
| 2092 | schedule(); | |
| 2093 | } | |
| 2094 | } | |
| 2095 | ||
| 2096 | /** A check that finished green while its base moved is asked again, once, on the tip now. */ | |
| 2097 | async function recheckFinished(runId: string, repo: Repo): Promise<void> { | |
| 2098 | // The common path, a red or a gated repository, reads no ref and no pull. | |
| 2099 | if (landGated(repo)) return; | |
| 2100 | const run = getRun(app.db, runId); | |
| 2101 | const number = /^pull-(\d+)$/.exec(run?.env ?? "")?.[1]; | |
| 2102 | if (run === undefined || number === undefined || run.state !== "ok" || run.baseOid === "") return; | |
| 2103 | const pull = getPull(app.db, repo.id, Number(number)); | |
| 2104 | if (!pull || pull.id !== run.issueId || pull.headOid !== run.commitOid) return; | |
| 2105 | // A branch's tip is a file read (`resolveOid`), no process. | |
| 2106 | const tip = await resolveOid(await pullDir(app.db, app.reposRoot, repo, pull), `refs/heads/${pull.baseBranch}`); | |
| 2107 | if (tip === undefined || tip === run.baseOid) return; | |
| 2108 | if (dueOf(pull, tip) !== undefined) await askInOrder(repo, [pull]); | |
| 2109 | } | |
| 2110 | ||
| 2111 | async function headOf(repo: Repo, pull: Pull): Promise<string | undefined> { | |
| 2112 | const dir = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 2113 | const ref = pull.crossRepo | |
| 2114 | ? `refs/pull/${pull.number}/head` | |
| 2115 | : `refs/heads/${pull.headBranch}`; | |
| 2116 | return (await resolveOid(dir, ref)) ?? (pull.headOid || undefined); | |
| 2117 | } | |
| 2118 | ||
| 2119 | 118 | // A hosted child's lane runs on its parent's thread: its ticks are named for it |
| 2120 | 119 | // (`perf/child_time.ts`). |
| 2121 | 120 | const owner = options.label === undefined ? undefined : childLabel(options.label); |
| 2122 | 121 | const timer = setInterval(() => { |
| 2123 | if (!stopped) during("lane", schedule, owner, app); | |
| 122 | if (!lane.stopped) during("lane", lane.schedule, owner, app); | |
| 2124 | 123 | }, options.tickMs ?? TICK_MS); |
| 2125 | 124 | // Main is verified after landing on a schedule, when the forge says how often. |
| 2126 | 125 | const sweeper = |
| 2127 | 126 | options.verifyEveryMs === undefined |
| 2128 | 127 | ? undefined |
| 2129 | 128 | : setInterval(() => { |
| 2130 | if (!stopped) | |
| 2131 | during("verify-sweep", askVerifications, owner, app).catch((error) => | |
| 129 | if (!lane.stopped) | |
| 130 | during("verify-sweep", () => askVerifications(lane), owner, app).catch((error) => | |
| 2132 | 131 | log.error("verification not asked", { error: String(error) }), |
| 2133 | 132 | ); |
| 2134 | 133 | }, Math.min(VERIFY_SWEEP_MS, options.verifyEveryMs)); |
| 2135 | 134 | // A gate's slot frees: the pull check held for it starts now, not at the tick. |
| 2136 | const unwatch = watchGates(schedule); | |
| 2137 | during("lane", schedule, owner, app); | |
| 2138 | void during("lane-catch-up", catchUp, owner, app); | |
| 135 | const unwatch = watchGates(lane.schedule); | |
| 136 | during("lane", lane.schedule, owner, app); | |
| 137 | void during("lane-catch-up", () => catchUp(lane), owner, app); | |
| 2139 | 138 | |
| 2140 | /** One gate's run on the lane: the full tier on `commitOid`, traced, cut off by `controller`. */ | |
| 2141 | async function gateOnce( | |
| 2142 | repo: Repo, | |
| 2143 | pull: Pull, | |
| 2144 | commitOid: string, | |
| 2145 | controller: AbortController, | |
| 2146 | signal: AbortSignal | undefined, | |
| 2147 | ): Promise<GateDone> { | |
| 2148 | const lines: string[] = []; | |
| 2149 | const abandon = (): void => controller.abort(); | |
| 2150 | signal?.addEventListener("abort", abandon, { once: true }); | |
| 2151 | // The slot minutes it spends are the flow's gate minutes (#6426). | |
| 2152 | const traced = beginGate(app.db, repo.id, pull.id, commitOid); | |
| 2153 | try { | |
| 2154 | const bare = await pullDir(app.db, app.reposRoot, repo, pull); | |
| 2155 | const done = await runTier({ | |
| 2156 | bare, | |
| 2157 | repo, | |
| 2158 | id: gateRunId(commitOid), | |
| 2159 | commitOid, | |
| 2160 | pull, | |
| 2161 | tier: "full", | |
| 2162 | write: (line) => lines.push(line), | |
| 2163 | signal: controller.signal, | |
| 2164 | }); | |
| 2165 | if (done.ok) return { ok: true, error: "", lines }; | |
| 2166 | const stage = done.stage ?? "setup"; | |
| 2167 | // Cut off by this forge stopping, the Job lost under it, the check | |
| 2168 | // killed by a signal from outside (the pod going, the OOM killer), or | |
| 2169 | // an install the machine failed twice: the code was not judged, so | |
| 2170 | // this is no red. A gate the drain lets finish is judged (#6611, #6722). | |
| 2171 | if ( | |
| 2172 | controller.signal.aborted || | |
| 2173 | done.error === JOB_VANISHED || | |
| 2174 | done.infrastructure === true || | |
| 2175 | done.killed === true | |
| 2176 | ) | |
| 2177 | return { ok: false, error: done.error, lines, interrupted: true, stage }; | |
| 2178 | return { ok: false, error: done.error, lines, stage }; | |
| 2179 | } finally { | |
| 2180 | endGate(app.db, traced); | |
| 2181 | signal?.removeEventListener("abort", abandon); | |
| 2182 | gates.delete(controller); | |
| 2183 | } | |
| 2184 | } | |
| 2185 | ||
| 2186 | /** | |
| 2187 | * A gate asked of a forge already stopping is not started: the forge that carries on asks it | |
| 2188 | * again. | |
| 2189 | */ | |
| 2190 | function gateRun(repo: Repo, pull: Pull, commitOid: string, signal?: AbortSignal): Promise<GateDone> { | |
| 2191 | if (stopped) return Promise.resolve({ ok: false, error: INTERRUPTED, lines: [], interrupted: true }); | |
| 2192 | const controller = new AbortController(); | |
| 2193 | const done = gateOnce(repo, pull, commitOid, controller, signal); | |
| 2194 | gates.set(controller, { commitOid, done }); | |
| 2195 | return done; | |
| 2196 | } | |
| 2197 | 139 | |
| 2198 | 140 | const handle: ChecksHandle = { |
| 2199 | request, | |
| 2200 | gate: gateRun, | |
| 141 | request: lane.request, | |
| 142 | gate: (repo, pull, commitOid, signal) => gateRun(lane, repo, pull, commitOid, signal), | |
| 2201 | 143 | adopt: (commitOid) => { |
| 2202 | for (const g of gates.values()) if (g.commitOid === commitOid) return g.done; | |
| 144 | for (const g of lane.gates.values()) if (g.commitOid === commitOid) return g.done; | |
| 2203 | 145 | return undefined; |
| 2204 | 146 | }, |
| 2205 | spareSlots: () => Math.max(0, capacity() - PULL_RESERVE - slotted() - extraGates()), | |
| 2206 | verifyDue: (now) => askVerifications(now), | |
| 2207 | lane: gate, | |
| 147 | spareSlots: () => Math.max(0, capacity(lane) - PULL_RESERVE - slotted(lane) - extraGates()), | |
| 148 | verifyDue: (now) => askVerifications(lane, now), | |
| 149 | lane: lane.gate, | |
| 2208 | 150 | stop: async (stopOptions) => { |
| 2209 | stopped = true; | |
| 151 | lane.stopped = true; | |
| 2210 | 152 | clearInterval(timer); |
| 2211 | 153 | if (sweeper !== undefined) clearInterval(sweeper); |
| 2212 | 154 | unsubscribe(); |
| @@ -2216,11 +158,11 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 2216 | 158 | // A merge-queue gate is under way too: its verdict, if it comes in the |
| 2217 | 159 | // drain, is kept for the forge that carries on (#6611). |
| 2218 | 160 | const running = Promise.all([ |
| 2219 | ...[...inFlight.values()].map((f) => f.promise), | |
| 2220 | ...[...gates.values()].map((g) => g.done), | |
| 161 | ...[...lane.inFlight.values()].map((f) => f.promise), | |
| 162 | ...[...lane.gates.values()].map((g) => g.done), | |
| 2221 | 163 | ]); |
| 2222 | 164 | const drainMs = stopOptions?.drain === false ? 0 : (options.drainMs ?? 0); |
| 2223 | if (drainMs > 0 && inFlight.size + gates.size > 0) { | |
| 165 | if (drainMs > 0 && lane.inFlight.size + lane.gates.size > 0) { | |
| 2224 | 166 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 2225 | 167 | await Promise.race([ |
| 2226 | 168 | running, |
| @@ -2230,8 +172,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 2230 | 172 | ]); |
| 2231 | 173 | clearTimeout(timer); |
| 2232 | 174 | } |
| 2233 | for (const { abort } of inFlight.values()) abort.abort(); | |
| 2234 | for (const abort of gates.keys()) abort.abort(); | |
| 175 | for (const { abort } of lane.inFlight.values()) abort.abort(); | |
| 176 | for (const abort of lane.gates.keys()) abort.abort(); | |
| 2235 | 177 | await running; |
| 2236 | 178 | }, |
| 2237 | 179 | }; |
src/checks/start.ts added +51
| @@ -0,0 +1,51 @@ | ||
| 1 | ||
| 2 | import type { Repo } from "../core/repos.ts"; | |
| 3 | import type { Run } from "../core/runs.ts"; | |
| 4 | import { inVerifyLane } from "./verify.ts"; | |
| 5 | import { createLog } from "../kernel/log.ts"; | |
| 6 | import { trackRun, untrackRun } from "../runs/stoppable.ts"; | |
| 7 | import { workBegins, workEnds } from "../runs/ending.ts"; | |
| 8 | import type { LaneState } from "./lane_state.ts"; | |
| 9 | import { execute } from "./execute.ts"; | |
| 10 | import { recheckFinished } from "./recheck_base.ts"; | |
| 11 | ||
| 12 | const log = createLog("checks"); | |
| 13 | ||
| 14 | /** Run a queued check here, whichever forge queued it. */ | |
| 15 | export function start(lane: LaneState, run: Run, repo: Repo): void { | |
| 16 | const { app } = lane; | |
| 17 | if (lane.inFlight.has(run.id)) return; | |
| 18 | lane.waiting.delete(run.id); | |
| 19 | lane.diskWaiting.delete(run.id); | |
| 20 | lane.memoryWaiting.delete(run.id); | |
| 21 | const abort = new AbortController(); | |
| 22 | const low = inVerifyLane(run.env); | |
| 23 | if (low) lane.verifying = run.id; | |
| 24 | // A person's cancel (`cancelRepoRun`) reaches it through this. | |
| 25 | trackRun(run.id, abort); | |
| 26 | workBegins(app.db, run.id); | |
| 27 | const promise = execute(lane, run, repo, abort.signal, !low) | |
| 28 | .catch((error) => | |
| 29 | log.error("check failed to execute", { | |
| 30 | run: run.id, | |
| 31 | error: String(error), | |
| 32 | }), | |
| 33 | ) | |
| 34 | .finally(() => { | |
| 35 | untrackRun(run.id, abort); | |
| 36 | // Every process the check started has ended: a cancel waiting on | |
| 37 | // that is heard now (#13291). | |
| 38 | workEnds(app.db, run.id); | |
| 39 | lane.inFlight.delete(run.id); | |
| 40 | if (lane.verifying === run.id) lane.verifying = undefined; | |
| 41 | // A pull's green that ran while its base moved is of the old tip: asked again once. | |
| 42 | if (run.issueId !== null && !lane.stopped) | |
| 43 | void recheckFinished(lane, run.id, repo).catch((error) => | |
| 44 | log.error("check could not be asked again after the base moved", { run: run.id, error: String(error) }), | |
| 45 | ); | |
| 46 | // The slot this run held is free again: give it to whichever queued | |
| 47 | // check now deserves it most, not whoever asked first. | |
| 48 | lane.schedule(); | |
| 49 | }); | |
| 50 | lane.inFlight.set(run.id, { promise, abort }); | |
| 51 | } |
src/checks/sweep.ts added +138
| @@ -0,0 +1,138 @@ | ||
| 1 | ||
| 2 | import { existsSync, lstatSync, readdirSync } from "node:fs"; | |
| 3 | import { chmod, lstat, readdir, rm } from "node:fs/promises"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import type { App } from "../app.ts"; | |
| 6 | import { getRun, leaseLapsed } from "../core/runs.ts"; | |
| 7 | import { gateHolding } from "./gate_record.ts"; | |
| 8 | import { diskRecovered } from "./disk_red.ts"; | |
| 9 | import { IMPACT_DIR } from "./impact-store.ts"; | |
| 10 | import { KEPT_DIR } from "./kept-store.ts"; | |
| 11 | import { createLog } from "../kernel/log.ts"; | |
| 12 | import { STALE_AFTER_MS } from "../runs/lease.ts"; | |
| 13 | import { CHECK_TIMEOUT_MAX } from "../apps/manifest.ts"; | |
| 14 | import { GATE_DEADLINE_MS } from "./queue.ts"; | |
| 15 | ||
| 16 | ||
| 17 | const log = createLog("checks"); | |
| 18 | ||
| 19 | /** | |
| 20 | * Remove a run's directory even when the cell left read-only files in it (a | |
| 21 | * bun cache is): `rmSync` cannot unlink inside a directory without write | |
| 22 | * permission, so permissions are restored and the removal tried again. A | |
| 23 | * removal that still fails is logged, never swallowed. | |
| 24 | */ | |
| 25 | export async function removeRunDir(path: string): Promise<void> { | |
| 26 | try { | |
| 27 | await rm(path, { recursive: true, force: true }); | |
| 28 | return; | |
| 29 | } catch { | |
| 30 | /* fall through to restore permissions */ | |
| 31 | } | |
| 32 | const open = async (p: string): Promise<void> => { | |
| 33 | try { | |
| 34 | if ((await lstat(p)).isSymbolicLink()) return; | |
| 35 | if ((await lstat(p)).isDirectory()) { | |
| 36 | await chmod(p, 0o700); | |
| 37 | for (const n of await readdir(p)) await open(join(p, n)); | |
| 38 | } | |
| 39 | } catch { | |
| 40 | /* gone, or not ours: the retry says */ | |
| 41 | } | |
| 42 | }; | |
| 43 | await open(path); | |
| 44 | try { | |
| 45 | await rm(path, { recursive: true, force: true }); | |
| 46 | } catch (error) { | |
| 47 | log.error("could not remove a run directory", { path, error: String(error) }); | |
| 48 | } | |
| 49 | } | |
| 50 | ||
| 51 | /** The sweep of the checkouts under way, if any: one at a time. */ | |
| 52 | let sweeping: Promise<void> | undefined; | |
| 53 | ||
| 54 | /** | |
| 55 | * Sweep the checkouts in the background and answer when done. A sweep | |
| 56 | * already running is the one answered, not doubled; a failure is logged. | |
| 57 | */ | |
| 58 | export function sweepSoon(app: App, now: number = Date.now()): Promise<void> { | |
| 59 | sweeping ??= sweepCheckouts(app, now) | |
| 60 | .catch((error) => log.error("could not sweep the checkouts", { error: String(error) })) | |
| 61 | .finally(() => { | |
| 62 | sweeping = undefined; | |
| 63 | }); | |
| 64 | return sweeping; | |
| 65 | } | |
| 66 | ||
| 67 | /** Where a check's checkout lives while it runs. */ | |
| 68 | export const checksRoot = (dataDir: string): string => join(dataDir, "checks"); | |
| 69 | ||
| 70 | /** | |
| 71 | * What is kept under the checks' root that is no checkout: every | |
| 72 | * repository's impact map (`impact-store.ts`) and kept results | |
| 73 | * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`), | |
| 74 | * the gates' logs (`gate_log.ts`) and the last gate's results | |
| 75 | * (`gate_last.ts`). The sweep took each for a stranger's checkout and | |
| 76 | * removed it at every boot and every 30 seconds after: a map lived half a | |
| 77 | * minute, so no check was ever seeded with one (#12178). | |
| 78 | */ | |
| 79 | const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]); | |
| 80 | ||
| 81 | /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */ | |
| 82 | const GATE_DIR = /^gate-([0-9a-f]{12})$/; | |
| 83 | ||
| 84 | /** | |
| 85 | * Whether a gate's checkout, this forge's or a peer's during a roll, is younger than a gate may | |
| 86 | * run. | |
| 87 | */ | |
| 88 | function gateMayBeRunning(path: string, now: number): boolean { | |
| 89 | try { | |
| 90 | return now - lstatSync(path).mtimeMs < GATE_DEADLINE_MS; | |
| 91 | } catch { | |
| 92 | return false; | |
| 93 | } | |
| 94 | } | |
| 95 | ||
| 96 | /** | |
| 97 | * Remove the checkouts nobody is using: one whose run is not running with a | |
| 98 | * live lease, or has no run at all. A forge booting during a roll shares this | |
| 99 | * directory and database with the old one, whose checks are still running in | |
| 100 | * theirs; those are left alone. A gate writes no run: its checkout is left | |
| 101 | * while it is younger than a gate's deadline, and past that while its | |
| 102 | * `check_gate` row is open, for as long as a check may run. What the checks | |
| 103 | * keep between runs ({@link STORES}) is no checkout and is never swept. | |
| 104 | */ | |
| 105 | export async function sweepCheckouts(app: App, now: number = Date.now()): Promise<void> { | |
| 106 | const root = checksRoot(app.config.dataDir); | |
| 107 | if (!existsSync(root)) return; | |
| 108 | const since = new Date(now - CHECK_TIMEOUT_MAX * 1000); | |
| 109 | for (const name of readdirSync(root)) { | |
| 110 | if (STORES.has(name)) continue; | |
| 111 | // A gate has no run row: its checkout is spared until no gate could still be using it. | |
| 112 | if (name.startsWith("gate-") && gateMayBeRunning(join(root, name), now)) continue; | |
| 113 | // `<id>.home` is the home of run `<id>`: it lives and dies with it. | |
| 114 | const id = name.endsWith(".home") ? name.slice(0, -5) : name; | |
| 115 | const gate = GATE_DIR.exec(id); | |
| 116 | if (gate !== null) { | |
| 117 | if (!gateHolding(app.db, gate[1]!, since)) await removeRunDir(join(root, name)); | |
| 118 | continue; | |
| 119 | } | |
| 120 | const run = getRun(app.db, id); | |
| 121 | const live = | |
| 122 | run !== undefined && | |
| 123 | run.state === "running" && | |
| 124 | !leaseLapsed(run, STALE_AFTER_MS, now); | |
| 125 | if (!live) await removeRunDir(join(root, name)); | |
| 126 | } | |
| 127 | } | |
| 128 | ||
| 129 | /** | |
| 130 | * Under the free floor, give back what no check is using before taking more | |
| 131 | * (#6945): checkouts leaked by a crash sat on the disk until the next boot | |
| 132 | * while every check failed red on it. Answers whether it swept. | |
| 133 | */ | |
| 134 | export async function reclaimIfFull(app: App, now: number = Date.now()): Promise<boolean> { | |
| 135 | if (diskRecovered(app.config.dataDir)) return false; | |
| 136 | await sweepCheckouts(app, now); | |
| 137 | return true; | |
| 138 | } |
src/checks/tier_flake.ts added +140
| @@ -0,0 +1,140 @@ | ||
| 1 | ||
| 2 | import { readFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import type { Repo } from "../core/repos.ts"; | |
| 5 | import { runGit } from "../git/env.ts"; | |
| 6 | import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts"; | |
| 7 | import { parseBunTestFailure } from "./bun-test.ts"; | |
| 8 | import { recordCrashFlakes, passedAlone, recordFlakes, rerunCommand, rerunnable, pullOwnsRed } from "./flake.ts"; | |
| 9 | import { childEnv, runStep } from "../apps/process.ts"; | |
| 10 | import type { CellSpec } from "../apps/contain.ts"; | |
| 11 | import type { Result } from "../kernel/result.ts"; | |
| 12 | import type { LaneState } from "./lane_state.ts"; | |
| 13 | import type { Execution } from "./local_runner.ts"; | |
| 14 | import type { CheckManifest } from "../apps/manifest.ts"; | |
| 15 | ||
| 16 | /** The longest a failed test is given to pass alone, in seconds. */ | |
| 17 | const RERUN_TIMEOUT_S = 300; | |
| 18 | ||
| 19 | /** What the flake passes look at once the check step has ended. */ | |
| 20 | export interface FlakeCheck { | |
| 21 | readonly run: { readonly id: string; readonly commitOid: string }; | |
| 22 | readonly repo: Repo; | |
| 23 | readonly bare: string; | |
| 24 | readonly dir: string; | |
| 25 | readonly env: ReturnType<typeof childEnv>; | |
| 26 | readonly cell: CellSpec | undefined; | |
| 27 | readonly manifest: CheckManifest; | |
| 28 | /** The base tip a pull's check merged onto; none for a check of a branch. */ | |
| 29 | readonly tip: string | undefined; | |
| 30 | /** Whether the check ran in a cluster Job, where it is not re-run here. */ | |
| 31 | readonly inJob: boolean; | |
| 32 | readonly checkLines: readonly string[]; | |
| 33 | readonly write: (line: string) => void; | |
| 34 | readonly signal: AbortSignal; | |
| 35 | } | |
| 36 | ||
| 37 | /** | |
| 38 | * What a red check's own log can still excuse: a file filed because it fails only beside others, | |
| 39 | * a failed test that passes alone (a flake, so green), a shard that crashed outside any test. | |
| 40 | * Answers the check step's result, made green when the red was a flake. | |
| 41 | */ | |
| 42 | export async function settleFlakes( | |
| 43 | lane: LaneState, | |
| 44 | x: Execution, | |
| 45 | c: FlakeCheck, | |
| 46 | result: Result<void, string>, | |
| 47 | ): Promise<Result<void, string>> { | |
| 48 | const { app } = lane; | |
| 49 | const { run, repo, bare, dir, env, cell, manifest, tip, inJob, checkLines, write, signal } = c; | |
| 50 | let ran = result; | |
| 51 | // A file the gate cleared because it passed alone is still a failure | |
| 52 | // to track: the cell cannot reach the forge, so it is filed from here. | |
| 53 | try { | |
| 54 | // Only the default branch's own run files: a pull branch controls | |
| 55 | // its check's output, and the issues are authored as the owner. | |
| 56 | const all = | |
| 57 | x.fileOrderDependent === true | |
| 58 | ? clearedFiles(checkLines) | |
| 59 | : []; | |
| 60 | const cleared = all.slice(0, MAX_FILED); | |
| 61 | if (cleared.length > 0) { | |
| 62 | fileOrderDependent(app.db, app.bus, repo, cleared, { | |
| 63 | runId: run.id, | |
| 64 | commit: run.commitOid, | |
| 65 | }); | |
| 66 | write(`filed: ${cleared.length} test file(s) that fail only beside others`); | |
| 67 | if (all.length > cleared.length) | |
| 68 | write(`not filed: ${all.length - cleared.length} more (at most ${MAX_FILED} per run)`); | |
| 69 | } | |
| 70 | } catch (error) { | |
| 71 | write(`order-dependent tests not filed: ${String(error)}`); | |
| 72 | } | |
| 73 | // Main's red costs the forge every pull's check (#6643): one or two | |
| 74 | // failed tests are re-run alone, on this commit in this cell, before | |
| 75 | // main counts as red. Passing alone is a flake: green, and recorded. | |
| 76 | // A pull's red is the same unless the pull touched the test (#7426). | |
| 77 | if (!ran.ok && !inJob && (x.isMain === true || tip !== undefined)) { | |
| 78 | try { | |
| 79 | let tests = rerunnable(checkLines, parseBunTestFailure(checkLines)); | |
| 80 | if (tests !== undefined && x.isMain !== true) { | |
| 81 | const diff = await runGit( | |
| 82 | ["diff", "--name-only", "-z", `${tip}...${x.commitOid}`], | |
| 83 | { cwd: bare }, | |
| 84 | ); | |
| 85 | const changed = diff.ok ? diff.value.split("\0").filter((p) => p !== "") : undefined; | |
| 86 | const read = (path: string): string | undefined => { | |
| 87 | try { | |
| 88 | return readFileSync(join(dir, path), "utf8"); | |
| 89 | } catch { | |
| 90 | return undefined; | |
| 91 | } | |
| 92 | }; | |
| 93 | if (pullOwnsRed(tests, changed, read)) { | |
| 94 | write("not re-run alone: the pull changed a failed test or what it imports"); | |
| 95 | tests = undefined; | |
| 96 | } | |
| 97 | } | |
| 98 | if (tests !== undefined) { | |
| 99 | write(`re-running ${tests.length} failed test(s) alone`); | |
| 100 | let alone = true; | |
| 101 | for (const test of tests) { | |
| 102 | const out: string[] = []; | |
| 103 | const again = await runStep("flake", rerunCommand(test), { | |
| 104 | cwd: dir, | |
| 105 | env, | |
| 106 | log: (line) => { | |
| 107 | out.push(line); | |
| 108 | write(line); | |
| 109 | }, | |
| 110 | signal, | |
| 111 | timeoutMs: Math.min(manifest.timeout, RERUN_TIMEOUT_S) * 1000, | |
| 112 | cell, | |
| 113 | }); | |
| 114 | if (!again.ok || !passedAlone(out)) { | |
| 115 | alone = false; | |
| 116 | write(`"${test.name}" fails alone too: ${x.isMain === true ? "main" : "the check"} is red`); | |
| 117 | break; | |
| 118 | } | |
| 119 | } | |
| 120 | if (alone) { | |
| 121 | recordFlakes(app.db, app.bus, repo, tests, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }); | |
| 122 | for (const t of tests) write(`flake: ${t.file} > ${t.name} passed alone`); | |
| 123 | write(`flaky: ${tests.map((t) => `"${t.name}"`).join(", ")} failed in the full run and passed alone; recorded, ${x.isMain === true ? "main" : "the check"} stays green`); | |
| 124 | ran = { ok: true, value: undefined }; | |
| 125 | } | |
| 126 | } | |
| 127 | } catch (error) { | |
| 128 | write(`flake re-run failed: ${String(error)}`); | |
| 129 | } | |
| 130 | } | |
| 131 | // A shard that crashed outside any test and ran green once more (#7781). | |
| 132 | { | |
| 133 | try { | |
| 134 | recordCrashFlakes(app.db, app.bus, repo, checkLines, { runId: run.id, commit: run.commitOid, pull: x.isMain !== true }, ran.ok); | |
| 135 | } catch (error) { | |
| 136 | write(`crash flakes not recorded: ${String(error)}`); | |
| 137 | } | |
| 138 | } | |
| 139 | return ran; | |
| 140 | } |
src/checks/tier_install.ts added +114
| @@ -0,0 +1,114 @@ | ||
| 1 | ||
| 2 | import { mkdirSync } from "node:fs"; | |
| 3 | import type { Repo } from "../core/repos.ts"; | |
| 4 | import { childEnv, runStep } from "../apps/process.ts"; | |
| 5 | import type { CellSpec } from "../apps/contain.ts"; | |
| 6 | import type { Result } from "../kernel/result.ts"; | |
| 7 | import { cellInstall, checkInstallCache, meteredInstall, installWithRetry, prefillInstallCache, withLock } from "./install.ts"; | |
| 8 | import { evictSoon, lend } from "./install_evict.ts"; | |
| 9 | import type { CheckOutcome } from "./runner.ts"; | |
| 10 | import type { LaneState } from "./lane_state.ts"; | |
| 11 | import type { CheckManifest } from "../apps/manifest.ts"; | |
| 12 | ||
| 13 | /** What an install step is run with. */ | |
| 14 | export interface InstallStep { | |
| 15 | readonly dir: string; | |
| 16 | readonly env: ReturnType<typeof childEnv>; | |
| 17 | readonly cell: CellSpec | undefined; | |
| 18 | readonly repo: Repo; | |
| 19 | readonly runId: string; | |
| 20 | readonly manifest: CheckManifest; | |
| 21 | readonly write: (line: string) => void; | |
| 22 | readonly signal: AbortSignal; | |
| 23 | } | |
| 24 | ||
| 25 | /** | |
| 26 | * The check's install step, here, in its cell or on the shared cache. Answers a failure, else | |
| 27 | * nothing. | |
| 28 | */ | |
| 29 | export async function runInstall(lane: LaneState, c: InstallStep): Promise<CheckOutcome | undefined> { | |
| 30 | const { app, options } = lane; | |
| 31 | const { dir, env, cell, repo, manifest, write, signal } = c; | |
| 32 | const run = { id: c.runId }; | |
| 33 | if (manifest.install !== null) { | |
| 34 | const install = manifest.install; | |
| 35 | // A cell's home is its own cache. Uncontained, the repository's | |
| 36 | // cache is shared, so installs into it take turns (`checks/install.ts`). | |
| 37 | const shared = | |
| 38 | cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined; | |
| 39 | // A cell installs from that same cache, bound read-only (#7543): | |
| 40 | // the forge fills it first, on the host, so no pull writes what | |
| 41 | // the gate on main installs. Cluster jobs (`runJobStep`) are | |
| 42 | // unchanged: a Job has its own cache and never takes this path. | |
| 43 | const warm = cell === undefined ? undefined : checkInstallCache(app.config.dataDir, repo.id); | |
| 44 | // Set once the fill has run: a cell installs from the shared cache | |
| 45 | // only when the forge filled it; otherwise it keeps a cache of its own. | |
| 46 | let filled = false; | |
| 47 | const once = (log: (line: string) => void): Promise<Result<void, string>> => { | |
| 48 | const base = shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared }; | |
| 49 | const from = cellInstall(install, base, filled ? warm : undefined); | |
| 50 | const revealed = cell !== undefined && filled && warm !== undefined ? { ...cell, reveal: [warm] } : cell; | |
| 51 | return runStep("install", from.command, { | |
| 52 | cwd: dir, | |
| 53 | env: from.env, | |
| 54 | log, | |
| 55 | signal, | |
| 56 | timeoutMs: manifest.timeout * 1000, | |
| 57 | cell: revealed, | |
| 58 | }); | |
| 59 | }; | |
| 60 | // What the cell's proxy received, for the one line that shows the cache | |
| 61 | // working (#7546). Uncontained, nothing is logged: the shared cache | |
| 62 | // is not metered, and its growth is wrong under concurrent checks. | |
| 63 | // Stopped in `finally`, so a throw never leaves the label metered; | |
| 64 | // over a MiB from the npm registry adds the miss line (#8382). | |
| 65 | const installed = await meteredInstall( | |
| 66 | cell === undefined ? undefined : `check:${run.id}`, | |
| 67 | repo.id, | |
| 68 | () => | |
| 69 | installWithRetry( | |
| 70 | async (log) => { | |
| 71 | if (shared === undefined) { | |
| 72 | if (warm !== undefined) { | |
| 73 | const fill = await prefillInstallCache({ | |
| 74 | cache: warm, | |
| 75 | dir, | |
| 76 | env, | |
| 77 | log, | |
| 78 | signal, | |
| 79 | timeoutMs: manifest.timeout * 1000, | |
| 80 | }); | |
| 81 | if (!fill.ok) return fill; | |
| 82 | filled = fill.value; | |
| 83 | void evictSoon(warm, process.env); | |
| 84 | } | |
| 85 | return once(log); | |
| 86 | } | |
| 87 | mkdirSync(shared, { recursive: true }); | |
| 88 | const held = await withLock( | |
| 89 | `${shared}.lock`, | |
| 90 | async () => { | |
| 91 | const done = await once(log); | |
| 92 | if (done.ok) lend(shared, dir); | |
| 93 | return done; | |
| 94 | }, | |
| 95 | { signal }, | |
| 96 | ); | |
| 97 | if (held.ok && held.value.ok) void evictSoon(shared, process.env); | |
| 98 | return held.ok ? held.value : held; | |
| 99 | }, | |
| 100 | { log: write, signal, retryMs: options.installRetryMs }, | |
| 101 | ), | |
| 102 | write, | |
| 103 | { fetched: true }, | |
| 104 | ); | |
| 105 | if (!installed.ok) | |
| 106 | return { | |
| 107 | ok: false, | |
| 108 | error: installed.error, | |
| 109 | infrastructure: installed.infrastructure, | |
| 110 | stage: "install", | |
| 111 | }; | |
| 112 | } | |
| 113 | return undefined; | |
| 114 | } |
src/checks/tier_manifest.ts added +106
| @@ -0,0 +1,106 @@ | ||
| 1 | ||
| 2 | import { outsideAgentWaits } from "../origins/guest.ts"; | |
| 3 | import { ASKS_NEXT, asksLine, heldPull } from "./held.ts"; | |
| 4 | import { OUTSIDE_CHECK_LINE, OUTSIDE_NO_GATE, OUTSIDE_UNCONTAINED } from "./outside.ts"; | |
| 5 | import { resolveOid } from "../git/merge.ts"; | |
| 6 | import { readFileAt } from "../apps/checkout.ts"; | |
| 7 | import { parseCheck } from "../apps/manifest.ts"; | |
| 8 | import { cellsAvailable } from "../apps/contain.ts"; | |
| 9 | import type { LaneState } from "./lane_state.ts"; | |
| 10 | import type { Execution } from "./local_runner.ts"; | |
| 11 | import type { CheckManifest } from "../apps/manifest.ts"; | |
| 12 | ||
| 13 | /** The manifest a check runs by, and whether an outside contributor's code is being run. */ | |
| 14 | export type TierManifest = | |
| 15 | | { readonly ok: true; readonly manifest: CheckManifest; readonly outside: boolean } | |
| 16 | | { readonly ok: false; readonly error: string }; | |
| 17 | ||
| 18 | /** | |
| 19 | * The manifest this check runs by. A pull does not choose its own gate: the commands and the | |
| 20 | * timeout come from the base branch (`checks/held.ts`). | |
| 21 | */ | |
| 22 | export async function resolveManifest(lane: LaneState, x: Execution, source: string): Promise<TierManifest> { | |
| 23 | const { app } = lane; | |
| 24 | const { bare, write } = x; | |
| 25 | const [forgeJson, packageJson] = await Promise.all([ | |
| 26 | readFileAt(bare, source, "forge.json"), | |
| 27 | readFileAt(bare, source, "package.json"), | |
| 28 | ]); | |
| 29 | const declared = parseCheck(forgeJson, packageJson); | |
| 30 | if (!declared.ok) | |
| 31 | return { | |
| 32 | ok: false, | |
| 33 | error: | |
| 34 | declared.error.kind === "invalid" | |
| 35 | ? declared.error.message | |
| 36 | : "this commit declares no check", | |
| 37 | }; | |
| 38 | if (declared.value === undefined) | |
| 39 | return { ok: false, error: "this commit declares no check" }; | |
| 40 | // A pull does not choose its own gate: the commands and the timeout come | |
| 41 | // from the base branch's `forge.json` where the pull left it, and so do | |
| 42 | // the hosts its steps may reach and the limits its cell is given | |
| 43 | // (`checks/held.ts`). The branch's own supplies only its install step. | |
| 44 | let manifest = declared.value; | |
| 45 | let baseTip: string | undefined; | |
| 46 | /** | |
| 47 | * An outside agent's pull nobody has approved: its code is checked, inside a cell or not at | |
| 48 | * all. | |
| 49 | */ | |
| 50 | let outside = false; | |
| 51 | if (x.pull !== undefined) { | |
| 52 | baseTip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); | |
| 53 | if (baseTip === undefined) | |
| 54 | return { ok: false, error: "the pull's base branch could not be found" }; | |
| 55 | const [baseJson, basePackage] = await Promise.all([ | |
| 56 | readFileAt(bare, baseTip, "forge.json"), | |
| 57 | readFileAt(bare, baseTip, "package.json"), | |
| 58 | ]); | |
| 59 | const gate = parseCheck(baseJson, basePackage); | |
| 60 | if (!gate.ok) | |
| 61 | return { | |
| 62 | ok: false, | |
| 63 | error: | |
| 64 | gate.error.kind === "invalid" | |
| 65 | ? `${x.pull.baseBranch}: ${gate.error.message}` | |
| 66 | : "the base branch declares no check", | |
| 67 | }; | |
| 68 | const held = heldPull(manifest, baseJson, "check"); | |
| 69 | manifest = held.manifest; | |
| 70 | if (gate.value !== undefined) { | |
| 71 | manifest = { | |
| 72 | ...manifest, | |
| 73 | run: gate.value.run, | |
| 74 | fast: gate.value.fast, | |
| 75 | full: gate.value.full, | |
| 76 | timeout: gate.value.timeout, | |
| 77 | }; | |
| 78 | write( | |
| 79 | `tier ${x.tier} from ${x.pull.baseBranch}@${baseTip.slice(0, 7)}`, | |
| 80 | ); | |
| 81 | // An outside agent's pull nobody has approved is checked at once, with nobody's | |
| 82 | // permission: in a cell, held to that list whatever the forge's egress setting, until a | |
| 83 | // writer approves the pull (`origins/guest.ts`). | |
| 84 | if (outsideAgentWaits(app.db, x.pull)) { | |
| 85 | // Uncontained, a check is a plain process beside the forge's own data: a writer's code | |
| 86 | // may be; an outside contributor's waits for a writer here, and says how to go on. | |
| 87 | if (!(await cellsAvailable()).ok) return { ok: false, error: OUTSIDE_UNCONTAINED }; | |
| 88 | outside = true; | |
| 89 | write(OUTSIDE_CHECK_LINE(x.pull.baseBranch)); | |
| 90 | } | |
| 91 | } else if (outsideAgentWaits(app.db, x.pull)) { | |
| 92 | // No gate on the base branch to hold it to: an outside contributor's pull does not bring | |
| 93 | // its own. | |
| 94 | return { ok: false, error: OUTSIDE_NO_GATE(x.pull.baseBranch) }; | |
| 95 | } | |
| 96 | // What it asked for and does not have yet, said where a refused request is looked for. | |
| 97 | if (held.asks !== undefined) write(`${asksLine(held.asks, x.pull.baseBranch)} ${ASKS_NEXT}`); | |
| 98 | } | |
| 99 | if (x.commandsFrom !== undefined) { | |
| 100 | const from = parseCheck(await readFileAt(bare, x.commandsFrom, "forge.json"), undefined); | |
| 101 | if (from.ok && from.value !== undefined) manifest = { ...manifest, verify: from.value.verify, timeout: from.value.timeout }; | |
| 102 | } | |
| 103 | if (x.tier === "verify" && manifest.verify === null) | |
| 104 | return { ok: false, error: "this commit declares no check.verify" }; | |
| 105 | return { ok: true, manifest, outside }; | |
| 106 | } |
src/checks/tier_runner.ts added +112
| @@ -0,0 +1,112 @@ | ||
| 1 | ||
| 2 | import type { Repo } from "../core/repos.ts"; | |
| 3 | import { noteRunProgress } from "../core/runs.ts"; | |
| 4 | import { runGit } from "../git/env.ts"; | |
| 5 | import { recordProven } from "./proven.ts"; | |
| 6 | import { headDepth } from "./history.ts"; | |
| 7 | import { clearedIn } from "./base_red.ts"; | |
| 8 | import { recordCrashFlakes } from "./flake.ts"; | |
| 9 | import { IMPACT_FILE } from "./impact-store.ts"; | |
| 10 | import { KEPT_FILE } from "./kept-store.ts"; | |
| 11 | import { egressSetting } from "../core/capacity.ts"; | |
| 12 | import { STEP_HOSTS } from "../apps/contain.ts"; | |
| 13 | import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts"; | |
| 14 | import type { CheckOutcome } from "./runner.ts"; | |
| 15 | import type { LaneState } from "./lane_state.ts"; | |
| 16 | import type { Execution } from "./local_runner.ts"; | |
| 17 | import type { CheckManifest } from "../apps/manifest.ts"; | |
| 18 | ||
| 19 | /** What a check handed to a runner is made of. */ | |
| 20 | export interface RunnerCheck { | |
| 21 | readonly run: { readonly id: string; readonly commitOid: string }; | |
| 22 | readonly repo: Repo; | |
| 23 | readonly bare: string; | |
| 24 | readonly manifest: CheckManifest; | |
| 25 | readonly command: string; | |
| 26 | readonly base: string | undefined; | |
| 27 | readonly builtOnBase: string | undefined; | |
| 28 | readonly learned: string; | |
| 29 | readonly tree: Awaited<ReturnType<typeof runGit>>; | |
| 30 | readonly key: string | undefined; | |
| 31 | readonly outside: boolean; | |
| 32 | readonly write: (line: string) => void; | |
| 33 | readonly signal: AbortSignal; | |
| 34 | } | |
| 35 | ||
| 36 | /** | |
| 37 | * Give the check to a runner that joined, if one takes it (`runner_step.ts`, #13652). Answers its | |
| 38 | * verdict, or nothing when it was not taken or answered with something that is not the check's own | |
| 39 | * verdict: it then runs here as it always has. | |
| 40 | */ | |
| 41 | export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck): Promise<CheckOutcome | undefined> { | |
| 42 | const { app, options } = lane; | |
| 43 | const { run, repo, bare, manifest, command, base, builtOnBase, learned, tree, key, outside, write, signal } = c; | |
| 44 | // A runner that joined takes the steps (`runner_step.ts`, #13652): only a check that hands | |
| 45 | // back nothing but its log and its verdict, and only while a runner is connected. Not taken, | |
| 46 | // or an answer that is not the check's own verdict, and it runs here as it always has. | |
| 47 | if (lane.placement.mode === "runner" && x.runId !== undefined && runnerTakes(x)) { | |
| 48 | const lines: string[] = []; | |
| 49 | const sent = await runRunnerStep(app, { | |
| 50 | runId: x.runId, | |
| 51 | repoId: repo.id, | |
| 52 | dir: bare, | |
| 53 | job: { | |
| 54 | run: x.runId, | |
| 55 | repo: `${repo.ownerName}/${repo.name}`, | |
| 56 | commit: run.commitOid, | |
| 57 | ...(base === undefined ? {} : { base }), | |
| 58 | depth: base === undefined ? 1 : await headDepth(bare, base, run.commitOid), | |
| 59 | steps: [ | |
| 60 | ...(manifest.install === null ? [] : [{ name: "install" as const, command: manifest.install }]), | |
| 61 | { name: "check" as const, command }, | |
| 62 | ], | |
| 63 | // Only the check's own variables cross, never the forge's; the runner adds its cell's. | |
| 64 | env: { | |
| 65 | CI: "1", | |
| 66 | FORGE_URL: options.forgeUrl, | |
| 67 | FORGE_CHECK: run.id, | |
| 68 | FORGE_COMMIT: x.commitOid, | |
| 69 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 70 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 71 | ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }), | |
| 72 | FORGE_CELLS: "off", | |
| 73 | }, | |
| 74 | limits: { | |
| 75 | ...(manifest.limits.cpus === undefined ? {} : { cpus: manifest.limits.cpus }), | |
| 76 | ...(manifest.limits.memoryMiB === undefined ? {} : { memoryMiB: manifest.limits.memoryMiB }), | |
| 77 | ...(manifest.limits.diskMiB === undefined ? {} : { diskMiB: manifest.limits.diskMiB }), | |
| 78 | }, | |
| 79 | timeoutS: manifest.timeout, | |
| 80 | egress: [...STEP_HOSTS, ...manifest.egress], | |
| 81 | egressMode: outside ? "enforce" : egressSetting(app.db), | |
| 82 | // An outside contributor's code runs in a cell or not at all, there as here. | |
| 83 | cell: outside ? "required" : "preferred", | |
| 84 | cache: { repo: repo.id, tree: tree.ok ? tree.value.trim() : "", base: builtOnBase ?? "" }, | |
| 85 | }, | |
| 86 | log: write, | |
| 87 | output: lines, | |
| 88 | signal, | |
| 89 | waitMs: lane.placement.waitMs, | |
| 90 | }); | |
| 91 | const verdict = sent.taken ? runnerVerdict(sent, lines, signal.aborted) : sent.why; | |
| 92 | if (typeof verdict === "string") write(`placement: runner; ${verdict}; running here`); | |
| 93 | else { | |
| 94 | if (sent.taken && sent.runner !== "") | |
| 95 | noteRunProgress(app.db, x.runId, { | |
| 96 | placement: "runner", | |
| 97 | node: sent.runner, | |
| 98 | sandbox: { enforced: sent.contained, note: sent.note }, | |
| 99 | }); | |
| 100 | try { | |
| 101 | recordCrashFlakes(app.db, app.bus, repo, lines, { runId: run.id, commit: run.commitOid, pull: true }, verdict.ok); | |
| 102 | } catch (error) { | |
| 103 | write(`crash flakes not recorded: ${String(error)}`); | |
| 104 | } | |
| 105 | if (!verdict.ok) return verdict; | |
| 106 | if (key !== undefined) recordProven(app.db, key, x.id); | |
| 107 | const cleared = clearedIn(lines); | |
| 108 | return { ok: true, ...(cleared === undefined ? {} : { cleared }) }; | |
| 109 | } | |
| 110 | } | |
| 111 | return undefined; | |
| 112 | } |
src/checks/tier_source.ts added +66
| @@ -0,0 +1,66 @@ | ||
| 1 | ||
| 2 | import { setRunBase, setRunCombined } from "../core/runs.ts"; | |
| 3 | import { resolveOid } from "../git/merge.ts"; | |
| 4 | import { mergeForCheck } from "./merged.ts"; | |
| 5 | import { readCombined } from "../git/combined.ts"; | |
| 6 | import type { LaneState } from "./lane_state.ts"; | |
| 7 | import type { Execution } from "./local_runner.ts"; | |
| 8 | ||
| 9 | /** The commit a tier checks, and what it builds on. */ | |
| 10 | export type TierSource = | |
| 11 | | { | |
| 12 | readonly ok: true; | |
| 13 | readonly source: string; | |
| 14 | readonly tip: string | undefined; | |
| 15 | readonly builtOnBase: string | undefined; | |
| 16 | readonly mergeLine: string; | |
| 17 | } | |
| 18 | | { readonly ok: false; readonly error: string }; | |
| 19 | ||
| 20 | /** | |
| 21 | * The commit this tier checks. A pull's fast check is of the pull as it would land, and a kept | |
| 22 | * fork's is of its combined commit; any other check is of the commit asked. | |
| 23 | */ | |
| 24 | export async function resolveSource(lane: LaneState, x: Execution): Promise<TierSource> { | |
| 25 | const { app } = lane; | |
| 26 | const { bare } = x; | |
| 27 | // A pull's fast check is of the pull as it would land: its head merged | |
| 28 | // into the base's tip. The gate comes from that tree, so a pull forked | |
| 29 | // before the base learned something is judged with it. | |
| 30 | let source = x.commitOid; | |
| 31 | let tip: string | undefined; | |
| 32 | // The commit the checked tree builds on: the base tip for a merge made | |
| 33 | // now, the base a combined commit names for a kept fork's pull. The fast | |
| 34 | // tier diffs against it, so a combined built before main moved is not | |
| 35 | // charged with main's later changes. | |
| 36 | let builtOnBase: string | undefined; | |
| 37 | let mergeLine = ""; | |
| 38 | if (x.tier === "fast" && x.pull !== undefined) { | |
| 39 | tip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); | |
| 40 | if (tip === undefined) | |
| 41 | return { ok: false, error: `the base ${x.pull.baseBranch} could not be found` }; | |
| 42 | // A kept fork's pull is checked on its combined commit (#9932): the | |
| 43 | // tree that lands, built on the base tip it names (`combined/N`). | |
| 44 | const combined = x.pull.crossRepo | |
| 45 | ? await readCombined(bare, x.pull.number, x.commitOid) | |
| 46 | : undefined; | |
| 47 | if (combined !== undefined && combined.oid !== x.commitOid) { | |
| 48 | source = combined.oid; | |
| 49 | const builtOn = combined.baseOid === "" ? tip : combined.baseOid; | |
| 50 | if (x.runId !== undefined) { | |
| 51 | setRunBase(app.db, x.runId, builtOn); | |
| 52 | setRunCombined(app.db, x.runId, combined.oid); | |
| 53 | } | |
| 54 | builtOnBase = builtOn; | |
| 55 | mergeLine = `checked as combined with ${x.pull.baseBranch} at ${builtOn.slice(0, 7)} (${combined.oid.slice(0, 7)})`; | |
| 56 | } else { | |
| 57 | const merged = await mergeForCheck(bare, x.pull.baseBranch, tip, x.commitOid); | |
| 58 | if (!merged.ok) return { ok: false, error: merged.error }; | |
| 59 | source = merged.commit; | |
| 60 | if (x.runId !== undefined) setRunBase(app.db, x.runId, tip); | |
| 61 | builtOnBase = tip; | |
| 62 | mergeLine = merged.line; | |
| 63 | } | |
| 64 | } | |
| 65 | return { ok: true, source, tip, builtOnBase, mergeLine }; | |
| 66 | } |
src/checks/verify_lane.ts added +112
| @@ -0,0 +1,112 @@ | ||
| 1 | ||
| 2 | import { getRepoById } from "../core/repos.ts"; | |
| 3 | import { checksForHead, listActive, type Run } from "../core/runs.ts"; | |
| 4 | import { bisectOrigin, inVerifyLane, VERIFY_ENV, VERIFY_EVERY_MS, VERIFY_REASON, verifyDue } from "./verify.ts"; | |
| 5 | import { createLog } from "../kernel/log.ts"; | |
| 6 | import type { LaneState } from "./lane_state.ts"; | |
| 7 | import { start } from "./start.ts"; | |
| 8 | ||
| 9 | const log = createLog("checks"); | |
| 10 | ||
| 11 | /** | |
| 12 | * The verification lane (#7786): one run at a time, forge-wide, beside | |
| 13 | * the check slots rather than in one, so no landing and no pull's check | |
| 14 | * ever waits for it. A bisection step goes before a new verification. | |
| 15 | */ | |
| 16 | export function startVerify(lane: LaneState): void { | |
| 17 | const { app } = lane; | |
| 18 | if (lane.verifying !== undefined) return; | |
| 19 | const queued = listActive(app.db, "check").filter( | |
| 20 | (run) => run.state === "queued" && inVerifyLane(run.env) && !lane.inFlight.has(run.id), | |
| 21 | ); | |
| 22 | const next = | |
| 23 | queued.find((run) => bisectOrigin(run.reason) !== undefined) ?? queued[0]; | |
| 24 | if (next === undefined) return; | |
| 25 | const repo = getRepoById(app.db, next.repoId); | |
| 26 | if (repo !== undefined) start(lane, next, repo); | |
| 27 | } | |
| 28 | ||
| 29 | /** Every verification due now: main moved since the last, and that one is old enough. */ | |
| 30 | export async function askVerifications(lane: LaneState, now = Date.now()): Promise<Run[]> { | |
| 31 | const { app, options } = lane; | |
| 32 | const everyMs = options.verifyEveryMs ?? VERIFY_EVERY_MS; | |
| 33 | const since = new Date(now - 24 * 3_600_000).toISOString(); | |
| 34 | // The newest default-branch check of each repository that had one today. | |
| 35 | const tips = app.db | |
| 36 | .query<{ repo_id: string; commit_oid: string }, [string]>( | |
| 37 | `SELECT r.repo_id, r.commit_oid FROM run r JOIN repo ON repo.id = r.repo_id | |
| 38 | WHERE r.id IN (SELECT MAX(x.id) FROM run x JOIN repo y ON y.id = x.repo_id | |
| 39 | WHERE x.kind = 'check' AND x.issue_id IS NULL | |
| 40 | AND x.env = y.default_branch AND x.created_at >= ?1 | |
| 41 | GROUP BY x.repo_id)`, | |
| 42 | ) | |
| 43 | .all(since); | |
| 44 | const asked: Run[] = []; | |
| 45 | for (const tip of tips) { | |
| 46 | const last = app.db | |
| 47 | .query<{ commit_oid: string; created_at: string }, [string]>( | |
| 48 | `SELECT commit_oid, created_at FROM run | |
| 49 | WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}' | |
| 50 | ORDER BY id DESC LIMIT 1`, | |
| 51 | ) | |
| 52 | .get(tip.repo_id); | |
| 53 | const active = listActive(app.db, "check").some( | |
| 54 | (run) => run.repoId === tip.repo_id && inVerifyLane(run.env), | |
| 55 | ); | |
| 56 | const due = verifyDue({ | |
| 57 | tip: tip.commit_oid, | |
| 58 | last: last === null ? undefined : { commitOid: last.commit_oid, createdAt: last.created_at }, | |
| 59 | active, | |
| 60 | now, | |
| 61 | everyMs, | |
| 62 | }); | |
| 63 | if (!due) continue; | |
| 64 | const repo = getRepoById(app.db, tip.repo_id); | |
| 65 | if (repo === undefined) continue; | |
| 66 | const run = await lane.request(repo, VERIFY_ENV, tip.commit_oid, VERIFY_REASON); | |
| 67 | if (run !== undefined) asked.push(run); | |
| 68 | } | |
| 69 | return asked; | |
| 70 | } | |
| 71 | ||
| 72 | /** | |
| 73 | * An open pull whose head was never checked — the event that should have | |
| 74 | * asked landed while a forge was on its way out, or before this forge | |
| 75 | * existed — is asked for on boot. A pull that stays unchecked would never | |
| 76 | * land on green, however green it is. | |
| 77 | */ | |
| 78 | export async function catchUp(lane: LaneState): Promise<void> { | |
| 79 | const { app } = lane; | |
| 80 | const heads = app.db | |
| 81 | .query< | |
| 82 | { repo_id: string; issue_id: string; number: number; head_oid: string }, | |
| 83 | [] | |
| 84 | >( | |
| 85 | `SELECT issue.repo_id, issue.id AS issue_id, issue.number, pull.head_oid | |
| 86 | FROM pull JOIN issue ON issue.id = pull.issue_id | |
| 87 | WHERE issue.state = 'open' AND pull.merged_at IS NULL | |
| 88 | ORDER BY issue.created_at DESC LIMIT 200`, | |
| 89 | ) | |
| 90 | .all(); | |
| 91 | for (const head of heads) { | |
| 92 | if (head.head_oid === "") continue; | |
| 93 | if (checksForHead(app.db, head.issue_id, head.head_oid).length > 0) | |
| 94 | continue; | |
| 95 | const repo = getRepoById(app.db, head.repo_id); | |
| 96 | if (!repo) continue; | |
| 97 | try { | |
| 98 | await lane.request( | |
| 99 | repo, | |
| 100 | `pull-${head.number}`, | |
| 101 | head.head_oid, | |
| 102 | "boot", | |
| 103 | head.issue_id, | |
| 104 | ); | |
| 105 | } catch (error) { | |
| 106 | log.error("check could not be requested on boot", { | |
| 107 | pull: head.number, | |
| 108 | error: String(error), | |
| 109 | }); | |
| 110 | } | |
| 111 | } | |
| 112 | } |
src/kernel/destroy_reach.ts +2 −2
| @@ -68,8 +68,8 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [ | ||
| 68 | 68 | "src/git/seed_history.ts:prune", |
| 69 | 69 | ]), |
| 70 | 70 | ...unitsOn("scratch under the checks root, per run or commit (checkouts, caches, locks, the install cache): never a forge's or an app's directory", [ |
| 71 | "src/checks/service.ts:startChecks", | |
| 72 | "src/checks/service.ts:removeRunDir", | |
| 71 | "src/checks/run_tier.ts:runTier", | |
| 72 | "src/checks/sweep.ts:removeRunDir", | |
| 73 | 73 | "src/checks/cache.ts:restoreCache", |
| 74 | 74 | "src/checks/cache.ts:keepCache", |
| 75 | 75 | "src/checks/impact-store.ts:seedImpact", |
test/checks-service-split.test.ts added +59
| @@ -0,0 +1,59 @@ | ||
| 1 | /** | |
| 2 | * `src/checks/service.ts` was one 2,240-line closure; it is cut along its seams (#14018). What | |
| 3 | * the cut keeps: every name it was imported by, one module holding the handle registry behind | |
| 4 | * `checksFor`, and no module over 600 lines. | |
| 5 | */ | |
| 6 | ||
| 7 | import { describe, expect, test } from "bun:test"; | |
| 8 | import { readdirSync, readFileSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import type { App } from "../src/app.ts"; | |
| 11 | import * as service from "../src/checks/service.ts"; | |
| 12 | import type { ChecksHandle } from "../src/checks/service.ts"; | |
| 13 | ||
| 14 | const dir = join(import.meta.dir, "..", "src", "checks"); | |
| 15 | const modules = readdirSync(dir).filter((name) => name.endsWith(".ts")); | |
| 16 | ||
| 17 | describe("the checks service is split", () => { | |
| 18 | test("the names service.ts was imported by are still exported from it", () => { | |
| 19 | for (const name of [ | |
| 20 | "startChecks", | |
| 21 | "checksFor", | |
| 22 | "provideChecksForTest", | |
| 23 | "pickNext", | |
| 24 | "holdsLastSlot", | |
| 25 | "requeueAbandonedChecks", | |
| 26 | "removeRunDir", | |
| 27 | "sweepSoon", | |
| 28 | "sweepCheckouts", | |
| 29 | "reclaimIfFull", | |
| 30 | "makeLocalRunner", | |
| 31 | "PULL_RESERVE", | |
| 32 | "LAND_SETTLE_MS", | |
| 33 | "STARVATION_MS", | |
| 34 | ]) | |
| 35 | expect(name in service).toBe(true); | |
| 36 | }); | |
| 37 | ||
| 38 | test("the handle registry behind checksFor lives in exactly one module", () => { | |
| 39 | const holders = modules.filter((name) => /new WeakMap\s*<\s*App\s*,\s*ChecksHandle/.test(readFileSync(join(dir, name), "utf8"))); | |
| 40 | expect(holders).toEqual(["service.ts"]); | |
| 41 | }); | |
| 42 | ||
| 43 | test("checksFor answers the handle a test provided, per app", () => { | |
| 44 | const one = {} as App; | |
| 45 | const other = {} as App; | |
| 46 | const handle = { spareSlots: () => 3 } as ChecksHandle; | |
| 47 | expect(service.checksFor(one)).toBeUndefined(); | |
| 48 | service.provideChecksForTest(one, handle); | |
| 49 | expect(service.checksFor(one)).toBe(handle); | |
| 50 | expect(service.checksFor(other)).toBeUndefined(); | |
| 51 | }); | |
| 52 | ||
| 53 | test("no module of the lane is over 600 lines", () => { | |
| 54 | const long = modules | |
| 55 | .map((name) => [name, readFileSync(join(dir, name), "utf8").split("\n").length] as const) | |
| 56 | .filter(([name, lines]) => lines > 600 && ["service.ts", "run_tier.ts", "scheduler.ts", "event_handler.ts", "execute.ts", "lane_state.ts"].includes(name)); | |
| 57 | expect(long).toEqual([]); | |
| 58 | }); | |
| 59 | }); |
test/timers-named.test.ts +1 −1
| @@ -134,7 +134,7 @@ const ALLOWED: readonly Allowed[] = [ | ||
| 134 | 134 | ["src/looks/thumb.ts", "() => reject("], |
| 135 | 135 | ["src/looks/capture.ts", "() => reject("], |
| 136 | 136 | ["src/actions/search.ts", "() => resolve(\"timeout\")"], |
| 137 | ["src/checks/service.ts", "() => { overran = true; budget.abort(); }"], | |
| 137 | ["src/checks/execute.ts", "() => { overran = true; budget.abort(); }"], | |
| 138 | 138 | ["src/kernel/mail.ts", "() => finish(false)"], |
| 139 | 139 | ["src/apps/domains.ts", "() => reject("], |
| 140 | 140 | ["src/apps/preview_spec.ts", "() => { stop.abort(); reject("], |
new above
offline
Blank first line: add a header comment naming the one concept (what starts a check). Same in every new module.