Public: anyone can read this; the crew can change it.
The merge queue says why a gate went red, and never bisects a merge conflict or an install failure (8-pull batch red in 6 s, then bisected) #5293
mason/forge:mason/5276-the-merge-queue-says-why-a-gate-went-red → main 3 commits fork
7 files
changed +353
−26
src/checks/gate_reason.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | /** | |
| 2 | * Why a full gate on land was not green (#5276), read once from what the | |
| 3 | * gate already has: whether it failed before the check step, whether the | |
| 4 | * check step ran out of time, and the check's own lines. Plain data in, | |
| 5 | * one reason out, and the reason in the queue log's words. | |
| 6 | * | |
| 7 | * `install` is the machine's, not the code's: the queue gates it once more | |
| 8 | * and then calls the gate infrastructure, and no pull goes red for it. | |
| 9 | */ | |
| 10 | ||
| 11 | import { failuresOf } from "./failures.ts"; | |
| 12 | import { parseBunTestFailure } from "./bun-test.ts"; | |
| 13 | ||
| 14 | export type GateReason = | |
| 15 | | { readonly kind: "install" } | |
| 16 | | { readonly kind: "typecheck" } | |
| 17 | | { readonly kind: "tests"; readonly failed: number } | |
| 18 | | { readonly kind: "timeout" } | |
| 19 | /** Red, and the check's lines named neither a test nor a type error. */ | |
| 20 | | { readonly kind: "check" }; | |
| 21 | ||
| 22 | /** What a gate that was not green reports, beside its lines. */ | |
| 23 | export interface GateFailure { | |
| 24 | readonly lines: readonly string[]; | |
| 25 | /** It failed before the check step: the checkout, its history, or install. */ | |
| 26 | readonly setup?: boolean | undefined; | |
| 27 | /** The check step ran past the manifest's timeout. */ | |
| 28 | readonly timedOut?: boolean | undefined; | |
| 29 | } | |
| 30 | ||
| 31 | /** `tsc`'s own error line, in either of its two shapes. */ | |
| 32 | const TYPE_ERROR = /\berror TS\d+:/; | |
| 33 | ||
| 34 | /** | |
| 35 | * An install that failed for the machine (#5093): a racing extraction into | |
| 36 | * a shared cache, a registry that dropped the connection. Read from the | |
| 37 | * lines too, because a cluster Job runs install and check as one step. | |
| 38 | */ | |
| 39 | const INSTALL_BROKE = /Fail extracting tarball|EINTEGRITY|ECONNRESET/; | |
| 40 | ||
| 41 | export function gateReason(failure: GateFailure): GateReason { | |
| 42 | if (failure.setup === true) return { kind: "install" }; | |
| 43 | if (failure.timedOut === true) return { kind: "timeout" }; | |
| 44 | const failed = parseBunTestFailure(failure.lines)?.failed ?? failuresOf(failure.lines).tests.length; | |
| 45 | if (failed > 0) return { kind: "tests", failed }; | |
| 46 | if (failure.lines.some((line) => TYPE_ERROR.test(line))) return { kind: "typecheck" }; | |
| 47 | if (failure.lines.some((line) => INSTALL_BROKE.test(line))) return { kind: "install" }; | |
| 48 | return { kind: "check" }; | |
| 49 | } | |
| 50 | ||
| 51 | /** `install`, `typecheck`, `tests: 3 failed`, `timeout`, `check failed`. */ | |
| 52 | export function reasonWords(reason: GateReason): string { | |
| 53 | switch (reason.kind) { | |
| 54 | case "tests": | |
| 55 | return `tests: ${reason.failed} failed`; | |
| 56 | case "check": | |
| 57 | return "check failed"; | |
| 58 | default: | |
| 59 | return reason.kind; | |
| 60 | } | |
| 61 | } |
src/checks/landing.ts +7 −1
| @@ -25,6 +25,7 @@ import { recordRelease, type Release } from "../release/record.ts"; | ||
| 25 | 25 | import type { Pull } from "../core/pulls.ts"; |
| 26 | 26 | import type { Repo } from "../core/repos.ts"; |
| 27 | 27 | import { failuresOf } from "./failures.ts"; |
| 28 | import { gateReason } from "./gate_reason.ts"; | |
| 28 | 29 | import { keepForgeGate } from "./gate_last.ts"; |
| 29 | 30 | import { checksFor } from "./service.ts"; |
| 30 | 31 | import { landVerdict, type LandVerdict } from "./tiers.ts"; |
| @@ -59,7 +60,12 @@ export async function gateCandidate( | ||
| 59 | 60 | if (done.interrupted === true) return { lands: false, interrupted: true }; |
| 60 | 61 | keepForgeGate(app.config.dataDir, app.config.forgeRepo, `${repo.ownerName}/${repo.name}`, done.lines); |
| 61 | 62 | const failed = failuresOf(done.lines); |
| 62 | return landVerdict(done.ok, failed.tests, failed.files); | |
| 63 | const verdict = landVerdict(done.ok, failed.tests, failed.files); | |
| 64 | if (verdict.lands) return verdict; | |
| 65 | // Every red says why (#5276); an install that broke is the machine's, | |
| 66 | // no verdict on the code, and the queue gates it once more. | |
| 67 | const reason = gateReason(done); | |
| 68 | return reason.kind === "install" ? { lands: false, interrupted: true, reason } : { ...verdict, reason }; | |
| 63 | 69 | } |
| 64 | 70 | |
| 65 | 71 | /** |
src/checks/queue.ts +51 −3
| @@ -41,6 +41,12 @@ | ||
| 41 | 41 | * green alone it rides at the tail of a batch, never at its head, and the |
| 42 | 42 | * search finds it there if it is red on top of others. |
| 43 | 43 | * |
| 44 | * A gate says why it went red (#5276), and only the code's reds are | |
| 45 | * searched. A pull that conflicts with a step before it is left out of the | |
| 46 | * candidate as it is built, named with that step in the log, and the rest | |
| 47 | * gate at once. A gate whose install broke is gated once more, and a second | |
| 48 | * is the gate's infrastructure: no verdict, no one handed back, no search. | |
| 49 | * | |
| 44 | 50 | * What is queued is not stored: it is the pulls `landIfGreen` admitted, and |
| 45 | 51 | * every sweep admits them again. The queue only orders them and remembers |
| 46 | 52 | * what is at the gate and what its searches found, for the tick line and |
| @@ -51,6 +57,7 @@ import type { Pull } from "../core/pulls.ts"; | ||
| 51 | 57 | import { runGit } from "../git/env.ts"; |
| 52 | 58 | import { mergeable, resolveOid, treeOf } from "../git/merge.ts"; |
| 53 | 59 | import { createLog } from "../kernel/log.ts"; |
| 60 | import { reasonWords } from "./gate_reason.ts"; | |
| 54 | 61 | import type { LandVerdict } from "./tiers.ts"; |
| 55 | 62 | |
| 56 | 63 | /** The base a step must land on, and the tree it must write. */ |
| @@ -388,10 +395,21 @@ function hhmm(ms: number): string { | ||
| 388 | 395 | return new Date(ms).toISOString().slice(11, 16); |
| 389 | 396 | } |
| 390 | 397 | |
| 391 | /** A gate's verdict in a word, for the log. */ | |
| 398 | /** | |
| 399 | * A gate's verdict in a few words, for the log: a red always with its | |
| 400 | * reason (#5276) — `red (tests: 3 failed)` — and a gate whose install | |
| 401 | * broke twice `infrastructure (install)`. | |
| 402 | */ | |
| 392 | 403 | function outcome(verdict: LandVerdict): string { |
| 393 | 404 | if (verdict.lands) return "green"; |
| 394 | return "interrupted" in verdict ? "cut off" : "red"; | |
| 405 | if ("interrupted" in verdict) | |
| 406 | return verdict.reason === undefined ? "cut off" : `infrastructure (${reasonWords(verdict.reason)})`; | |
| 407 | return `red (${reasonWords(verdict.reason ?? { kind: "check" })})`; | |
| 408 | } | |
| 409 | ||
| 410 | /** The gate failed for the machine, not the code (#5093): it is gated once more. */ | |
| 411 | function infrastructure(verdict: LandVerdict): boolean { | |
| 412 | return "interrupted" in verdict && verdict.reason !== undefined; | |
| 395 | 413 | } |
| 396 | 414 | |
| 397 | 415 | /** Gate runs a repository's line has made since boot. */ |
| @@ -442,7 +460,13 @@ function counted(line: Line, gate: GateCandidate): Gated { | ||
| 442 | 460 | else signal?.addEventListener("abort", dropped, { once: true }); |
| 443 | 461 | }); |
| 444 | 462 | try { |
| 445 | const verdict = await Promise.race([gate(candidate, abandon.signal), cut]); | |
| 463 | let verdict = await Promise.race([gate(candidate, abandon.signal), cut]); | |
| 464 | if (infrastructure(verdict)) { | |
| 465 | // An install that broke is no red (#5276): once more, and a second | |
| 466 | // is the gate's infrastructure — no one is handed back for it. | |
| 467 | log.warn(`gate ${outcome(verdict)}: gating once more`, { repo: line.repoId, pulls }); | |
| 468 | verdict = await Promise.race([gate(candidate, abandon.signal), cut]); | |
| 469 | } | |
| 446 | 470 | log.info(`gated ${pullsWord(n)}: ${outcome(verdict)}`, { |
| 447 | 471 | repo: line.repoId, |
| 448 | 472 | pulls, |
| @@ -887,6 +911,29 @@ async function land(candidate: Candidate, steps: readonly Waiting[]): Promise<bo | ||
| 887 | 911 | } |
| 888 | 912 | } |
| 889 | 913 | |
| 914 | /** | |
| 915 | * A pull that conflicts with a step before it is left out of the candidate | |
| 916 | * and the rest gate at once (#5276): no gate is spent on a merge, and no | |
| 917 | * one is red for it. Said in the log by the pair, the earlier step found | |
| 918 | * as the first whose tip it no longer merges onto. Its answer is that it | |
| 919 | * did not land; once the earlier one has, the next sweep finds it | |
| 920 | * conflicting with the base and hands it to its builder to bring up to date. | |
| 921 | */ | |
| 922 | async function leftOut(dir: string, baseOid: string, steps: readonly Step[], entry: Queued): Promise<void> { | |
| 923 | let clash: Step | undefined; | |
| 924 | if ((await mergeable(dir, baseOid, entry.pull.headOid)).ok) { | |
| 925 | for (const step of steps) { | |
| 926 | if ((await mergeable(dir, step.commitOid, entry.pull.headOid)).ok) continue; | |
| 927 | clash = step; | |
| 928 | break; | |
| 929 | } | |
| 930 | } | |
| 931 | const b = `#${entry.pull.number}`; | |
| 932 | log.info(`left out: merge-conflict (${clash === undefined ? b : `#${clash.entry.pull.number} × ${b}`})`, { | |
| 933 | repo: entry.pull.repoId, | |
| 934 | }); | |
| 935 | } | |
| 936 | ||
| 890 | 937 | /** |
| 891 | 938 | * The candidate of `batch` on its base's tip now — or on `on`'s tip, to |
| 892 | 939 | * gate it ahead of `on` landing: each pull's squash on the step before it. |
| @@ -906,6 +953,7 @@ async function build(batch: readonly Waiting[], on?: Candidate): Promise<Candida | ||
| 906 | 953 | const steps: Step[] = []; |
| 907 | 954 | for (const { entry } of batch) { |
| 908 | 955 | const merged = await mergeable(dir, tipOid, entry.pull.headOid); |
| 956 | if (!merged.ok && merged.error.kind === "conflict") await leftOut(dir, baseOid, steps, entry); | |
| 909 | 957 | if (!merged.ok || merged.value.treeOid === treeOid) continue; |
| 910 | 958 | const commit = await runGit( |
| 911 | 959 | ["commit-tree", merged.value.treeOid, "-p", tipOid, "-m", `merge queue: #${entry.pull.number}`], |
src/checks/service.ts +46 −18
| @@ -135,6 +135,37 @@ export interface ChecksOptions { | ||
| 135 | 135 | readonly installRetryMs?: number | undefined; |
| 136 | 136 | } |
| 137 | 137 | |
| 138 | /** | |
| 139 | * What a tier ran to. A red one says, for the merge queue's reason (#5276), | |
| 140 | * whether it failed before the check step (`setup`: the checkout, its | |
| 141 | * history, install), whether the check step ran out of time, and whether | |
| 142 | * the install broke for the machine (`infrastructure`). | |
| 143 | */ | |
| 144 | type TierOutcome = | |
| 145 | | { ok: true; proven?: string } | |
| 146 | | { | |
| 147 | ok: false; | |
| 148 | error: string; | |
| 149 | detail?: CheckDetail | undefined; | |
| 150 | setup?: boolean | undefined; | |
| 151 | timedOut?: boolean | undefined; | |
| 152 | /** The install failed twice for the machine, not the code (#5093). */ | |
| 153 | infrastructure?: boolean | undefined; | |
| 154 | }; | |
| 155 | ||
| 156 | /** What `runStep` and a Job say of a check step past its timeout. */ | |
| 157 | const CHECK_TIMED_OUT = /^check took longer than \d+s$/; | |
| 158 | ||
| 159 | /** A full gate's answer: its lines, and what `TierOutcome` says of a red. */ | |
| 160 | export interface GateDone { | |
| 161 | ok: boolean; | |
| 162 | error: string; | |
| 163 | lines: string[]; | |
| 164 | interrupted?: boolean; | |
| 165 | setup?: boolean | undefined; | |
| 166 | timedOut?: boolean | undefined; | |
| 167 | } | |
| 168 | ||
| 138 | 169 | export interface ChecksHandle { |
| 139 | 170 | /** Ask for a check of one commit; the answer is the run, or nothing when the commit declares no check. */ |
| 140 | 171 | request( |
| @@ -158,7 +189,7 @@ export interface ChecksHandle { | ||
| 158 | 189 | pull: Pull, |
| 159 | 190 | commitOid: string, |
| 160 | 191 | signal?: AbortSignal, |
| 161 | ): Promise<{ ok: boolean; error: string; lines: string[]; interrupted?: boolean }>; | |
| 192 | ): Promise<GateDone>; | |
| 162 | 193 | /** |
| 163 | 194 | * Stop checking. Checks under way get `drainMs` to finish unless `drain` |
| 164 | 195 | * is false (a person's Ctrl-C): then they are cut off and queued again now. |
| @@ -633,10 +664,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 633 | 664 | repo: Repo, |
| 634 | 665 | write: (line: string) => void, |
| 635 | 666 | signal: AbortSignal, |
| 636 | ): Promise< | |
| 637 | | { ok: true; proven?: string } | |
| 638 | | { ok: false; error: string; detail?: CheckDetail | undefined; infrastructure?: boolean } | |
| 639 | > { | |
| 667 | ): Promise<TierOutcome> { | |
| 640 | 668 | const bare = await runDir(app.db, app.reposRoot, repo, run.issueId); |
| 641 | 669 | // A pull's check is the fast tier, judged against where the pull left |
| 642 | 670 | // the base; anything else is the whole check (`apps/manifest.ts`). |
| @@ -681,12 +709,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 681 | 709 | readonly signal: AbortSignal; |
| 682 | 710 | } |
| 683 | 711 | |
| 684 | async function runTier( | |
| 685 | x: Execution, | |
| 686 | ): Promise< | |
| 687 | | { ok: true; proven?: string } | |
| 688 | | { ok: false; error: string; detail?: CheckDetail | undefined; infrastructure?: boolean } | |
| 689 | > { | |
| 712 | async function runTier(x: Execution): Promise<TierOutcome> { | |
| 690 | 713 | const { bare, repo, write, signal } = x; |
| 691 | 714 | // A pull's fast check is of the pull as it would land: its head merged |
| 692 | 715 | // into the base's tip. The gate comes from that tree, so a pull forked |
| @@ -697,7 +720,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 697 | 720 | if (x.tier === "fast" && x.pull !== undefined) { |
| 698 | 721 | tip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); |
| 699 | 722 | if (tip === undefined) |
| 700 | return { ok: false, error: `the base ${x.pull.baseBranch} could not be found` }; | |
| 723 | return { ok: false, error: `the base ${x.pull.baseBranch} could not be found`, setup: true }; | |
| 701 | 724 | const merged = await mergeForCheck(bare, x.pull.baseBranch, tip, x.commitOid); |
| 702 | 725 | if (!merged.ok) return { ok: false, error: merged.error }; |
| 703 | 726 | source = merged.commit; |
| @@ -729,7 +752,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 729 | 752 | if (x.pull !== undefined) { |
| 730 | 753 | baseTip = await resolveOid(bare, `refs/heads/${x.pull.baseBranch}`); |
| 731 | 754 | if (baseTip === undefined) |
| 732 | return { ok: false, error: "the pull's base branch could not be found" }; | |
| 755 | return { ok: false, error: "the pull's base branch could not be found", setup: true }; | |
| 733 | 756 | const [baseJson, basePackage] = await Promise.all([ |
| 734 | 757 | readFileAt(bare, baseTip, "forge.json"), |
| 735 | 758 | readFileAt(bare, baseTip, "package.json"), |
| @@ -793,12 +816,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 793 | 816 | mkdirSync(dir, { recursive: true }); |
| 794 | 817 | write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine); |
| 795 | 818 | const exported = await exportTree(bare, run.commitOid, dir); |
| 796 | if (!exported.ok) return { ok: false, error: exported.error }; | |
| 819 | if (!exported.ok) return { ok: false, error: exported.error, setup: true }; | |
| 797 | 820 | if (base !== undefined) { |
| 798 | 821 | const given = await giveHistory(bare, dir, base, run.commitOid); |
| 799 | 822 | if (!given.ok) { |
| 800 | 823 | rmSync(dir, { recursive: true, force: true }); |
| 801 | return { ok: false, error: given.error }; | |
| 824 | return { ok: false, error: given.error, setup: true }; | |
| 802 | 825 | } |
| 803 | 826 | } |
| 804 | 827 | // What the base's own check left (`checks/cache.ts`): tsc's build info |
| @@ -935,7 +958,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 935 | 958 | { log: write, signal, retryMs: options.installRetryMs }, |
| 936 | 959 | ); |
| 937 | 960 | if (!installed.ok) |
| 938 | return { ok: false, error: installed.error, infrastructure: installed.infrastructure }; | |
| 961 | return { | |
| 962 | ok: false, | |
| 963 | error: installed.error, | |
| 964 | setup: true, | |
| 965 | infrastructure: installed.infrastructure, | |
| 966 | }; | |
| 939 | 967 | } |
| 940 | 968 | // The check step's own lines are kept beside what `write` already |
| 941 | 969 | // logs, so a failure can be summarised once here — never by reading |
| @@ -984,7 +1012,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 984 | 1012 | checkLines, |
| 985 | 1013 | ran.error, |
| 986 | 1014 | ); |
| 987 | return { ok: false, error: summary, detail }; | |
| 1015 | return { ok: false, error: summary, detail, timedOut: CHECK_TIMED_OUT.test(ran.error) }; | |
| 988 | 1016 | } |
| 989 | 1017 | // Only a green check of the default branch is kept for the next. |
| 990 | 1018 | if ( |
| @@ -1226,7 +1254,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 1226 | 1254 | done.infrastructure === true |
| 1227 | 1255 | ) |
| 1228 | 1256 | return { ok: false, error: done.error, lines, interrupted: true }; |
| 1229 | return { ok: false, error: done.error, lines }; | |
| 1257 | return { ok: false, error: done.error, lines, setup: done.setup, timedOut: done.timedOut }; | |
| 1230 | 1258 | } finally { |
| 1231 | 1259 | signal?.removeEventListener("abort", abandon); |
| 1232 | 1260 | gates.delete(controller); |
src/checks/tiers.ts +10 −3
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | */ |
| 11 | 11 | |
| 12 | 12 | import { GATE_FILE, needsFull, reach, reachLine, type Impact } from "./impact.ts"; |
| 13 | import type { GateReason } from "./gate_reason.ts"; | |
| 13 | 14 | |
| 14 | 15 | /** A test under this many seconds is always in the fast tier. */ |
| 15 | 16 | export const QUICK_SECONDS = 1; |
| @@ -250,15 +251,21 @@ export type Judged = | ||
| 250 | 251 | readonly comment: string; |
| 251 | 252 | /** Joins the pull's fast set. */ |
| 252 | 253 | readonly learn: readonly string[]; |
| 254 | /** Why it was red, for the queue's log (#5276). */ | |
| 255 | readonly reason?: GateReason; | |
| 253 | 256 | }; |
| 254 | 257 | |
| 255 | 258 | /** |
| 256 | 259 | * What the gate on land answers: judged, or interrupted — the gate did not |
| 257 | 260 | * finish for a reason that is not the code (the forge stopped under it, the |
| 258 | * cluster lost its Job). Interrupted is no verdict; the queue gates it | |
| 259 | * again (#3155). | |
| 261 | * cluster lost its Job, its install broke). Interrupted is no verdict; the | |
| 262 | * queue gates it again (#3155). `reason` is said when the gate ran and | |
| 263 | * failed for the machine: an install, gated once more before the queue | |
| 264 | * calls it infrastructure (#5276). | |
| 260 | 265 | */ |
| 261 | export type LandVerdict = Judged | { readonly lands: false; readonly interrupted: true }; | |
| 266 | export type LandVerdict = | |
| 267 | | Judged | |
| 268 | | { readonly lands: false; readonly interrupted: true; readonly reason?: GateReason }; | |
| 262 | 269 | |
| 263 | 270 | /** |
| 264 | 271 | * What the full gate on the merge with its base decides. Green lands the |
test/gate-reason.test.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | /** Why a full gate on land was not green, in the queue log's words (#5276). */ | |
| 2 | import { describe, expect, test } from "bun:test"; | |
| 3 | import { gateReason, reasonWords } from "../src/checks/gate_reason.ts"; | |
| 4 | ||
| 5 | describe("a gate's reason", () => { | |
| 6 | test("a failure before the check step is install, whatever its lines", () => { | |
| 7 | expect(gateReason({ lines: ["(fail) x"], setup: true })).toEqual({ kind: "install" }); | |
| 8 | }); | |
| 9 | ||
| 10 | test("a check step past its timeout is timeout", () => { | |
| 11 | expect(gateReason({ lines: ["test/a.test.ts:", "(fail) slow"], timedOut: true })).toEqual({ kind: "timeout" }); | |
| 12 | }); | |
| 13 | ||
| 14 | test("bun test's tail counts the failed tests", () => { | |
| 15 | const lines = ["test/a.test.ts:", "(fail) one [1.00ms]", "(fail) two", " 10 pass", " 2 fail"]; | |
| 16 | expect(gateReason({ lines })).toEqual({ kind: "tests", failed: 2 }); | |
| 17 | }); | |
| 18 | ||
| 19 | test("named failures without a tail are counted by name", () => { | |
| 20 | expect(gateReason({ lines: ["test/a.test.ts:", "(fail) one"] })).toEqual({ kind: "tests", failed: 1 }); | |
| 21 | }); | |
| 22 | ||
| 23 | test("tsc's errors in either shape are typecheck", () => { | |
| 24 | expect(gateReason({ lines: ["src/a.ts(3,1): error TS2322: no"] })).toEqual({ kind: "typecheck" }); | |
| 25 | expect(gateReason({ lines: ["src/a.ts:3:1 - error TS2304: no"] })).toEqual({ kind: "typecheck" }); | |
| 26 | }); | |
| 27 | ||
| 28 | test("a broken install read from a Job's lines is install", () => { | |
| 29 | expect(gateReason({ lines: ['error: Fail extracting tarball for "x"'] })).toEqual({ kind: "install" }); | |
| 30 | expect(gateReason({ lines: ["npm ERR! code EINTEGRITY"] })).toEqual({ kind: "install" }); | |
| 31 | }); | |
| 32 | ||
| 33 | test("anything else is the check's own red", () => { | |
| 34 | expect(gateReason({ lines: ["lint: 3 problems"] })).toEqual({ kind: "check" }); | |
| 35 | }); | |
| 36 | ||
| 37 | test("words", () => { | |
| 38 | expect(reasonWords({ kind: "tests", failed: 3 })).toBe("tests: 3 failed"); | |
| 39 | expect(reasonWords({ kind: "install" })).toBe("install"); | |
| 40 | expect(reasonWords({ kind: "typecheck" })).toBe("typecheck"); | |
| 41 | expect(reasonWords({ kind: "timeout" })).toBe("timeout"); | |
| 42 | expect(reasonWords({ kind: "check" })).toBe("check failed"); | |
| 43 | }); | |
| 44 | }); |
test/merge-queue.test.ts +134 −1
| @@ -20,6 +20,7 @@ import { getPull, type Pull } from "../src/core/pulls.ts"; | ||
| 20 | 20 | import { getRepoById, type Repo } from "../src/core/repos.ts"; |
| 21 | 21 | import { claimRun, createRun, finishRun } from "../src/core/runs.ts"; |
| 22 | 22 | import { createUser, type User } from "../src/core/users.ts"; |
| 23 | import { mergeable } from "../src/git/merge.ts"; | |
| 23 | 24 | import { landIfGreen, REGATE_RED_MS, regateDue, sweepGreen } from "../src/crew/autonomy.ts"; |
| 24 | 25 | import { listComments } from "../src/core/issues.ts"; |
| 25 | 26 | import { readQueues } from "../src/flow/queue.ts"; |
| @@ -44,6 +45,8 @@ describe("the merge queue", () => { | ||
| 44 | 45 | let cutOff = 0; |
| 45 | 46 | /** How many gates from now go red whatever the tree, as a machine's red would. */ |
| 46 | 47 | let flaky = 0; |
| 48 | /** How many gates from now fail at install, before any test runs (#5276). */ | |
| 49 | let broken = 0; | |
| 47 | 50 | /** The signal the last gate was handed: the queue aborts it at the deadline. */ |
| 48 | 51 | let lastSignal: AbortSignal | undefined; |
| 49 | 52 | |
| @@ -75,6 +78,11 @@ describe("the merge queue", () => { | ||
| 75 | 78 | cutOff--; |
| 76 | 79 | return { ok: false, error: "aborted", lines: ["(fail) killed [1.00ms]"], interrupted: true }; |
| 77 | 80 | } |
| 81 | if (broken > 0) { | |
| 82 | broken--; | |
| 83 | gated.push({ repo: repo.id, commit: commitOid, ok: false }); | |
| 84 | return { ok: false, error: "install exited with code 1", lines: ["error: Fail extracting tarball"], setup: true }; | |
| 85 | } | |
| 78 | 86 | const files = git(repo, "ls-tree", "-r", "--name-only", commitOid).split("\n"); |
| 79 | 87 | const ok = flaky > 0 ? (flaky--, false) : !files.some((f) => f.startsWith("red-")); |
| 80 | 88 | gated.push({ repo: repo.id, commit: commitOid, ok }); |
| @@ -225,6 +233,8 @@ describe("the merge queue", () => { | ||
| 225 | 233 | // The eight are red; halving finds the culprit (4, 2, 1); then the six |
| 226 | 234 | // left of the eight and two of the four behind are one candidate. |
| 227 | 235 | expect(sizes).toEqual([8, 4, 2, 1, 8, 2]); |
| 236 | // Every red says why (#5276). | |
| 237 | expect(said.some((l) => l.includes("[queue] gated 8 pulls: red (tests: 1 failed)"))).toBe(true); | |
| 228 | 238 | for (const [i, pull] of pulls.entries()) expect(merged(repo, pull)).toBe(i !== 1); |
| 229 | 239 | expect(latestDecision(app.db, pulls[1]!.id, "land")?.answer).toBe("red"); |
| 230 | 240 | }, SLOW); |
| @@ -321,12 +331,135 @@ describe("the merge queue", () => { | ||
| 321 | 331 | claimRun(app.db, check.id); |
| 322 | 332 | finishRun(app.db, check.id, { state: "ok", summary: "check passed" }); |
| 323 | 333 | const last = await greenPull(repo, "last.ts"); |
| 324 | expect(await sweepGreen(app, repo)).toBe(2); | |
| 334 | const said: string[] = []; | |
| 335 | const info = console.info; | |
| 336 | console.info = (line: string) => said.push(line); | |
| 337 | try { | |
| 338 | expect(await sweepGreen(app, repo)).toBe(2); | |
| 339 | } finally { | |
| 340 | console.info = info; | |
| 341 | } | |
| 325 | 342 | expect(gatesOf(repo)).toHaveLength(1); |
| 326 | 343 | expect(merged(repo, first)).toBe(true); |
| 327 | 344 | expect(merged(repo, last)).toBe(true); |
| 328 | 345 | expect(merged(repo, other)).toBe(false); |
| 329 | 346 | expect(tree(repo, "refs/heads/main")).toBe(tree(repo, gatesOf(repo)[0]!.commit)); |
| 347 | // Said by the pair, and no one is red for it (#5276). | |
| 348 | expect(said.some((l) => l.includes(`[queue] left out: merge-conflict (#${first.number} × #${other.number})`))).toBe(true); | |
| 349 | expect(said.some((l) => l.includes("[queue] gated 2 pulls: green"))).toBe(true); | |
| 350 | expect(said.some((l) => l.includes(": red"))).toBe(false); | |
| 351 | expect(latestDecision(app.db, other.id, "land")?.answer).not.toBe("red"); | |
| 352 | // Back to its builder: with the earlier one landed it conflicts with | |
| 353 | // main, and the sweep's conflict path, not the gate, takes it from here. | |
| 354 | const main = git(repo, "rev-parse", "refs/heads/main"); | |
| 355 | const onMain = await mergeable(repoDir(app.reposRoot, repo.ownerName, repo.name), main, other.headOid); | |
| 356 | expect(onMain.ok ? "clean" : onMain.error.kind).toBe("conflict"); | |
| 357 | const before = gateRuns(repo.id); | |
| 358 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 359 | expect(gateRuns(repo.id)).toBe(before); | |
| 360 | }); | |
| 361 | ||
| 362 | test("a batch where #b conflicts with #a lands everything else in one gate, and #b goes back for update (#5276)", async () => { | |
| 363 | const repo = await autonomousRepo("conflict-mid"); | |
| 364 | const a = await greenPull(repo, "a.ts"); | |
| 365 | const pushed = await commitToBranch(app, alice, repo, { | |
| 366 | branch: "feature/b", | |
| 367 | base: "main", | |
| 368 | message: "b", | |
| 369 | files: [{ path: "a.ts", content: "export const n = 'b';\n" }], | |
| 370 | }); | |
| 371 | if (!pushed.ok) throw new Error(JSON.stringify(pushed.error)); | |
| 372 | const opened = await openPull(app, alice, repo, { title: "b", baseBranch: "main", headBranch: "feature/b" }); | |
| 373 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 374 | const b = getPull(app.db, repo.id, opened.value.number)!; | |
| 375 | const check = createRun(app.db, { | |
| 376 | repoId: repo.id, | |
| 377 | kind: "check", | |
| 378 | issueId: b.id, | |
| 379 | env: `pull-${b.number}`, | |
| 380 | commitOid: b.headOid, | |
| 381 | reason: "pull", | |
| 382 | }); | |
| 383 | claimRun(app.db, check.id); | |
| 384 | finishRun(app.db, check.id, { state: "ok", summary: "check passed" }); | |
| 385 | const rest = []; | |
| 386 | for (const name of ["c.ts", "d.ts", "e.ts"]) rest.push(await greenPull(repo, name)); | |
| 387 | const told: number[] = []; | |
| 388 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 389 | const gates = gateRuns(repo.id); | |
| 390 | const answers = await enqueue( | |
| 391 | repo.id, | |
| 392 | [a, b, ...rest].map((pull) => { | |
| 393 | const admitted = { pull, dir, release: () => {}, handBack: () => told.push(pull.number) }; | |
| 394 | return { | |
| 395 | ...admitted, | |
| 396 | land: async (expect: { baseOid: string; treeOid: string }) => { | |
| 397 | const tip = git(repo, "commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", pull.title); | |
| 398 | git(repo, "update-ref", "refs/heads/main", tip, expect.baseOid); | |
| 399 | return tip; | |
| 400 | }, | |
| 401 | }; | |
| 402 | }), | |
| 403 | async () => ({ lands: true }), | |
| 404 | ); | |
| 405 | expect(answers).toEqual([true, false, true, true, true]); | |
| 406 | expect(gateRuns(repo.id) - gates).toBe(1); | |
| 407 | expect(told).toEqual([]); | |
| 408 | expect(git(repo, "ls-tree", "--name-only", "refs/heads/main").split("\n")).toEqual( | |
| 409 | expect.arrayContaining(["a.ts", "c.ts", "d.ts", "e.ts"]), | |
| 410 | ); | |
| 411 | // Back to its builder: with the earlier one landed it conflicts with | |
| 412 | // main, and the sweep's conflict path, not the gate, takes it from here. | |
| 413 | const main = git(repo, "rev-parse", "refs/heads/main"); | |
| 414 | const onMain = await mergeable(repoDir(app.reposRoot, repo.ownerName, repo.name), main, b.headOid); | |
| 415 | expect(onMain.ok ? "clean" : onMain.error.kind).toBe("conflict"); | |
| 416 | const before = gateRuns(repo.id); | |
| 417 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 418 | expect(gateRuns(repo.id)).toBe(before); | |
| 419 | }); | |
| 420 | ||
| 421 | test("an install that breaks is gated once more, and the batch lands: no red, no search (#5276)", async () => { | |
| 422 | const repo = await autonomousRepo("install-once"); | |
| 423 | const pulls = []; | |
| 424 | for (const name of ["a.ts", "b.ts", "c.ts"]) pulls.push(await greenPull(repo, name)); | |
| 425 | broken = 1; | |
| 426 | const warned: string[] = []; | |
| 427 | const warn = console.warn; | |
| 428 | console.warn = (line: string) => warned.push(line); | |
| 429 | try { | |
| 430 | expect(await sweepGreen(app, repo)).toBe(3); | |
| 431 | } finally { | |
| 432 | console.warn = warn; | |
| 433 | } | |
| 434 | expect(broken).toBe(0); | |
| 435 | expect(gatesOf(repo)).toHaveLength(2); | |
| 436 | expect(warned.some((l) => l.includes("[queue] gate infrastructure (install): gating once more"))).toBe(true); | |
| 437 | for (const pull of pulls) expect(merged(repo, pull)).toBe(true); | |
| 438 | }); | |
| 439 | ||
| 440 | test("an install that breaks twice is the gate's infrastructure: no pull goes red, and the next sweep lands (#5276)", async () => { | |
| 441 | const repo = await autonomousRepo("install-twice"); | |
| 442 | const pulls = []; | |
| 443 | for (const name of ["a.ts", "b.ts", "c.ts", "d.ts"]) pulls.push(await greenPull(repo, name)); | |
| 444 | broken = 2; | |
| 445 | const said: string[] = []; | |
| 446 | const info = console.info; | |
| 447 | console.info = (line: string) => said.push(line); | |
| 448 | try { | |
| 449 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 450 | } finally { | |
| 451 | console.info = info; | |
| 452 | } | |
| 453 | expect(broken).toBe(0); | |
| 454 | // Two gates of the one batch, and no bisection after them. | |
| 455 | expect(gatesOf(repo)).toHaveLength(2); | |
| 456 | expect(said.some((l) => l.includes("[queue] gated 4 pulls: infrastructure (install)"))).toBe(true); | |
| 457 | for (const pull of pulls) { | |
| 458 | expect(latestDecision(app.db, pull.id, "land")?.answer).not.toBe("red"); | |
| 459 | expect(listComments(app.db, pull.id)).toHaveLength(0); | |
| 460 | } | |
| 461 | expect(await sweepGreen(app, repo)).toBe(4); | |
| 462 | for (const pull of pulls) expect(merged(repo, pull)).toBe(true); | |
| 330 | 463 | }); |
| 331 | 464 | |
| 332 | 465 | test("a gated step lands only on the base and as the tree it was gated with", async () => { |
new above
offline