Public: anyone can read this; the crew can change it.
CI lockups: main can never silently go red (pre-push check + auto-revert), bisect by halves, reserved gate capacity, every red names its test, gate logs kept #6466
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/5994-ci-lockups-main-can-never-silently-go-re → main 1 commit fork 2 threads · 2 unresolved
3 files
changed +95
−79
src/checks/queue.ts +41 −49
| @@ -11,10 +11,12 @@ | ||
| 11 | 11 | * candidate gates on main, the next gates on top of it (`depth` deep): if |
| 12 | 12 | * the first lands whole, the next lands right after on the tree it was |
| 13 | 13 | * gated on; if not, the next is discarded and formed again on main. A red |
| 14 | * candidate is searched by gating disjoint segments of it at once, one per | |
| 15 | * free slot: the culprit is in the first red segment, searched again. With | |
| 16 | * no slot free that is one segment, the front half: plain bisection. A | |
| 17 | * single red step is the culprit and is handed back. | |
| 14 | * candidate is searched by gating prefixes of its own chain at once, one per | |
| 15 | * free slot, each stacked on the one before (#5994): the longest green | |
| 16 | * prefix lands, and the culprit is in the run the first red one adds, | |
| 17 | * searched again on top of it. With no slot free that is one prefix, the | |
| 18 | * front half: plain bisection. A single red step is the culprit and is | |
| 19 | * handed back. | |
| 18 | 20 | * |
| 19 | 21 | * A gate cut off by the forge stopping is no verdict: nothing is handed |
| 20 | 22 | * back, and the forge that carries on admits the line again (#3155). A gate |
| @@ -853,14 +855,17 @@ async function round( | ||
| 853 | 855 | |
| 854 | 856 | /** |
| 855 | 857 | * A red candidate: one step is the culprit, handed back on the base it was |
| 856 | * red on. More are searched (#4053, #6353): its steps are cut into disjoint | |
| 857 | * segments, one gate per free check slot (at least the front half), each | |
| 858 | * built and gated on the base alone, so no pull is in two gates of a round. | |
| 859 | * The first red segment holds a culprit and is searched again, on top of a | |
| 860 | * green first segment, which lands; the other green ones, each cleared | |
| 861 | * alone, and the steps after wait for the next candidate. | |
| 862 | * All green, the interaction is between them: the first lands and the rest | |
| 863 | * are searched on it. | |
| 858 | * red on. More are searched by stacked prefixes (#5994): its steps are cut | |
| 859 | * into runs, one gate per free check slot (at least the front half), and | |
| 860 | * the gates are the candidate's own chain cut short — the first run, the | |
| 861 | * first two, and so on — each on the one before, as the line stacks its | |
| 862 | * candidates. The whole is known red, so it is not gated again. The first | |
| 863 | * red prefix holds a culprit in its last run: every green prefix before it | |
| 864 | * is one piece that lands as it was gated, and that run is searched again on | |
| 865 | * top of it. All green, the culprit is in the last run. The runs after the | |
| 866 | * red one wait, in line order, for the next candidate. No green verdict is | |
| 867 | * thrown away: a disjoint segment gated on the base alone was void once | |
| 868 | * another landed (#6353). | |
| 864 | 869 | */ |
| 865 | 870 | async function judged( |
| 866 | 871 | search: Search, |
| @@ -888,50 +893,37 @@ async function judged( | ||
| 888 | 893 | return []; |
| 889 | 894 | } |
| 890 | 895 | const n = steps.length; |
| 891 | // One gate per free slot, at least the front half; with no slot free the | |
| 892 | // back half is not gated and waits for the next round. | |
| 896 | // One gate per free slot and one more, at least the front half, never the | |
| 897 | // whole: it is known red. | |
| 893 | 898 | const free = Math.max(0, room()); |
| 894 | const cut = free > 0 ? Math.min(n, 1 + free) : 2; | |
| 895 | const runs = split(steps, cut - 1); | |
| 896 | const segments = runs.slice(0, free > 0 ? cut : 1); | |
| 897 | const rest = runs.slice(segments.length).flat(); | |
| 898 | // Each segment is built and gated on the base alone: no step is in two | |
| 899 | // gates, so one culprit turns one gate red, not every prefix after it. | |
| 900 | const built = await Promise.all(segments.map((seg) => build(seg))); | |
| 901 | const gates = built.filter((c): c is Candidate => c !== undefined && c.steps.length > 0); | |
| 902 | if (gates.length !== segments.length || gates.some((c) => c.baseOid !== gates[0]!.baseOid)) { | |
| 903 | // The base moved under the search: nothing here is decided, and the | |
| 904 | // next sweep admits these again onto the base as it is now. | |
| 905 | for (const w of steps) w.settle(false); | |
| 906 | return []; | |
| 907 | } | |
| 908 | const verdicts = await Promise.all(gates.map((c) => gated(c))); | |
| 899 | const runs = split(steps, Math.min(n - 1, 1 + free)); | |
| 900 | const ends: number[] = []; | |
| 901 | for (const run of runs) ends.push((ends.at(-1) ?? 0) + run.length); | |
| 902 | const prefixes = ends.slice(0, -1).map((end) => prefixOf(candidate, end)); | |
| 903 | const verdicts = await Promise.all(prefixes.map((c) => gated(c))); | |
| 909 | 904 | if (verdicts.some((v) => !v.lands && "interrupted" in v)) { |
| 910 | 905 | for (const w of steps) w.settle(false); |
| 911 | 906 | return []; |
| 912 | 907 | } |
| 913 | const inGate = gates.map((c, i) => inCandidate(segments[i]!, c)); | |
| 914 | const bad = verdicts.findIndex((v) => !v.lands); | |
| 915 | if (bad < 0) { | |
| 916 | // Each green alone, the whole red: the interaction is between them. The | |
| 917 | // first lands, gated as it is on the base; the rest are searched on it. | |
| 918 | await land(gates[0]!, inGate[0]!); | |
| 919 | const next = [...inGate.slice(1).flat(), ...rest]; | |
| 920 | return round(search, next, { treeOid: candidate.treeOid, red }, [...landed, ...inGate[0]!.map((w) => w.entry)]); | |
| 921 | } | |
| 922 | // The first red segment holds a culprit and is searched again. A green | |
| 923 | // first segment before it lands as it was gated and the search goes on top | |
| 924 | // of it; the other green segments, each gated alone, and the steps after | |
| 925 | // the red one wait, in line order, for the next candidate. | |
| 926 | const first = bad > 0; | |
| 927 | if (first) await land(gates[0]!, inGate[0]!); | |
| 928 | const now = first ? [...landed, ...inGate[0]!.map((w) => w.entry)] : landed; | |
| 929 | const before = inGate.slice(first ? 1 : 0, bad).flat(); | |
| 930 | const after = [...inGate.slice(bad + 1).flat(), ...rest]; | |
| 931 | const left = await round(search, inGate[bad]!, { treeOid: gates[bad]!.treeOid, red: verdicts[bad] as Red }, now); | |
| 932 | return [...before, ...left, ...after]; | |
| 908 | // The first red prefix, the whole when every gated one was green. | |
| 909 | const found = verdicts.findIndex((v) => !v.lands); | |
| 910 | const bad = found < 0 ? prefixes.length : found; | |
| 911 | const known = bad < prefixes.length ? { treeOid: prefixes[bad]!.treeOid, red: verdicts[bad] as Red } : { treeOid: candidate.treeOid, red }; | |
| 912 | const green = bad > 0 ? ends[bad - 1]! : 0; | |
| 913 | if (green > 0) await land(prefixes[bad - 1]!, steps.slice(0, green)); | |
| 914 | const now = [...landed, ...steps.slice(0, green).map((w) => w.entry)]; | |
| 915 | const after = runs.slice(bad + 1).flat(); | |
| 916 | const left = await round(search, runs[bad]!, known, now); | |
| 917 | return [...left, ...after]; | |
| 933 | 918 | } |
| 934 | 919 | |
| 920 | /** The candidate's chain cut after its first `end` steps: the tree a prefix gates. */ | |
| 921 | function prefixOf(candidate: Candidate, end: number): Candidate { | |
| 922 | const last = candidate.steps[end - 1]!; | |
| 923 | return { ...candidate, steps: candidate.steps.slice(0, end), tipOid: last.commitOid, treeOid: last.treeOid }; | |
| 924 | } | |
| 925 | ||
| 926 | ||
| 935 | 927 | /** The steps cut into `k` + 1 runs, evenly, none shared. */ |
| 936 | 928 | function split<T>(steps: readonly T[], k: number): T[][] { |
| 937 | 929 | const n = steps.length; |
test/merge-queue-parallel.test.ts +10 −11
| @@ -151,7 +151,7 @@ describe("the merge queue in parallel", () => { | ||
| 151 | 151 | expect(git(repo, "rev-list", "--count", "refs/heads/main")).toBe("17"); |
| 152 | 152 | }); |
| 153 | 153 | |
| 154 | test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of prefixes, and the nine land by the fourth gate-time", async () => { | |
| 154 | test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of stacked prefixes, and the nine land by the fourth gate-time", async () => { | |
| 155 | 155 | const repo = await autonomousRepo("ten-one-red"); |
| 156 | 156 | const pulls = []; |
| 157 | 157 | for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 4 ? `red-${i}.ts` : `p${i}.ts`)); |
| @@ -163,15 +163,14 @@ describe("the merge queue in parallel", () => { | ||
| 163 | 163 | // beside the other seven and the next on main and the last on top. |
| 164 | 164 | await letGo(3); |
| 165 | 165 | // Wave three: the head was green alone; the eight without it are red |
| 166 | // again, and the eight are gated at once, no pull in two gates: one is red. | |
| 167 | await letGo(8); | |
| 168 | // Wave four: the first of the eight landed as gated; the culprit, red | |
| 169 | // alone, is gated on top of it and handed back. | |
| 170 | await letGo(1); | |
| 171 | // Wave five: the pulls cleared alone, with the head, are one candidate. | |
| 166 | // again, and are searched by prefixes of their own chain, each on the | |
| 167 | // one before (#5994): the first red one names the culprit, the green | |
| 168 | // one before it lands, and the culprit is handed back with no gate more. | |
| 169 | await letGo(7); | |
| 170 | // Wave four: the pulls after the culprit, with the head, are one candidate. | |
| 172 | 171 | await letGo(1); |
| 173 | 172 | expect(await sweep).toBe(9); |
| 174 | expect(wave - first).toBe(5); | |
| 173 | expect(wave - first).toBe(4); | |
| 175 | 174 | const red = pulls[4]!; |
| 176 | 175 | for (const pull of pulls) expect(merged(repo, pull)).toBe(pull !== red); |
| 177 | 176 | expect(latestDecision(app.db, red.id, "land")?.answer).toBe("red"); |
| @@ -186,9 +185,9 @@ describe("the merge queue in parallel", () => { | ||
| 186 | 185 | for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 9 ? `red-${i}.ts` : `p${i}.ts`)); |
| 187 | 186 | const sweep = sweepGreen(app, repo); |
| 188 | 187 | await letGo(2); |
| 189 | // The first eight landed; the two behind are red on them, so each is | |
| 190 | // gated alone: the culprit is handed back, the other gated again on main. | |
| 191 | await letGo(2); | |
| 188 | // The first eight landed; the two behind are red on them, so the first | |
| 189 | // of the two is gated alone on main: green, it lands, and the culprit, | |
| 190 | // red on top of it, is handed back with no gate more (#5994). | |
| 192 | 191 | await letGo(1); |
| 193 | 192 | expect(await sweep).toBe(9); |
| 194 | 193 | for (const pull of pulls.slice(0, 9)) expect(merged(repo, pull)).toBe(true); |
test/merge-queue-search.test.ts +44 −19
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | /** |
| 2 | * A red candidate is searched by disjoint segments, never by nested | |
| 3 | * prefixes holding the same suspects (#6353). The gate here is held open and | |
| 2 | * A red candidate is searched by prefixes of its own chain, each stacked on | |
| 3 | * the one before (#5994), so every green piece lands in order: a disjoint | |
This header line now runs far past the others; rewrap the comment. |
||
| 4 | * segment gated on the base alone was void once another landed (#6353). The gate here is held open and | |
| 4 | 5 | * let go a wave at a time, so "at once" is counted, never timed; it reads |
| 5 | 6 | * the candidate's tree: red when it holds `bad.txt`. |
| 6 | 7 | */ |
| @@ -12,7 +13,7 @@ import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks | ||
| 12 | 13 | import type { LandVerdict } from "../src/checks/tiers.ts"; |
| 13 | 14 | import type { Pull } from "../src/core/pulls.ts"; |
| 14 | 15 | |
| 15 | describe("the merge queue searches a red candidate by disjoint segments", () => { | |
| 16 | describe("the merge queue searches a red candidate by stacked prefixes", () => { | |
| 16 | 17 | let dir: string; |
| 17 | 18 | const told: number[] = []; |
| 18 | 19 | |
| @@ -59,39 +60,63 @@ describe("the merge queue searches a red candidate by disjoint segments", () => | ||
| 59 | 60 | }; |
| 60 | 61 | } |
| 61 | 62 | |
| 62 | test("a red 7-step candidate with 3 free slots: the gates share no step, and a culprit at step 5 is found in 2 rounds beyond the head's", async () => { | |
| 63 | const pulls: Queued[] = []; | |
| 64 | for (let i = 1; i <= 7; i++) pulls.push(entry(i, await headAdding(`s${i}`, i === 5 ? "bad.txt" : `f${i}.txt`))); | |
| 63 | /** A gate held open and let go a wave at a time: "at once" is counted, never timed. */ | |
| 64 | function waves() { | |
| 65 | 65 | let pending: { candidate: Candidate; go: (v: LandVerdict) => void }[] = []; |
| 66 | const waves: number[][][] = []; | |
| 66 | const seen: number[][][] = []; | |
| 67 | const greens: number[][] = []; | |
| 67 | 68 | const gate = (candidate: Candidate) => new Promise<LandVerdict>((go) => pending.push({ candidate, go })); |
| 68 | 69 | const letGo = async (n: number) => { |
| 69 | 70 | for (let i = 0; i < 2000 && pending.length < n; i++) await Bun.sleep(5); |
| 70 | 71 | expect(pending).toHaveLength(n); |
| 71 | 72 | const going = pending; |
| 72 | 73 | pending = []; |
| 73 | waves.push(going.map((p) => p.candidate.steps.map((s) => s.entry.pull.number))); | |
| 74 | for (const p of going) p.go(verdictOf(p.candidate)); | |
| 74 | seen.push(going.map((p) => p.candidate.steps.map((s) => s.entry.pull.number))); | |
| 75 | for (const p of going) { | |
| 76 | const verdict = verdictOf(p.candidate); | |
| 77 | if (verdict.lands) greens.push(p.candidate.steps.map((s) => s.entry.pull.number)); | |
| 78 | p.go(verdict); | |
| 79 | } | |
| 75 | 80 | }; |
| 81 | return { gate, letGo, seen, greens }; | |
| 82 | } | |
| 83 | ||
| 84 | test("a red 7-step candidate with 3 free slots: the rest is searched by stacked prefixes, and every green prefix's pulls land", async () => { | |
| 85 | told.length = 0; | |
| 86 | const pulls: Queued[] = []; | |
| 87 | for (let i = 1; i <= 7; i++) pulls.push(entry(i, await headAdding(`s${i}`, i === 5 ? "bad.txt" : `f${i}.txt`))); | |
| 88 | const { gate, letGo, seen } = waves(); | |
| 76 | 89 | const landed = enqueue("search", pulls, gate, { depth: 1, room: () => 3 }); |
| 77 | 90 | await letGo(1); // the whole candidate: red |
| 78 | 91 | await letGo(2); // its head alone (#4999), beside the six without it |
| 79 | await letGo(4); // the six, cut into four segments gated at once | |
| 80 | await letGo(1); // the first segment landed as gated; the red one is gated on it | |
| 81 | await letGo(2); // the red segment, halved: the culprit is in it alone | |
| 82 | await letGo(1); // the pulls cleared alone, formed again as one candidate | |
| 92 | await letGo(4); // the six: four prefixes of their own chain, each on the one before | |
| 93 | await letGo(1); // the pulls after the culprit, with the head, as one candidate | |
| 83 | 94 | const answers = await landed; |
| 84 | for (const wave of waves) { | |
| 85 | const seen = wave.flat(); | |
| 86 | expect(new Set(seen).size).toBe(seen.length); | |
| 87 | } | |
| 88 | expect(waves[3]).toEqual([[5, 6]]); | |
| 89 | expect(waves[4]!.flat().sort()).toEqual([5, 6]); | |
| 95 | expect(seen[2]).toEqual([[2, 3], [2, 3, 4], [2, 3, 4, 5], [2, 3, 4, 5, 6]]); | |
| 90 | 96 | expect(told).toEqual([5]); |
| 91 | 97 | expect(answers[4]).toBe(false); |
| 92 | 98 | expect(answers.filter(Boolean)).toHaveLength(6); |
| 93 | 99 | }); |
| 94 | 100 | |
| 101 | test("an 8-pull batch with one bad pull: the 7 good land, the search takes 2 gate rounds past the head's, and no green verdict is thrown away (#5994)", async () => { | |
| 102 | told.length = 0; | |
| 103 | const pulls: Queued[] = []; | |
| 104 | for (let i = 21; i <= 28; i++) pulls.push(entry(i, await headAdding(`u${i}`, i === 24 ? "bad.txt" : `f${i}.txt`))); | |
| 105 | const { gate, letGo, seen, greens } = waves(); | |
| 106 | const landed = enqueue("search-eight", pulls, gate, { depth: 1, room: () => 7 }); | |
| 107 | await letGo(1); // the eight: red | |
| 108 | await letGo(2); // its head alone (#4999), beside the seven without it: red | |
| 109 | await letGo(6); // round one: the seven, six prefixes at once; the culprit is named | |
| 110 | await letGo(1); // round two: the pulls after the culprit, with the head | |
| 111 | const answers = await landed; | |
| 112 | expect(seen).toHaveLength(4); | |
| 113 | expect(told).toEqual([24]); | |
| 114 | expect(answers).toEqual(pulls.map((q) => q.pull.number !== 24)); | |
| 115 | // Every gate that came back green holds only pulls that landed. | |
| 116 | for (const g of greens) for (const n of g) expect(n).not.toBe(24); | |
| 117 | expect(git("ls-tree", "--name-only", "refs/heads/main").split("\n")).not.toContain("bad.txt"); | |
| 118 | }); | |
| 119 | ||
| 95 | 120 | test("with no slot free, the green front half lands and the red back half is not gated again", async () => { |
| 96 | 121 | told.length = 0; |
| 97 | 122 | const pulls: Queued[] = []; |
new above
offline
Stray double blank line after prefixOf; keep one.