Public: anyone can read this; the crew can change it.

After a red merge batch, the queue gates the second half on top of the first, so both land when both are green #6531

Parked opened this
Closed without merging

scout/forge:scout/6515-after-a-red-merge-batch-the-queue-gates → main 2 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
4 files changed +232 −3
src/checks/queue.ts +70 −3
@@ -11,12 +11,22 @@
1111 * candidate gates on main, the next gates on top of it (`depth` deep): if
1212 * the first lands whole, the next lands right after on the tree it was
1313 * gated on; if not, the next is discarded and formed again on main. A red
14 <<<<<<< HEAD
15 * candidate is searched by gating segments of it at once, one per free
16 * slot, each stacked on the one before (#6515): the green ones before the
17 * first red land as gated, the first red holds the culprit and is searched
18 * again with the verdict it already has, and those after it, stacked on red,
19 * are void and wait to be gated again. With no slot free that is one
20 * segment, the front half: plain bisection. A single red step is the
21 * culprit and is handed back.
22 =======
1423 * candidate is searched by gating prefixes of its own chain at once, one per
1524 * free slot, each stacked on the one before (#5994): the longest green
1625 * prefix lands, and the culprit is in the run the first red one adds,
1726 * searched again on top of it. With no slot free that is one prefix, the
1827 * front half: plain bisection. A single red step is the culprit and is
1928 * handed back.
29 >>>>>>> origin/main
2030 *
2131 * A gate cut off by the forge stopping is no verdict: nothing is handed
2232 * back, and the forge that carries on admits the line again (#3155). A gate
@@ -772,8 +782,12 @@ async function settleHead(search: Search, head: Flight, verdict: LandVerdict, ah
772782 const { line } = search;
773783 const split =
774784 !head.recheck && head.steps.length > 2 && suspectOf(line, head.steps[0]!) === undefined && search.room() > 0;
775 suspected(line, head.steps, split);
776 if (split) {
785 // With the line behind it waiting, the head goes alone beside the line. With
786 // nothing behind, the search is the only use of the slots: the head is its
787 // first segment, and the rest are gated at once, stacked on it (#6515).
788 const apart = split && (behind.length > 0 || line.waiting.length > 0);
789 suspected(line, head.steps, apart);
790 if (apart) {
777791 log.info("red batch: its head is gated alone", {
778792 repo: line.repoId,
779793 pull: head.steps[0]!.entry.pull.number,
@@ -781,7 +795,7 @@ async function settleHead(search: Search, head: Flight, verdict: LandVerdict, ah
781795 });
782796 return [...head.steps, ...behind];
783797 }
784 const left = await judged(search, candidate, head.steps, verdict, []);
798 const left = await judged(search, candidate, head.steps, verdict, [], split);
785799 if (head.steps.some((w) => line.reds.get(w.entry.pull.id)?.head === w.entry.pull.headOid)) explained(line, head.steps);
786800 return [...left, ...behind];
787801 }
@@ -873,6 +887,7 @@ async function judged(
873887 steps: readonly Waiting[],
874888 red: Red,
875889 landed: readonly Queued[],
890 headFirst = false,
876891 ): Promise<Waiting[]> {
877892 const { gated, room } = search;
878893 if (steps.length === 1) {
@@ -896,15 +911,66 @@ async function judged(
896911 // One gate per free slot and one more, at least the front half, never the
897912 // whole: it is known red.
898913 const free = Math.max(0, room());
914 <<<<<<< HEAD
915 const cut = free > 0 ? Math.min(n, 1 + free) : 2;
916 // A head not tried before is the first segment alone (#4999): red, it is
917 // the culprit at once; green, it lands and the rest are searched on it.
918 const runs = headFirst && cut > 2 ? [[steps[0]!], ...split(steps.slice(1), cut - 2)] : split(steps, cut - 1);
919 const segments = runs.slice(0, free > 0 ? cut : 1);
920 const rest = runs.slice(segments.length).flat();
921 // The first segment is built on the base, each next on the one before:
922 // what a green verdict says of a segment stands once those before it land,
923 // so greens in a row all land and no gate is spent twice (#6515).
924 const gates: Candidate[] = [];
925 for (const seg of segments) {
926 const c = await build(seg, gates.at(-1));
927 if (c === undefined || c.steps.length === 0) break;
928 gates.push(c);
929 }
930 if (gates.length !== segments.length) {
931 // The base moved under the search: nothing here is decided, and the
932 // next sweep admits these again onto the base as it is now.
933 for (const w of steps) w.settle(false);
934 return [];
935 }
936 const verdicts = await Promise.all(gates.map((c) => gated(c)));
937 =======
899938 const runs = split(steps, Math.min(n - 1, 1 + free));
900939 const ends: number[] = [];
901940 for (const run of runs) ends.push((ends.at(-1) ?? 0) + run.length);
902941 const prefixes = ends.slice(0, -1).map((end) => prefixOf(candidate, end));
903942 const verdicts = await Promise.all(prefixes.map((c) => gated(c)));
943 >>>>>>> origin/main
904944 if (verdicts.some((v) => !v.lands && "interrupted" in v)) {
905945 for (const w of steps) w.settle(false);
906946 return [];
907947 }
948 <<<<<<< HEAD
949 const inGate = gates.map((c, i) => inCandidate(segments[i]!, c));
950 const bad = verdicts.findIndex((v) => !v.lands);
951 // The green segments before the red one land in order, each as gated. A
952 // red one is stacked on them, so its verdict stands and the search goes on
953 // from it; the segments after it were stacked on red and are void: they
954 // wait, in line order, for the next candidate.
955 const upto = bad < 0 ? gates.length : bad;
956 const branch = steps[0]!.entry.pull.baseBranch;
957 let now = landed;
958 for (let i = 0; i < upto; i++) {
959 let gate = gates[i]!;
960 if (i > 0) {
961 const tip = await onTip(gate, branch);
962 if (tip === undefined) return [...inGate.slice(i).flat(), ...rest];
963 gate = { ...gate, baseOid: tip };
964 }
965 if (!(await land(gate, inGate[i]!))) return [...inGate.slice(i + 1).flat(), ...rest];
966 now = [...now, ...inGate[i]!.map((w) => w.entry)];
967 }
968 // All green: with every segment gated that is the whole candidate, landed.
969 // With only the front gated, the back half is the whole's tree on it: known red.
970 if (bad < 0) return rest.length === 0 ? [] : round(search, rest, { treeOid: candidate.treeOid, red }, now);
971 const left = await round(search, inGate[bad]!, { treeOid: gates[bad]!.treeOid, red: verdicts[bad] as Red }, now);
972 return [...left, ...inGate.slice(bad + 1).flat(), ...rest];
973 =======
908974 // The first red prefix, the whole when every gated one was green.
909975 const found = verdicts.findIndex((v) => !v.lands);
910976 const bad = found < 0 ? prefixes.length : found;
@@ -915,6 +981,7 @@ async function judged(
915981 const after = runs.slice(bad + 1).flat();
916982 const left = await round(search, runs[bad]!, known, now);
917983 return [...left, ...after];
984 >>>>>>> origin/main
918985 }
919986
920987 /** The candidate's chain cut after its first `end` steps: the tree a prefix gates. */
test/merge-queue-parallel.test.ts +21
@@ -151,7 +151,11 @@ describe("the merge queue in parallel", () => {
151151 expect(git(repo, "rev-list", "--count", "refs/heads/main")).toBe("17");
152152 });
153153
154 <<<<<<< HEAD
155 test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of stacked segments, and the nine land by the fourth gate-time", async () => {
156 =======
154157 test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of stacked prefixes, and the nine land by the fourth gate-time", async () => {
158 >>>>>>> origin/main
155159 const repo = await autonomousRepo("ten-one-red");
156160 const pulls = [];
157161 for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 4 ? `red-${i}.ts` : `p${i}.ts`));
@@ -162,12 +166,22 @@ describe("the merge queue in parallel", () => {
162166 // Wave two: the eight are red, so their head is gated alone (#4999),
163167 // beside the other seven and the next on main and the last on top.
164168 await letGo(3);
169 <<<<<<< HEAD
170 // Wave three: the head was green alone; the seven without it are red
171 // again, and are gated at once, no pull in two gates, each stacked on the
172 // one before: the four before the culprit are green and land as gated, the
173 // culprit is red on top of them and handed back with no gate of its own.
174 await letGo(8);
175 // Wave four: the pulls after it were stacked on red: void, gated again
176 // on main with the head, as one candidate.
177 =======
165178 // Wave three: the head was green alone; the eight without it are red
166179 // again, and are searched by prefixes of their own chain, each on the
167180 // one before (#5994): the first red one names the culprit, the green
168181 // one before it lands, and the culprit is handed back with no gate more.
169182 await letGo(7);
170183 // Wave four: the pulls after the culprit, with the head, are one candidate.
184 >>>>>>> origin/main
171185 await letGo(1);
172186 expect(await sweep).toBe(9);
173187 expect(wave - first).toBe(4);
@@ -185,10 +199,17 @@ describe("the merge queue in parallel", () => {
185199 for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 9 ? `red-${i}.ts` : `p${i}.ts`));
186200 const sweep = sweepGreen(app, repo);
187201 await letGo(2);
202 <<<<<<< HEAD
203 // The first eight landed; the two behind are red on them, so each is
204 // gated, the second on the first: the culprit is red on top of the other,
205 // which lands, and is handed back with no gate of its own.
206 await letGo(2);
207 =======
188208 // The first eight landed; the two behind are red on them, so the first
189209 // of the two is gated alone on main: green, it lands, and the culprit,
190210 // red on top of it, is handed back with no gate more (#5994).
191211 await letGo(1);
212 >>>>>>> origin/main
192213 expect(await sweep).toBe(9);
193214 for (const pull of pulls.slice(0, 9)) expect(merged(repo, pull)).toBe(true);
194215 expect(merged(repo, pulls[9]!)).toBe(false);
test/merge-queue-search.test.ts +20
@@ -60,8 +60,14 @@ describe("the merge queue searches a red candidate by stacked prefixes", () => {
6060 };
6161 }
6262
63 <<<<<<< HEAD
64 test("a red 7-step candidate with 3 free slots: the gates of a round share no step, and a culprit at step 5 is found in 3 rounds after the red whole", async () => {
65 const pulls: Queued[] = [];
66 for (let i = 1; i <= 7; i++) pulls.push(entry(i, await headAdding(`s${i}`, i === 5 ? "bad.txt" : `f${i}.txt`)));
67 =======
6368 /** A gate held open and let go a wave at a time: "at once" is counted, never timed. */
6469 function waves() {
70 >>>>>>> origin/main
6571 let pending: { candidate: Candidate; go: (v: LandVerdict) => void }[] = [];
6672 const seen: number[][][] = [];
6773 const greens: number[][] = [];
@@ -88,11 +94,25 @@ describe("the merge queue searches a red candidate by stacked prefixes", () => {
8894 const { gate, letGo, seen } = waves();
8995 const landed = enqueue("search", pulls, gate, { depth: 1, room: () => 3 });
9096 await letGo(1); // the whole candidate: red
97 <<<<<<< HEAD
98 await letGo(4); // its head alone and the rest in three segments, each stacked on the one before
99 await letGo(2); // the first two landed as gated; the red segment halved, the culprit alone in a gate
100 await letGo(1); // the last segment, stacked on the red one: void, gated again on main
101 const answers = await landed;
102 for (const wave of waves) {
103 const seen = wave.flat();
104 expect(new Set(seen).size).toBe(seen.length);
105 }
106 expect(waves[1]).toEqual([[1], [2, 3], [4, 5], [6, 7]]);
107 expect(waves[2]).toEqual([[4], [5]]);
108 expect(waves[3]).toEqual([[6, 7]]);
109 =======
91110 await letGo(2); // its head alone (#4999), beside the six without it
92111 await letGo(4); // the six: four prefixes of their own chain, each on the one before
93112 await letGo(1); // the pulls after the culprit, with the head, as one candidate
94113 const answers = await landed;
95114 expect(seen[2]).toEqual([[2, 3], [2, 3, 4], [2, 3, 4, 5], [2, 3, 4, 5, 6]]);
115 >>>>>>> origin/main
96116 expect(told).toEqual([5]);
97117 expect(answers[4]).toBe(false);
98118 expect(answers.filter(Boolean)).toHaveLength(6);
test/merge-queue-stacked.test.ts added +121
@@ -0,0 +1,121 @@
1 /**
2 * After a red batch the queue gates the halves of it stacked, the second on
3 * the first (#6515): a green verdict of the second stands once the first
4 * lands, so the good pulls land without a gate spent twice. The gate here
5 * is held open and let go a wave at a time; a wave is one round.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks/queue.ts";
12 import type { LandVerdict } from "../src/checks/tiers.ts";
13 import type { Pull } from "../src/core/pulls.ts";
14
15 describe("the merge queue gates the halves of a red batch stacked", () => {
16 let dir: string;
17 let told: number[] = [];
18
19 const git = (...args: string[]) =>
20 Bun.spawnSync(["git", "-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", ...args]).stdout.toString().trim();
21
22 beforeAll(async () => {
23 dir = await mkdtemp(join(tmpdir(), "forge-merge-queue-stacked-"));
24 Bun.spawnSync(["git", "init", "-q", "-b", "main", dir]);
25 git("commit", "-q", "--allow-empty", "-m", "start");
26 });
27
28 afterAll(async () => {
29 await rm(dir, { recursive: true, force: true });
30 });
31
32 async function headAdding(branch: string, file: string): Promise<string> {
33 git("checkout", "-q", "-B", branch, "main");
34 git("reset", "-q", "--hard", "main");
35 await Bun.write(join(dir, file), `${file}\n`);
36 git("add", file);
37 git("commit", "-q", "-m", file);
38 git("checkout", "-q", "--detach");
39 return git("rev-parse", branch);
40 }
41
42 const verdictOf = (candidate: Candidate): LandVerdict =>
43 git("ls-tree", "--name-only", candidate.tipOid).split("\n").includes("bad.txt")
44 ? { lands: false, reopen: true, comment: "red", learn: [] }
45 : { lands: true };
46
47 function entry(number: number, headOid: string): Queued {
48 return {
49 pull: { id: `stacked-${number}`, number, baseBranch: "main", headOid } as Pull,
50 dir,
51 land: async (expect: Expect) => {
52 if (git("rev-parse", "refs/heads/main") !== expect.baseOid) return undefined;
53 const tip = git("commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", `#${number}`);
54 git("update-ref", "refs/heads/main", tip);
55 return tip;
56 },
57 release: () => {},
58 handBack: () => told.push(number),
59 };
60 }
61
62 /**
63 * Eight pulls, `bad` the red one. Every gate is held until the ones started
64 * together are all open, then they answer at once: a round is such a wave.
65 */
66 async function run(tag: string, bad: number, room: number) {
67 told = [];
68 const pulls: Queued[] = [];
69 for (let i = 1; i <= 8; i++) pulls.push(entry(i, await headAdding(`${tag}${i}`, i === bad ? "bad.txt" : `${tag}-f${i}.txt`)));
70 const rounds: number[][][] = [];
71 let open: { candidate: Candidate; go: (v: LandVerdict) => void }[] = [];
72 const gate = (candidate: Candidate) => new Promise<LandVerdict>((go) => open.push({ candidate, go }));
73 const answers = enqueue(tag, pulls, gate, { depth: 2, room: () => room });
74 let settled = false;
75 void answers.then(() => (settled = true));
76 while (!settled) {
77 const seen = open.length;
78 await Bun.sleep(40);
79 if (open.length === 0 || open.length !== seen) continue;
80 const going = open;
81 open = [];
82 rounds.push(going.map((g) => g.candidate.steps.map((s) => s.entry.pull.number)));
83 for (const g of going) g.go(verdictOf(g.candidate));
84 }
85 return { answers: await answers, rounds };
86 }
87
88 test("a red second half is stacked on the green first: the first lands and the second's verdict stands", async () => {
89 const { answers, rounds } = await run("a", 8, 1);
90 expect(told).toEqual([8]);
91 expect(answers).toEqual([true, true, true, true, true, true, true, false]);
92 // After the whole: its halves at once, the second stacked on the first.
93 // The first landed, so the second is searched on it without a gate of
94 // its own: its next gates are its halves, then theirs.
95 expect(rounds).toEqual([
96 [[1, 2, 3, 4, 5, 6, 7, 8]],
97 [[1, 2, 3, 4], [5, 6, 7, 8]],
98 [[5, 6], [7, 8]],
99 [[7], [8]],
100 ]);
101 });
102
103 test("a red first half voids the stacked second: it is gated again alone on main", async () => {
104 const { answers, rounds } = await run("c", 2, 1);
105 expect(told).toEqual([2]);
106 expect(answers).toEqual([true, false, true, true, true, true, true, true]);
107 expect(rounds[1]).toEqual([[1, 2, 3, 4], [5, 6, 7, 8]]);
108 // The second was stacked on a red first: its green verdict is void, so
109 // its pulls are gated again, after the culprit is found, on main.
110 expect(rounds.at(-1)).toEqual([[3, 4, 5, 6, 7, 8]]);
111 });
112
113 test("an 8-pull batch with one bad pull lands all 7 good ones in at most 2 gate rounds after the red", async () => {
114 for (const bad of [1, 4, 8]) {
115 const { answers, rounds } = await run(`b${bad}`, bad, 7);
116 expect(told).toEqual([bad]);
117 expect(answers.filter(Boolean)).toHaveLength(7);
118 expect(rounds.length - 1).toBeLessThanOrEqual(2);
119 }
120 });
121 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs