4793b74
@@ -1279,6 +1279,13 @@
gated tree. Red is searched: prefixes of the candidate are gated on main
— the front half, plus one more cut per free check slot, all at once — the
longest green prefix lands, the search goes on in the steps up to the first
red prefix, and the red pull is handed back with the failing tests named.
A red batch is never formed again around the same head: with a check slot
free, the head of a red batch of three or more is gated alone beside the
line (`[queue] red batch: its head is gated alone`) while the rest gates
without it, stacked as deep as `queue.depth` goes, so the good pulls land
while the suspect is judged. A pull in two red batches is never in a third:
it waits for a gate of its own. Red alone, it is handed back (`[queue] red
alone`); green alone, it rides at the tail of a later batch, never its head.
While a candidate gates, the next one gates on top of it (`queue.depth`
under Settings → Capacity, `FORGE_QUEUE_DEPTH`, default 2): when the first
lands whole the next lands right after it, on the tree it was gated on; when
@@ -32,6 +32,15 @@
 * stacked on, so a red re-check cuts off no batch of fresh pulls. It is
 * forgotten when it lands or its head changes.
 *
 * A red batch is never formed again around the same head (#4999). With a
 * check slot free, the head of a red batch of three or more — the pull
 * every prefix of it holds — is a suspect, gated alone beside the line,
 * while the rest of the batch goes back to the front of the line and gates
 * without it, stacked as deep as the line goes. A pull in two red batches
 * is a suspect too, never in a third. A suspect red alone is handed back;
 * green alone it rides at the tail of a batch, never at its head, and the
 * search finds it there if it is red on top of others.
 *
 * What is queued is not stored: it is the pulls `landIfGreen` admitted, and
 * every sweep admits them again. The queue only orders them and remembers
 * what is at the gate and what its searches found, for the tick line and
@@ -159,6 +168,17 @@
interface Remembered {
  readonly together: readonly number[];
}
/** A pull in red batches at this head (#4999): gated alone once it is a suspect. */
interface Suspect {
  readonly head: string;
  /** Red batches it was in, less the ones a culprit found since explains. */
  readonly reds: number;
  /** Waits for a verdict of its own, gated alone beside the line. */
  readonly alone: boolean;
  /** Green alone at this head: never at the head of a batch again. */
  readonly cleared: boolean;
}
/** A first candidate's verdict, for the red-batch rate. */
interface BatchVerdict {
  readonly at: number;
@@ -182,6 +202,8 @@
interface Line {
  gates: number;
  /** Pulls handed back red, by pull id: re-checked alone if they join again. */
  readonly reds: Map<string, Remembered>;
  /** Pulls in red batches, by pull id, while their head is the same (#4999). */
  readonly suspects: Map<string, Suspect>;
  /** The last hour's candidate verdicts, search prefixes and re-checks not counted. */
  batches: BatchVerdict[];
  draining: boolean;
@@ -236,6 +258,7 @@
function lineOf(repoId: string): Line {
      flights: new Set(),
      gates: 0,
      reds: new Map(),
      suspects: new Map(),
      batches: [],
      draining: false,
      wake: undefined,
@@ -263,13 +286,17 @@
export function enqueue(
    const red = line.reds.get(entry.pull.id);
    if (red !== undefined && red.head !== entry.pull.headOid) line.reds.delete(entry.pull.id);
    const recheck = red !== undefined && red.head === entry.pull.headOid;
    if (line.suspects.get(entry.pull.id)?.head !== entry.pull.headOid) line.suspects.delete(entry.pull.id);
    return new Promise<boolean>((resolve) =>
      line.waiting.push({
        entry,
        recheck,
        settle: (landed) => {
          line.holds.delete(entry.pull.id);
          if (landed) line.reds.delete(entry.pull.id);
          if (landed) {
            line.reds.delete(entry.pull.id);
            line.suspects.delete(entry.pull.id);
          }
          resolve(landed);
        },
      }),
@@ -441,7 +468,9 @@
const ARRIVED = Symbol("arrived");
 * one place and one base branch; a sealed pull whose objects live
 * elsewhere is a batch of its own, and is not stacked on another. A batch
 * carries at most `MAX_CANDIDATE` pulls. A gate beyond the line's first
 * starts only on a free check slot (`room`).
 * starts only on a free check slot (`room`). A suspect is gated alone,
 * beside the line and first (#4999); the line's first gate waits for a
 * free slot while one is.
 */
async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Promise<void> {
  line.draining = true;
@@ -449,19 +478,41 @@
async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
  const depth = Math.max(1, Math.floor(options.depth ?? DEFAULT_DEPTH));
  const room: Room = options.room ?? (() => 0);
  const ahead: Flight[] = [];
  const solos = new Set<Promise<void>>();
  try {
    while (line.waiting.length > 0 || ahead.length > 0) {
      while (ahead.length < depth && line.waiting.length > 0 && (ahead.length === 0 || room() > 0)) {
    while (line.waiting.length > 0 || ahead.length > 0 || solos.size > 0) {
      for (const w of line.waiting.filter((w) => suspectOf(line, w)?.alone === true)) {
        if (line.flights.size > 0 && room() <= 0) break;
        line.waiting = line.waiting.filter((x) => x !== w);
        const started = await solo(line, gated, w);
        if (started === undefined) continue;
        const done: Promise<void> = started.done.finally(() => {
          solos.delete(done);
          line.wake?.();
          line.wake = undefined;
        });
        solos.add(done);
      }
      while (
        ahead.length < depth &&
        line.waiting.some((w) => suspectOf(line, w)?.alone !== true) &&
        ((ahead.length === 0 && solos.size === 0) || room() > 0)
      ) {
        const flight = await launch(line, gated, ahead.at(-1));
        if (flight === "apart") break;
        if (flight !== undefined) ahead.push(flight);
      }
      const head = ahead[0];
      if (head === undefined) continue;
      const arrived =
        ahead.length < depth && line.waiting.length === 0
        head === undefined || ahead.length < depth || solos.size > 0
          ? new Promise<typeof ARRIVED>((resolve) => (line.wake = () => resolve(ARRIVED)))
          : undefined;
      if (head === undefined) {
        // Only suspects at the gate, or waiting on a slot: wait for one to come back.
        if (solos.size > 0) await arrived;
        line.wake = undefined;
        continue;
      }
      const verdict = arrived === undefined ? await head.verdict : await Promise.race([head.verdict, arrived]);
      line.wake = undefined;
      if (verdict === ARRIVED) continue;
@@ -482,23 +533,104 @@
async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
  }
}
/** What the line knows of a waiting pull's red batches, at its head. */
function suspectOf(line: Line, w: Waiting): Suspect | undefined {
  const s = line.suspects.get(w.entry.pull.id);
  return s?.head === w.entry.pull.headOid ? s : undefined;
}
/**
 * Start a suspect's gate alone on the base's tip, beside the line (#4999):
 * its answer is when the verdict is acted on. Red alone, it is handed back
 * and the pulls of its batch are cleared of that red; green alone, it goes
 * back to the line to ride at a batch's tail. Undefined when there is
 * nothing to gate — it conflicts, or is already in — and the next sweep
 * brings it up to date or closes it.
 */
async function solo(line: Line, gated: Gated, w: Waiting): Promise<{ readonly done: Promise<void> } | undefined> {
  const settled = (error: unknown): void => {
    log.warn("merge queue round failed", { error: String(error) });
    w.settle(false);
  };
  let candidate: Candidate | undefined;
  try {
    candidate = await build([w]);
  } catch (error) {
    settled(error);
    return undefined;
  }
  if (candidate === undefined || candidate.steps.length === 0) {
    w.settle(false);
    return undefined;
  }
  const on = candidate;
  const done = gated(on)
    .then((verdict) => {
      const pull = w.entry.pull;
      if (verdict.lands) {
        line.suspects.set(pull.id, { head: pull.headOid, reds: 0, alone: false, cleared: true });
        line.waiting.unshift(w);
        return;
      }
      if ("interrupted" in verdict) {
        w.settle(false);
        return;
      }
      line.suspects.delete(pull.id);
      line.reds.set(pull.id, { head: pull.headOid, together: [] });
      log.info("red alone", { repo: line.repoId, pull: pull.number });
      w.entry.handBack(verdict, on.baseOid, []);
      w.settle(false);
    })
    .catch(settled);
  return { done };
}
/**
 * Every pull of a red batch was in one more red batch (#4999). Its head is
 * the suspect when `split`; a pull in two red batches is one too, unless it
 * was green alone at this head.
 */
function suspected(line: Line, steps: readonly Waiting[], split: boolean): void {
  for (const [i, w] of steps.entries()) {
    const was = suspectOf(line, w);
    const reds = (was?.reds ?? 0) + 1;
    const cleared = was?.cleared ?? false;
    const alone = !cleared && (reds >= 2 || (split && i === 0));
    line.suspects.set(w.entry.pull.id, { head: w.entry.pull.headOid, reds, alone, cleared });
  }
}
/** A culprit found among `steps` explains the red batch they were in: one red each is forgiven. */
function explained(line: Line, steps: readonly Waiting[]): void {
  for (const w of steps) {
    const was = suspectOf(line, w);
    if (was === undefined) continue;
    const reds = Math.max(0, was.reds - 1);
    line.suspects.set(w.entry.pull.id, { ...was, reds, alone: !was.cleared && reds >= 2 });
  }
}
/**
 * Take the next batch from the front of the line, build it — on `on`'s tip
 * when there is a flight to stack on — and start its gate. Fresh pulls go
 * first; a re-check is a batch of one, once no fresh pull waits. `apart`
 * first, a pull green alone after a red batch at their tail; a re-check is
 * a batch of one, once no fresh pull waits. A suspect is never in a batch:
 * it is gated alone (`solo`). `apart`
 * when the front of the line cannot stack on `on`, or `on` is a re-check;
 * undefined when nothing of the batch could be built.
 */
async function launch(line: Line, gated: Gated, on: Flight | undefined): Promise<Flight | undefined | "apart"> {
  const front = line.waiting.find((w) => !w.recheck) ?? line.waiting[0]!;
  const cleared = (w: Waiting): boolean => suspectOf(line, w)?.cleared === true;
  const open = line.waiting.filter((w) => suspectOf(line, w)?.alone !== true);
  const front = open.find((w) => !w.recheck && !cleared(w)) ?? open.find((w) => !w.recheck) ?? open[0]!;
  const first = front.entry;
  const branch = first.pull.baseBranch;
  if (on !== undefined && (on.recheck || first.dir !== on.candidate.dir || branch !== on.branch)) return "apart";
  const same = open.filter((w) => !w.recheck && w.entry.dir === first.dir && w.entry.pull.baseBranch === branch);
  const batch = front.recheck
    ? [front]
    : line.waiting
        .filter((w) => !w.recheck && w.entry.dir === first.dir && w.entry.pull.baseBranch === branch)
        .slice(0, MAX_CANDIDATE);
    : [...same.filter((w) => !cleared(w)), ...same.filter(cleared)].slice(0, MAX_CANDIDATE);
  line.waiting = line.waiting.filter((w) => !batch.includes(w));
  let candidate: Candidate | undefined;
  try {
@@ -560,7 +692,24 @@
async function settleHead(search: Search, head: Flight, verdict: LandVerdict, ah
    for (const w of head.steps) w.settle(false);
    return behind;
  }
  return [...(await judged(search, candidate, head.steps, verdict, [])), ...behind];
  // A red batch is not formed again around its head (#4999): with a slot
  // free, the head is gated alone beside the line while the rest gate
  // without it. Two steps are the search's own split: the head alone.
  const { line } = search;
  const split =
    !head.recheck && head.steps.length > 2 && suspectOf(line, head.steps[0]!) === undefined && search.room() > 0;
  suspected(line, head.steps, split);
  if (split) {
    log.info("red batch: its head is gated alone", {
      repo: line.repoId,
      pull: head.steps[0]!.entry.pull.number,
      rest: head.steps.length - 1,
    });
    return [...head.steps, ...behind];
  }
  const left = await judged(search, candidate, head.steps, verdict, []);
  if (head.steps.some((w) => line.reds.get(w.entry.pull.id)?.head === w.entry.pull.headOid)) explained(line, head.steps);
  return [...left, ...behind];
}
/**
@@ -654,6 +803,7 @@
async function judged(
    // them to tell which. Remembered with them either way (#4235).
    const together = landed.map((q) => q.pull.number);
    search.line.reds.set(culprit.pull.id, { head: culprit.pull.headOid, together });
    search.line.suspects.delete(culprit.pull.id);
    log.info(together.length === 0 ? "red on its own merge" : "red on top of its candidate", {
      repo: search.line.repoId,
      pull: culprit.pull.number,
@@ -151,7 +151,7 @@
describe("the merge queue in parallel", () => {
    expect(git(repo, "rev-list", "--count", "refs/heads/main")).toBe("17");
  });
  test("ten pulls, one red: the culprit is found in one wave of prefixes, and the nine land by the third gate-time", async () => {
  test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of prefixes, and the nine land by the fourth gate-time", async () => {
    const repo = await autonomousRepo("ten-one-red");
    const pulls = [];
    for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 4 ? `red-${i}.ts` : `p${i}.ts`));
@@ -159,14 +159,17 @@
describe("the merge queue in parallel", () => {
    const sweep = sweepGreen(app, repo);
    // Wave one: eight on main, the other two on top of them.
    await letGo(2);
    // Wave two: the eight are red; the two on top are re-formed, and seven
    // prefixes of the eight gate at once: four green, the fifth red.
    // Wave two: the eight are red, so their head is gated alone (#4999),
    // beside the other seven and the next on main and the last on top.
    await letGo(3);
    // Wave three: the head was green alone; the eight without it are red
    // again, and seven prefixes of them gate at once: three green, the fourth red.
    await letGo(7);
    // Wave three: the culprit is handed back without a gate of its own;
    // the three after it and the two re-formed are one candidate.
    // Wave four: the culprit is handed back without a gate of its own; the
    // four after it, the last and the head (at the tail) are one candidate.
    await letGo(1);
    expect(await sweep).toBe(9);
    expect(wave - first).toBe(3);
    expect(wave - first).toBe(4);
    const red = pulls[4]!;
    for (const pull of pulls) expect(merged(repo, pull)).toBe(pull !== red);
    expect(latestDecision(app.db, red.id, "land")?.answer).toBe("red");
@@ -0,0 +1,122 @@
/**
 * A red batch is never formed again around the same head (#4999). The head
 * of a red batch is gated alone beside the line while the rest gate without
 * it, so the good pulls land while the suspect is judged; a pull in two red
 * batches waits for a verdict of its own and never heads a third. The gate
 * here reads the candidate's tree: red when it holds `bad.txt`.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks/queue.ts";
import type { LandVerdict } from "../src/checks/tiers.ts";
import type { Pull } from "../src/core/pulls.ts";
describe("the merge queue judges a suspect alone", () => {
  let dir: string;
  /** Every hand-back: the pull, and the pulls it was red on top of. */
  const told: { pull: number; together: number[] }[] = [];
  const git = (...args: string[]) =>
    Bun.spawnSync(["git", "-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", ...args]).stdout.toString().trim();
  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), "forge-merge-queue-suspect-"));
    Bun.spawnSync(["git", "init", "-q", "-b", "main", dir]);
    git("commit", "-q", "--allow-empty", "-m", "start");
  });
  afterAll(async () => {
    await rm(dir, { recursive: true, force: true });
  });
  /** A pull's head: one file added on main as it is now. */
  async function headAdding(branch: string, file: string): Promise<string> {
    git("checkout", "-q", "-B", branch, "main");
    git("reset", "-q", "--hard", "main");
    await Bun.write(join(dir, file), `${file}\n`);
    git("add", file);
    git("commit", "-q", "-m", file);
    git("checkout", "-q", "--detach");
    return git("rev-parse", branch);
  }
  const verdictOf = (candidate: Candidate): LandVerdict =>
    git("ls-tree", "--name-only", candidate.tipOid).split("\n").includes("bad.txt")
      ? { lands: false, reopen: true, comment: "red", learn: [] }
      : { lands: true };
  /** A queued pull that lands the way the forge does: only on the base it was gated on. */
  function entry(line: string, number: number, headOid: string): Queued {
    return {
      pull: { id: `${line}-${number}`, number, baseBranch: "main", headOid } as Pull,
      dir,
      land: async (expect: Expect) => {
        if (git("rev-parse", "refs/heads/main") !== expect.baseOid) return undefined;
        const tip = git("commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", `#${number}`);
        git("update-ref", "refs/heads/main", tip);
        return tip;
      },
      release: () => {},
      handBack: (_red, _base, together) => told.push({ pull: number, together: [...together] }),
    };
  }
  test("one bad pull at the head, thirty good behind: the thirty land in two gate-times, the bad one is handed back after its solo red", async () => {
    const bad = entry("thirty", 1, await headAdding("bad", "bad.txt"));
    const good: Queued[] = [];
    for (let i = 0; i < 30; i++) good.push(entry("thirty", 2 + i, await headAdding(`g${i}`, `g${i}.txt`)));
    // The gate is held open and let go a wave at a time: "at once" is counted, never timed.
    let pending: { candidate: Candidate; go: (v: LandVerdict) => void }[] = [];
    const waves: number[][][] = [];
    const gate = (candidate: Candidate) =>
      new Promise<LandVerdict>((go) => pending.push({ candidate, go }));
    const letGo = async (n: number) => {
      for (let i = 0; i < 2000 && pending.length < n; i++) await Bun.sleep(5);
      expect(pending).toHaveLength(n);
      const going = pending;
      pending = [];
      waves.push(going.map((p) => p.candidate.steps.map((s) => s.entry.pull.number)));
      for (const p of going) p.go(verdictOf(p.candidate));
    };
    const landed = enqueue("thirty", [bad, ...good], gate, { depth: 4, room: () => 8 });
    // Gate-time one: four candidates, the first red on its head.
    await letGo(4);
    // Gate-time two: the head alone, beside the thirty in four candidates without it.
    await letGo(5);
    expect(await landed).toEqual([false, ...good.map(() => true)]);
    expect(waves[1]!.filter((w) => w.includes(1))).toEqual([[1]]);
    expect(waves[1]!.flat().sort((a, b) => a - b)).toEqual([1, ...good.map((q) => q.pull.number)]);
    expect(told).toEqual([{ pull: 1, together: [] }]);
    const files = git("ls-tree", "--name-only", "refs/heads/main").split("\n");
    expect(files).toHaveLength(30);
    expect(files).not.toContain("bad.txt");
  });
  test("a pull in two red batches never heads a third: it is gated alone, and handed back on its own red", async () => {
    told.length = 0;
    const bad = entry("twice", 1, await headAdding("bad2", "bad.txt"));
    const [g1, g2] = [entry("twice", 2, await headAdding("h1", "h1.txt")), entry("twice", 3, await headAdding("h2", "h2.txt"))];
    const gates: number[][] = [];
    // The search is cut off twice (the forge stopping): the batch is red, and nothing is decided.
    let cut = true;
    const gate = async (candidate: Candidate): Promise<LandVerdict> => {
      gates.push(candidate.steps.map((s) => s.entry.pull.number));
      if (cut && candidate.steps.length < 3) return { lands: false, interrupted: true };
      return verdictOf(candidate);
    };
    expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, false, false]);
    expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, false, false]);
    expect(gates.filter((g) => g.length === 3)).toHaveLength(2);
    cut = false;
    const before = gates.length;
    expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, true, true]);
    const after = gates.slice(before);
    // Each was in two red batches: each is judged alone, and the bad one heads no batch.
    expect(after.slice(0, 3)).toEqual([[1], [2], [3]]);
    expect(after.every((g) => g[0] !== 1 || g.length === 1)).toBe(true);
    expect(told).toEqual([{ pull: 1, together: [] }]);
    expect(git("ls-tree", "--name-only", "refs/heads/main").split("\n")).not.toContain("bad.txt");
  });
});
		@@ -1279,6 +1279,13 @@ gated tree. Red is searched: prefixes of the candidate are gated on main
1279	1279	— the front half, plus one more cut per free check slot, all at once — the
1280	1280	longest green prefix lands, the search goes on in the steps up to the first
1281	1281	red prefix, and the red pull is handed back with the failing tests named.
	1282	A red batch is never formed again around the same head: with a check slot
	1283	free, the head of a red batch of three or more is gated alone beside the
	1284	line (`[queue] red batch: its head is gated alone`) while the rest gates
	1285	without it, stacked as deep as `queue.depth` goes, so the good pulls land
	1286	while the suspect is judged. A pull in two red batches is never in a third:
	1287	it waits for a gate of its own. Red alone, it is handed back (`[queue] red
	1288	alone`); green alone, it rides at the tail of a later batch, never its head.
1282	1289	While a candidate gates, the next one gates on top of it (`queue.depth`
1283	1290	under Settings → Capacity, `FORGE_QUEUE_DEPTH`, default 2): when the first
1284	1291	lands whole the next lands right after it, on the tree it was gated on; when
		@@ -32,6 +32,15 @@
32	32	* stacked on, so a red re-check cuts off no batch of fresh pulls. It is
33	33	* forgotten when it lands or its head changes.
34	34	*
	35	* A red batch is never formed again around the same head (#4999). With a
	36	* check slot free, the head of a red batch of three or more — the pull
	37	* every prefix of it holds — is a suspect, gated alone beside the line,
	38	* while the rest of the batch goes back to the front of the line and gates
	39	* without it, stacked as deep as the line goes. A pull in two red batches
	40	* is a suspect too, never in a third. A suspect red alone is handed back;
	41	* green alone it rides at the tail of a batch, never at its head, and the
	42	* search finds it there if it is red on top of others.
	43	*
35	44	* What is queued is not stored: it is the pulls `landIfGreen` admitted, and
36	45	* every sweep admits them again. The queue only orders them and remembers
37	46	* what is at the gate and what its searches found, for the tick line and
		@@ -159,6 +168,17 @@ interface Remembered {
159	168	readonly together: readonly number[];
160	169	}
161	170
	171	/** A pull in red batches at this head (#4999): gated alone once it is a suspect. */
	172	interface Suspect {
	173	readonly head: string;
	174	/** Red batches it was in, less the ones a culprit found since explains. */
	175	readonly reds: number;
	176	/** Waits for a verdict of its own, gated alone beside the line. */
	177	readonly alone: boolean;
	178	/** Green alone at this head: never at the head of a batch again. */
	179	readonly cleared: boolean;
	180	}
	181
162	182	/** A first candidate's verdict, for the red-batch rate. */
163	183	interface BatchVerdict {
164	184	readonly at: number;
		@@ -182,6 +202,8 @@ interface Line {
182	202	gates: number;
183	203	/** Pulls handed back red, by pull id: re-checked alone if they join again. */
184	204	readonly reds: Map<string, Remembered>;
	205	/** Pulls in red batches, by pull id, while their head is the same (#4999). */
	206	readonly suspects: Map<string, Suspect>;
185	207	/** The last hour's candidate verdicts, search prefixes and re-checks not counted. */
186	208	batches: BatchVerdict[];
187	209	draining: boolean;
		@@ -236,6 +258,7 @@ function lineOf(repoId: string): Line {
236	258	flights: new Set(),
237	259	gates: 0,
238	260	reds: new Map(),
	261	suspects: new Map(),
239	262	batches: [],
240	263	draining: false,
241	264	wake: undefined,
		@@ -263,13 +286,17 @@ export function enqueue(
263	286	const red = line.reds.get(entry.pull.id);
264	287	if (red !== undefined && red.head !== entry.pull.headOid) line.reds.delete(entry.pull.id);
265	288	const recheck = red !== undefined && red.head === entry.pull.headOid;
	289	if (line.suspects.get(entry.pull.id)?.head !== entry.pull.headOid) line.suspects.delete(entry.pull.id);
266	290	return new Promise<boolean>((resolve) =>
267	291	line.waiting.push({
268	292	entry,
269	293	recheck,
270	294	settle: (landed) => {
271	295	line.holds.delete(entry.pull.id);
272		if (landed) line.reds.delete(entry.pull.id);
	296	if (landed) {
	297	line.reds.delete(entry.pull.id);
	298	line.suspects.delete(entry.pull.id);
	299	}
273	300	resolve(landed);
274	301	},
275	302	}),
		@@ -441,7 +468,9 @@ const ARRIVED = Symbol("arrived");
441	468	* one place and one base branch; a sealed pull whose objects live
442	469	* elsewhere is a batch of its own, and is not stacked on another. A batch
443	470	* carries at most `MAX_CANDIDATE` pulls. A gate beyond the line's first
444		* starts only on a free check slot (`room`).
	471	* starts only on a free check slot (`room`). A suspect is gated alone,
	472	* beside the line and first (#4999); the line's first gate waits for a
	473	* free slot while one is.
445	474	*/
446	475	async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Promise<void> {
447	476	line.draining = true;
		@@ -449,19 +478,41 @@ async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
449	478	const depth = Math.max(1, Math.floor(options.depth ?? DEFAULT_DEPTH));
450	479	const room: Room = options.room ?? (() => 0);
451	480	const ahead: Flight[] = [];
	481	const solos = new Set<Promise<void>>();
452	482	try {
453		while (line.waiting.length > 0 \|\| ahead.length > 0) {
454		while (ahead.length < depth && line.waiting.length > 0 && (ahead.length === 0 \|\| room() > 0)) {
	483	while (line.waiting.length > 0 \|\| ahead.length > 0 \|\| solos.size > 0) {
	484	for (const w of line.waiting.filter((w) => suspectOf(line, w)?.alone === true)) {
	485	if (line.flights.size > 0 && room() <= 0) break;
	486	line.waiting = line.waiting.filter((x) => x !== w);
	487	const started = await solo(line, gated, w);
	488	if (started === undefined) continue;
	489	const done: Promise<void> = started.done.finally(() => {
	490	solos.delete(done);
	491	line.wake?.();
	492	line.wake = undefined;
	493	});
	494	solos.add(done);
	495	}
	496	while (
	497	ahead.length < depth &&
	498	line.waiting.some((w) => suspectOf(line, w)?.alone !== true) &&
	499	((ahead.length === 0 && solos.size === 0) \|\| room() > 0)
	500	) {
455	501	const flight = await launch(line, gated, ahead.at(-1));
456	502	if (flight === "apart") break;
457	503	if (flight !== undefined) ahead.push(flight);
458	504	}
459	505	const head = ahead[0];
460		if (head === undefined) continue;
461	506	const arrived =
462		ahead.length < depth && line.waiting.length === 0
	507	head === undefined \|\| ahead.length < depth \|\| solos.size > 0
463	508	? new Promise<typeof ARRIVED>((resolve) => (line.wake = () => resolve(ARRIVED)))
464	509	: undefined;
	510	if (head === undefined) {
	511	// Only suspects at the gate, or waiting on a slot: wait for one to come back.
	512	if (solos.size > 0) await arrived;
	513	line.wake = undefined;
	514	continue;
	515	}
465	516	const verdict = arrived === undefined ? await head.verdict : await Promise.race([head.verdict, arrived]);
466	517	line.wake = undefined;
467	518	if (verdict === ARRIVED) continue;
		@@ -482,23 +533,104 @@ async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
482	533	}
483	534	}
484	535
	536	/** What the line knows of a waiting pull's red batches, at its head. */
	537	function suspectOf(line: Line, w: Waiting): Suspect \| undefined {
	538	const s = line.suspects.get(w.entry.pull.id);
	539	return s?.head === w.entry.pull.headOid ? s : undefined;
	540	}
	541
	542	/**
	543	* Start a suspect's gate alone on the base's tip, beside the line (#4999):
	544	* its answer is when the verdict is acted on. Red alone, it is handed back
	545	* and the pulls of its batch are cleared of that red; green alone, it goes
	546	* back to the line to ride at a batch's tail. Undefined when there is
	547	* nothing to gate — it conflicts, or is already in — and the next sweep
	548	* brings it up to date or closes it.
	549	*/
	550	async function solo(line: Line, gated: Gated, w: Waiting): Promise<{ readonly done: Promise<void> } \| undefined> {
	551	const settled = (error: unknown): void => {
	552	log.warn("merge queue round failed", { error: String(error) });
	553	w.settle(false);
	554	};
	555	let candidate: Candidate \| undefined;
	556	try {
	557	candidate = await build([w]);
	558	} catch (error) {
	559	settled(error);
	560	return undefined;
	561	}
	562	if (candidate === undefined \|\| candidate.steps.length === 0) {
	563	w.settle(false);
	564	return undefined;
	565	}
	566	const on = candidate;
	567	const done = gated(on)
	568	.then((verdict) => {
	569	const pull = w.entry.pull;
	570	if (verdict.lands) {
	571	line.suspects.set(pull.id, { head: pull.headOid, reds: 0, alone: false, cleared: true });
	572	line.waiting.unshift(w);
	573	return;
	574	}
	575	if ("interrupted" in verdict) {
	576	w.settle(false);
	577	return;
	578	}
	579	line.suspects.delete(pull.id);
	580	line.reds.set(pull.id, { head: pull.headOid, together: [] });
	581	log.info("red alone", { repo: line.repoId, pull: pull.number });
	582	w.entry.handBack(verdict, on.baseOid, []);
	583	w.settle(false);
	584	})
	585	.catch(settled);
	586	return { done };
	587	}
	588
	589	/**
	590	* Every pull of a red batch was in one more red batch (#4999). Its head is
	591	* the suspect when `split`; a pull in two red batches is one too, unless it
	592	* was green alone at this head.
	593	*/
	594	function suspected(line: Line, steps: readonly Waiting[], split: boolean): void {
	595	for (const [i, w] of steps.entries()) {
	596	const was = suspectOf(line, w);
	597	const reds = (was?.reds ?? 0) + 1;
	598	const cleared = was?.cleared ?? false;
	599	const alone = !cleared && (reds >= 2 \|\| (split && i === 0));
	600	line.suspects.set(w.entry.pull.id, { head: w.entry.pull.headOid, reds, alone, cleared });
	601	}
	602	}
	603
	604	/** A culprit found among `steps` explains the red batch they were in: one red each is forgiven. */
	605	function explained(line: Line, steps: readonly Waiting[]): void {
	606	for (const w of steps) {
	607	const was = suspectOf(line, w);
	608	if (was === undefined) continue;
	609	const reds = Math.max(0, was.reds - 1);
	610	line.suspects.set(w.entry.pull.id, { ...was, reds, alone: !was.cleared && reds >= 2 });
	611	}
	612	}
	613
485	614	/**
486	615	* Take the next batch from the front of the line, build it — on `on`'s tip
487	616	* when there is a flight to stack on — and start its gate. Fresh pulls go
488		* first; a re-check is a batch of one, once no fresh pull waits. `apart`
	617	* first, a pull green alone after a red batch at their tail; a re-check is
	618	* a batch of one, once no fresh pull waits. A suspect is never in a batch:
	619	* it is gated alone (`solo`). `apart`
489	620	* when the front of the line cannot stack on `on`, or `on` is a re-check;
490	621	* undefined when nothing of the batch could be built.
491	622	*/
492	623	async function launch(line: Line, gated: Gated, on: Flight \| undefined): Promise<Flight \| undefined \| "apart"> {
493		const front = line.waiting.find((w) => !w.recheck) ?? line.waiting[0]!;
	624	const cleared = (w: Waiting): boolean => suspectOf(line, w)?.cleared === true;
	625	const open = line.waiting.filter((w) => suspectOf(line, w)?.alone !== true);
	626	const front = open.find((w) => !w.recheck && !cleared(w)) ?? open.find((w) => !w.recheck) ?? open[0]!;
494	627	const first = front.entry;
495	628	const branch = first.pull.baseBranch;
496	629	if (on !== undefined && (on.recheck \|\| first.dir !== on.candidate.dir \|\| branch !== on.branch)) return "apart";
	630	const same = open.filter((w) => !w.recheck && w.entry.dir === first.dir && w.entry.pull.baseBranch === branch);
497	631	const batch = front.recheck
498	632	? [front]
499		: line.waiting
500		.filter((w) => !w.recheck && w.entry.dir === first.dir && w.entry.pull.baseBranch === branch)
501		.slice(0, MAX_CANDIDATE);
	633	: [...same.filter((w) => !cleared(w)), ...same.filter(cleared)].slice(0, MAX_CANDIDATE);
502	634	line.waiting = line.waiting.filter((w) => !batch.includes(w));
503	635	let candidate: Candidate \| undefined;
504	636	try {
		@@ -560,7 +692,24 @@ async function settleHead(search: Search, head: Flight, verdict: LandVerdict, ah
560	692	for (const w of head.steps) w.settle(false);
561	693	return behind;
562	694	}
563		return [...(await judged(search, candidate, head.steps, verdict, [])), ...behind];
	695	// A red batch is not formed again around its head (#4999): with a slot
	696	// free, the head is gated alone beside the line while the rest gate
	697	// without it. Two steps are the search's own split: the head alone.
	698	const { line } = search;
	699	const split =
	700	!head.recheck && head.steps.length > 2 && suspectOf(line, head.steps[0]!) === undefined && search.room() > 0;
	701	suspected(line, head.steps, split);
	702	if (split) {
	703	log.info("red batch: its head is gated alone", {
	704	repo: line.repoId,
	705	pull: head.steps[0]!.entry.pull.number,
	706	rest: head.steps.length - 1,
	707	});
	708	return [...head.steps, ...behind];
	709	}
	710	const left = await judged(search, candidate, head.steps, verdict, []);
	711	if (head.steps.some((w) => line.reds.get(w.entry.pull.id)?.head === w.entry.pull.headOid)) explained(line, head.steps);
	712	return [...left, ...behind];
564	713	}
565	714
566	715	/**
		@@ -654,6 +803,7 @@ async function judged(
654	803	// them to tell which. Remembered with them either way (#4235).
655	804	const together = landed.map((q) => q.pull.number);
656	805	search.line.reds.set(culprit.pull.id, { head: culprit.pull.headOid, together });
	806	search.line.suspects.delete(culprit.pull.id);
657	807	log.info(together.length === 0 ? "red on its own merge" : "red on top of its candidate", {
658	808	repo: search.line.repoId,
659	809	pull: culprit.pull.number,
		@@ -151,7 +151,7 @@ describe("the merge queue in parallel", () => {
151	151	expect(git(repo, "rev-list", "--count", "refs/heads/main")).toBe("17");
152	152	});
153	153
154		test("ten pulls, one red: the culprit is found in one wave of prefixes, and the nine land by the third gate-time", async () => {
	154	test("ten pulls, one red: the head is gated alone, the culprit is found in one wave of prefixes, and the nine land by the fourth gate-time", async () => {
155	155	const repo = await autonomousRepo("ten-one-red");
156	156	const pulls = [];
157	157	for (let i = 0; i < 10; i++) pulls.push(await greenPull(repo, i === 4 ? `red-${i}.ts` : `p${i}.ts`));
		@@ -159,14 +159,17 @@ describe("the merge queue in parallel", () => {
159	159	const sweep = sweepGreen(app, repo);
160	160	// Wave one: eight on main, the other two on top of them.
161	161	await letGo(2);
162		// Wave two: the eight are red; the two on top are re-formed, and seven
163		// prefixes of the eight gate at once: four green, the fifth red.
	162	// Wave two: the eight are red, so their head is gated alone (#4999),
	163	// beside the other seven and the next on main and the last on top.
	164	await letGo(3);
	165	// Wave three: the head was green alone; the eight without it are red
	166	// again, and seven prefixes of them gate at once: three green, the fourth red.
164	167	await letGo(7);
165		// Wave three: the culprit is handed back without a gate of its own;
166		// the three after it and the two re-formed are one candidate.
	168	// Wave four: the culprit is handed back without a gate of its own; the
	169	// four after it, the last and the head (at the tail) are one candidate.
167	170	await letGo(1);
168	171	expect(await sweep).toBe(9);
169		expect(wave - first).toBe(3);
	172	expect(wave - first).toBe(4);
170	173	const red = pulls[4]!;
171	174	for (const pull of pulls) expect(merged(repo, pull)).toBe(pull !== red);
172	175	expect(latestDecision(app.db, red.id, "land")?.answer).toBe("red");
		@@ -0,0 +1,122 @@
	1	/**
	2	* A red batch is never formed again around the same head (#4999). The head
	3	* of a red batch is gated alone beside the line while the rest gate without
	4	* it, so the good pulls land while the suspect is judged; a pull in two red
	5	* batches waits for a verdict of its own and never heads a third. The gate
	6	* here reads the candidate's tree: red when it holds `bad.txt`.
	7	*/
	8	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks/queue.ts";
	13	import type { LandVerdict } from "../src/checks/tiers.ts";
	14	import type { Pull } from "../src/core/pulls.ts";
	15
	16	describe("the merge queue judges a suspect alone", () => {
	17	let dir: string;
	18	/** Every hand-back: the pull, and the pulls it was red on top of. */
	19	const told: { pull: number; together: number[] }[] = [];
	20
	21	const git = (...args: string[]) =>
	22	Bun.spawnSync(["git", "-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", ...args]).stdout.toString().trim();
	23
	24	beforeAll(async () => {
	25	dir = await mkdtemp(join(tmpdir(), "forge-merge-queue-suspect-"));
	26	Bun.spawnSync(["git", "init", "-q", "-b", "main", dir]);
	27	git("commit", "-q", "--allow-empty", "-m", "start");
	28	});
	29
	30	afterAll(async () => {
	31	await rm(dir, { recursive: true, force: true });
	32	});
	33
	34	/** A pull's head: one file added on main as it is now. */
	35	async function headAdding(branch: string, file: string): Promise<string> {
	36	git("checkout", "-q", "-B", branch, "main");
	37	git("reset", "-q", "--hard", "main");
	38	await Bun.write(join(dir, file), `${file}\n`);
	39	git("add", file);
	40	git("commit", "-q", "-m", file);
	41	git("checkout", "-q", "--detach");
	42	return git("rev-parse", branch);
	43	}
	44
	45	const verdictOf = (candidate: Candidate): LandVerdict =>
	46	git("ls-tree", "--name-only", candidate.tipOid).split("\n").includes("bad.txt")
	47	? { lands: false, reopen: true, comment: "red", learn: [] }
	48	: { lands: true };
	49
	50	/** A queued pull that lands the way the forge does: only on the base it was gated on. */
	51	function entry(line: string, number: number, headOid: string): Queued {
	52	return {
	53	pull: { id: `${line}-${number}`, number, baseBranch: "main", headOid } as Pull,
	54	dir,
	55	land: async (expect: Expect) => {
	56	if (git("rev-parse", "refs/heads/main") !== expect.baseOid) return undefined;
	57	const tip = git("commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", `#${number}`);
	58	git("update-ref", "refs/heads/main", tip);
	59	return tip;
	60	},
	61	release: () => {},
	62	handBack: (_red, _base, together) => told.push({ pull: number, together: [...together] }),
	63	};
	64	}
	65
	66	test("one bad pull at the head, thirty good behind: the thirty land in two gate-times, the bad one is handed back after its solo red", async () => {
	67	const bad = entry("thirty", 1, await headAdding("bad", "bad.txt"));
	68	const good: Queued[] = [];
	69	for (let i = 0; i < 30; i++) good.push(entry("thirty", 2 + i, await headAdding(`g${i}`, `g${i}.txt`)));
	70	// The gate is held open and let go a wave at a time: "at once" is counted, never timed.
	71	let pending: { candidate: Candidate; go: (v: LandVerdict) => void }[] = [];
	72	const waves: number[][][] = [];
	73	const gate = (candidate: Candidate) =>
	74	new Promise<LandVerdict>((go) => pending.push({ candidate, go }));
	75	const letGo = async (n: number) => {
	76	for (let i = 0; i < 2000 && pending.length < n; i++) await Bun.sleep(5);
	77	expect(pending).toHaveLength(n);
	78	const going = pending;
	79	pending = [];
	80	waves.push(going.map((p) => p.candidate.steps.map((s) => s.entry.pull.number)));
	81	for (const p of going) p.go(verdictOf(p.candidate));
	82	};
	83	const landed = enqueue("thirty", [bad, ...good], gate, { depth: 4, room: () => 8 });
	84	// Gate-time one: four candidates, the first red on its head.
	85	await letGo(4);
	86	// Gate-time two: the head alone, beside the thirty in four candidates without it.
	87	await letGo(5);
	88	expect(await landed).toEqual([false, ...good.map(() => true)]);
	89	expect(waves[1]!.filter((w) => w.includes(1))).toEqual([[1]]);
	90	expect(waves[1]!.flat().sort((a, b) => a - b)).toEqual([1, ...good.map((q) => q.pull.number)]);
	91	expect(told).toEqual([{ pull: 1, together: [] }]);
	92	const files = git("ls-tree", "--name-only", "refs/heads/main").split("\n");
	93	expect(files).toHaveLength(30);
	94	expect(files).not.toContain("bad.txt");
	95	});
	96
	97	test("a pull in two red batches never heads a third: it is gated alone, and handed back on its own red", async () => {
	98	told.length = 0;
	99	const bad = entry("twice", 1, await headAdding("bad2", "bad.txt"));
	100	const [g1, g2] = [entry("twice", 2, await headAdding("h1", "h1.txt")), entry("twice", 3, await headAdding("h2", "h2.txt"))];
	101	const gates: number[][] = [];
	102	// The search is cut off twice (the forge stopping): the batch is red, and nothing is decided.
	103	let cut = true;
	104	const gate = async (candidate: Candidate): Promise<LandVerdict> => {
	105	gates.push(candidate.steps.map((s) => s.entry.pull.number));
	106	if (cut && candidate.steps.length < 3) return { lands: false, interrupted: true };
	107	return verdictOf(candidate);
	108	};
	109	expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, false, false]);
	110	expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, false, false]);
	111	expect(gates.filter((g) => g.length === 3)).toHaveLength(2);
	112	cut = false;
	113	const before = gates.length;
	114	expect(await enqueue("twice", [bad, g1, g2], gate)).toEqual([false, true, true]);
	115	const after = gates.slice(before);
	116	// Each was in two red batches: each is judged alone, and the bad one heads no batch.
	117	expect(after.slice(0, 3)).toEqual([[1], [2], [3]]);
	118	expect(after.every((g) => g[0] !== 1 \|\| g.length === 1)).toBe(true);
	119	expect(told).toEqual([{ pull: 1, together: [] }]);
	120	expect(git("ls-tree", "--name-only", "refs/heads/main").split("\n")).not.toContain("bad.txt");
	121	});
	122	});