d110c97 · e9c03b1 · 0fc7886
@@ -32,11 +32,11 @@
spec:
      annotations:
        plat.sh/user-env-rev: {{ .Values.userEnvRev | default "init" | quote }}
    spec:
      # An hour for an SSH session to end on its own before the old pod is
      # killed anyway.
      # The crew drains for three minutes, then runs are cut and resume on
      # the next pod; a little more than that here.
      terminationGracePeriodSeconds: 300
      # the next pod. Checks and merge-queue gates under way drain for
      # FORGE_CHECKS_DRAIN_S (660 s, a full gate's ten minutes and more), so
      # a roll does not cut a gate off; a minute more than that here (#6722).
      terminationGracePeriodSeconds: 720
      {{- with .Values.imagePullSecrets }}
      imagePullSecrets:
        {{- toYaml . | nindent 8 }}
@@ -1456,7 +1456,7 @@
a link to the milestone (`docs/PLANNING.md`); its progress never does.
| `FORGE_CREW_CONCURRENCY`  | 1 per 3 GiB above 2 GiB, 2–16   | agent runs at once (live: Settings → Capacity)    |
| `FORGE_CHECKS_CONCURRENCY` | `1`                            | checks at once (live: Settings → Capacity)        |
| `FORGE_CHECKS_PLACEMENT`  | `process`                       | `job` runs a check's commands as a cluster Job (see Placement above) |
| `FORGE_CHECKS_DRAIN_S`       | `300`                          | seconds a roll lets running checks finish first   |
| `FORGE_CHECKS_DRAIN_S`       | `660`                          | seconds a roll lets running checks finish first   |
| `FORGE_CHECK_SHARDS`      | the machine's cores             | test shards per check (live: Settings → Capacity) |
| `FORGE_CELLS_EGRESS`      | `enforce`                       | `report` lets undeclared egress through on 80/443 (live: Settings → Capacity) |
| `FORGE_CELL_CGROUP`       | `/sys/fs/cgroup` at a namespace root | the delegated cgroup v2 whose children cap each cell's memory; `off` watches only |
@@ -152,6 +152,26 @@
Nothing is committed: a commit to main would void every gate in flight
(#6640). `src/core/releases-before-tags.json` keeps the releases cut before
this, frozen.
## Gates in flight across a roll (#6722)
A roll does not turn a merge-queue gate red. On SIGTERM the old forge
starts no new gate and lets the gates under way finish for
`FORGE_CHECKS_DRAIN_S` (660 s: a full gate takes about ten minutes). The
chart's `terminationGracePeriodSeconds` is 720 for that reason, a minute
past the drain; lower one and lower the other with it, or the kubelet kills
the pod while its gates still run. The old and new pods share the node for
up to those 12 minutes, which section 1's rule already sizes for.
What the drain does not finish is no verdict, never red:
- a gate cut off at the drain's end, or one whose check a signal from
  outside ended (the pod going, the kernel's OOM killer, exit 137 or 143),
  is `interrupted`: no red batch, no suspect, and the new pod gates the same
  candidate again (`src/checks/queue.ts`, #6611);
- a gate that came back `red (check)` — red with no test and no type error
  named — is gated once more before the queue bisects it, as an install red
  is. Red again, it is searched as the code's.
## The gate of record (#2794)
A release ships on two proofs, and nothing else holds it:
@@ -207,6 +207,23 @@
const VERSION_BANNER_RE = /^\S+ v[\d.]+ *\(/;
 */
const ERROR_LINE_RE = /^[A-Za-z]*Error\b/i;
const TAIL_LINES = 20;
/** A shell's code for a child a signal ended: 128 + the signal's number. */
const SIGNAL_CODES: Readonly<Record<number, NodeJS.Signals>> = {
  129: "SIGHUP",
  130: "SIGINT",
  137: "SIGKILL",
  143: "SIGTERM",
};
/** What `runStep` says of a step a signal no one here sent ended (#6722). */
const KILLED = / was killed by SIG[A-Z]+$/;
/**
 * Whether a step's error is a signal from outside ending it — the pod
 * going, the kernel's OOM killer — not the step's own exit (#6722).
 */
export function killedStep(error: string): boolean {
  return KILLED.test(error);
}
const REASON_MAX = 300;
/**
@@ -330,6 +347,9 @@
export async function runStep(
    // The kernel's OOM kill at the cell's cap is a breach no watch tick saw.
    const kernel = code === 0 ? undefined : cell?.breach();
    if (kernel !== undefined) return err(`${name} stopped: it ${kernel}`);
    // Ended by a signal this run never sent: no verdict of the step's own.
    const killer = code === 0 ? undefined : (proc.signalCode ?? SIGNAL_CODES[code]);
    if (killer !== undefined) return err(`${name} was killed by ${killer}`);
    if (code !== 0) {
      const reason = recent.length > 0 ? ` — ${failureReason(recent)}` : "";
      return err(`${name} exited with code ${code}${reason}`);
@@ -46,8 +46,13 @@
 * `typecheck` or `check` is the code's and searched as above; `install`,
 * `setup` or `timeout` names no pull's code, so the gate runs once more and
 * red that way twice is infrastructure — no verdict, nothing handed back,
 * nothing bisected. A pull that conflicts with an earlier one of its batch
 * is never red: it is left out, the pair named, and the rest gate at once.
 * nothing bisected. A `check` red at the line's own gate (a batch or a
 * suspect, not a search's prefix) is gated once more before it is searched
 * (#6722): a check cut short that printed no reason is not yet a red. A
 * check a signal from outside ended — the pod going, the OOM killer — is
 * no verdict at all, as a gate cut off is not. A pull that conflicts with
 * an earlier one of its batch is never red: it is left out, the pair named,
 * and the rest gate at once.
 *
 * What is queued is not stored: it is the pulls `landIfGreen` admitted, and
 * every sweep admits them again. The queue only orders them and remembers
@@ -649,6 +654,15 @@
function machines(verdict: LandVerdict): verdict is Red {
  return !verdict.lands && !("interrupted" in verdict) && !searched(verdict.reason);
}
/**
 * A red that names neither a test nor a type error (#6722): the check
 * exited red and said nothing a pull could be searched for — a check cut
 * short by a reason it did not print. Gated once more before it is searched.
 */
function unproven(verdict: LandVerdict): verdict is Red {
  return !verdict.lands && !("interrupted" in verdict) && verdict.reason?.kind === "check";
}
/** Gate runs a repository's line has made since boot. */
export function gateRuns(repoId: string): number {
  return lines.get(repoId)?.gates ?? 0;
@@ -658,13 +672,26 @@
export function gateRuns(repoId: string): number {
 * The line's gate (`attempt`), and once more when it was red for a reason
 * that names no pull's code (#5276, #5093): an install, the gate's own
 * setup, a timeout. Red that way twice is infrastructure — no verdict, as a
 * gate cut off is not: no pull is handed back and nothing is searched.
 * gate cut off is not: no pull is handed back and nothing is searched. A
 * line's own `red (check)` — no test named, no type error — is gated once
 * more too (#6722), and red again it is searched; a `search` gate is not.
 */
function counted(line: Line, gate: GateCandidate, adopt: QueueOptions["adopt"]): Gated {
function counted(line: Line, gate: GateCandidate, adopt: QueueOptions["adopt"], search: boolean): Gated {
  const once = attempt(line, gate, adopt);
  return async (candidate, signal, resumed) => {
    const first = await once(candidate, signal, resumed);
    if (!machines(first) || signal?.aborted === true) return first;
    // A stopped line gates nothing more: its verdict is kept for the next.
    if (signal?.aborted === true || line.stopped) return first;
    // The line's own red (check) is gated once more before the search
    // spends a gate on it (#6722); a search's prefix is not.
    if (!search && unproven(first)) {
      log.info(`gated ${pullsWord(candidate.steps.length)}: red (check) named no test, gating once more`, {
        repo: line.repoId,
        pulls: candidate.steps.map((s) => `#${s.entry.pull.number}`).join(","),
      
        junipercrew
        18d ago
        
      
      This retry costs a whole second gate for every real check red. Consider gating the retry fast, or only when the first run ended under a second or so, since a check cut short with no reason is usually quick to fail.
    
      });
      return once(candidate, signal);
    }
    if (!machines(first)) return first;
    const again = await once(candidate, signal);
    if (!machines(again)) return again;
    log.warn(`gated ${pullsWord(candidate.steps.length)}: infrastructure (${reasonWords(again.reason)}, twice)`, {
@@ -796,7 +823,8 @@
const ARRIVED = Symbol("arrived");
 */
async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Promise<void> {
  line.draining = true;
  const gated = counted(line, gate, options.adopt);
  const gated = counted(line, gate, options.adopt, false);
  const searching = counted(line, gate, options.adopt, true);
  const depth = Math.max(1, Math.floor(options.depth ?? DEFAULT_DEPTH));
  const room: Room = options.room ?? (() => 0);
  const ahead: Flight[] = [];
@@ -845,7 +873,7 @@
async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
      ahead.shift();
      let left: Waiting[];
      try {
        left = await settleHead({ line, gated, room, onBatch: options.onBatch }, head, verdict, ahead);
        left = await settleHead({ line, gated: searching, room, onBatch: options.onBatch }, head, verdict, ahead);
      } catch (error) {
        log.warn("merge queue round failed", { error: String(error) });
        for (const w of head.steps) w.settle(false);
@@ -71,7 +71,7 @@
import { exportTree, readFileAt } from "../apps/checkout.ts";
import { parseCheck } from "../apps/manifest.ts";
import { workerCpus } from "../core/capacity.ts";
import { accountActive } from "../core/accounts.ts";
import { childEnv, runStep } from "../apps/process.ts";
import { childEnv, killedStep, runStep } from "../apps/process.ts";
import { noteReached } from "../apps/egress_report.ts";
import {
  capacityCount,
@@ -737,6 +737,8 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
        infrastructure?: boolean;
        /** Where it stopped red (#5276); none is before any step ran. */
        stage?: GateStage;
        /** The check step was ended by a signal from outside: no verdict (#6722). */
        killed?: boolean;
      }
  > {
    const { bare, repo, write, signal } = x;
@@ -1041,7 +1043,8 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
          checkLines,
          ran.error,
        );
        return { ok: false, error: summary, detail, stage: checkStage(ran.error) };
        const killed = killedStep(ran.error);
        return { ok: false, error: killed ? ran.error : summary, detail, stage: checkStage(ran.error), ...(killed ? { killed } : {}) };
      }
      // Only a green check of the default branch is kept for the next.
      if (
@@ -1278,13 +1281,15 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
      });
      if (done.ok) return { ok: true, error: "", lines };
      const stage = done.stage ?? "setup";
      // Cut off by this forge stopping, the Job lost under it, or an
      // install the machine failed twice: the code was not judged, so
      // this is no red. A gate the drain lets finish is judged (#6611).
      // Cut off by this forge stopping, the Job lost under it, the check
      // killed by a signal from outside (the pod going, the OOM killer), or
      // an install the machine failed twice: the code was not judged, so
      // this is no red. A gate the drain lets finish is judged (#6611, #6722).
      if (
        controller.signal.aborted ||
        done.error === JOB_VANISHED ||
        done.infrastructure === true
        done.infrastructure === true ||
        done.killed === true
      )
        return { ok: false, error: done.error, lines, interrupted: true, stage };
      return { ok: false, error: done.error, lines, stage };
@@ -358,8 +358,10 @@
const checks =
        concurrency: positive(process.env.FORGE_CHECKS_CONCURRENCY, Number(capacityDefault("checks.concurrency"))),
        redHoldMs: RED_HOLD_MS,
        landSettleMs: LAND_SETTLE_MS,
        // A roll lets running checks finish this long before cutting them off.
        drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 300) * 1000,
        // A roll lets running checks and merge-queue gates finish this long
        // before cutting them off: a full gate takes about ten minutes, and
        // the pod's grace (charts/app, 720 s) is longer still (#6722).
        drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 660) * 1000,
      });
// Migrated repositories kept as a read-only mirror push to their source.
const mirrorOut = startMirrorOut(app);
@@ -0,0 +1,103 @@
/**
 * A gate whose check a signal from outside ended — the pod going, the
 * kernel's OOM killer — is no verdict (#6722): the check never said red,
 * so the gate answers `interrupted`, as one the forge stopping cut off
 * does, and the merge queue hands no one back and marks no suspect. A
 * check that exits red on its own is still red.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { killedStep, runStep } from "../src/apps/process.ts";
import { gateCandidate } from "../src/checks/landing.ts";
import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
import { getPull } from "../src/core/pulls.ts";
import { createUser, type User } from "../src/core/users.ts";
import { repoDir } from "../src/git/repos.ts";
describe("a step a signal from outside ended", () => {
  const step = (command: string) =>
    runStep("check", command, { cwd: tmpdir(), env: { PATH: process.env["PATH"] ?? "" }, log: () => {}, signal: new AbortController().signal });
  test("says it was killed, by which signal, whether the shell or its child took it", async () => {
    const own = await step("kill -KILL $$");
    expect(own).toEqual({ ok: false, error: "check was killed by SIGKILL" });
    const child = await step("sh -c 'kill -TERM $$'; exit $?");
    expect(child).toEqual({ ok: false, error: "check was killed by SIGTERM" });
    expect(killedStep(own.ok ? "" : own.error)).toBe(true);
  });
  test("a step that exits red on its own was not killed", async () => {
    const red = await step("exit 1");
    expect(red.ok).toBe(false);
    expect(killedStep(red.ok ? "" : red.error)).toBe(false);
  });
});
describe("a gate on land whose check was killed", () => {
  let dataDir: string;
  let app: App;
  let checks: ChecksHandle;
  let alice: User;
  const cells = process.env["FORGE_CELLS"];
  beforeAll(async () => {
    process.env["FORGE_CELLS"] = "off";
    dataDir = await mkdtemp(join(tmpdir(), "forge-gate-killed-"));
    app = createApp({ dataDir, port: 0 });
    checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 2 });
    const a = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!a.ok) throw new Error(a.error);
    alice = a.value;
  });
  afterAll(async () => {
    await checks.stop({ drain: false });
    app.close();
    if (cells === undefined) delete process.env["FORGE_CELLS"];
    else process.env["FORGE_CELLS"] = cells;
    await rm(dataDir, { recursive: true, force: true });
  });
  /** Gate a pull of a repository whose full check is `full`. */
  async function gateWith(name: string, full: string) {
    const made = await createRepo(app, alice, { name, visibility: "public" });
    if (!made.ok) throw new Error(String(made.error));
    const repo = made.value;
    const seeded = await commitToBranch(app, alice, repo, {
      message: "Start",
      files: [{ path: "forge.json", content: JSON.stringify({ check: { run: "true", fast: "true", full } }) }],
    });
    if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
    const branched = await commitToBranch(app, alice, repo, {
      branch: "feature",
      message: "Touch b",
      files: [{ path: "b.txt", content: "b\n" }],
    });
    if (!branched.ok) throw new Error(JSON.stringify(branched.error));
    const opened = await openPull(app, alice, repo, { title: "Feature", baseBranch: "main", headBranch: "feature" });
    if (!opened.ok) throw new Error(String(opened.error));
    const pull = getPull(app.db, repo.id, opened.value.number)!;
    return gateCandidate(app, repo, pull, repoDir(app.reposRoot, repo.ownerName, repo.name), pull.headOid);
  }
  test("answers interrupted, not red (check)", async () => {
    expect(await gateWith("killed", "echo test/slow.test.ts:; kill -KILL $$")).toEqual({ lands: false, interrupted: true });
  }, 30_000);
  test("a check that exits red on its own is red (check)", async () => {
    const verdict = await gateWith("exited", "echo test/slow.test.ts:; exit 1");
    expect(verdict).toMatchObject({ lands: false, reason: { kind: "check" } });
  }, 30_000);
});
@@ -0,0 +1,117 @@
/**
 * A `red (check)` — the check exited red and named no test and no type
 * error — is gated once more before the queue searches it (#6722): a check
 * cut short that printed no reason is not yet a red. Green the second time,
 * the batch lands whole with nothing bisected; red again, it is searched,
 * and the search's own prefixes are not gated twice.
 */
import { afterAll, afterEach, beforeAll, describe, expect, spyOn, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks/queue.ts";
import type { LandVerdict } from "../src/checks/tiers.ts";
import type { Pull } from "../src/core/pulls.ts";
describe("the merge queue gates a red (check) once more before it searches", () => {
  let dir: string;
  const told: number[] = [];
  let logged: string[] = [];
  const spies: { mockRestore(): void }[] = [];
  const git = (...args: string[]) =>
    Bun.spawnSync(["git", "-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", ...args]).stdout.toString().trim();
  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), "forge-merge-queue-check-retry-"));
    Bun.spawnSync(["git", "init", "-q", "-b", "main", dir]);
    git("commit", "-q", "--allow-empty", "-m", "start");
    for (const level of ["info", "warn"] as const)
      spies.push(spyOn(console, level).mockImplementation((...args: unknown[]) => void logged.push(args.join(" "))));
  });
  afterEach(() => {
    logged = [];
    told.length = 0;
  });
  afterAll(async () => {
    for (const s of spies) s.mockRestore();
    await rm(dir, { recursive: true, force: true });
  });
  async function headWriting(branch: string, file: string): Promise<string> {
    git("checkout", "-q", "-B", branch, "main");
    git("reset", "-q", "--hard", "main");
    await Bun.write(join(dir, file), `${file}\n`);
    git("add", file);
    git("commit", "-q", "-m", file);
    git("checkout", "-q", "--detach");
    return git("rev-parse", branch);
  }
  function entry(prefix: string, number: number, headOid: string): Queued {
    return {
      pull: { id: `${prefix}-${number}`, number, baseBranch: "main", headOid } as Pull,
      dir,
      land: async (expect: Expect) => {
        if (git("rev-parse", "refs/heads/main") !== expect.baseOid) return undefined;
        const tip = git("commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", `#${number}`);
        git("update-ref", "refs/heads/main", tip);
        return tip;
      },
      release: () => {},
      handBack: () => told.push(number),
    };
  }
  const pullsOf = (c: Candidate) => c.steps.map((s) => s.entry.pull.number);
  const redCheck: LandVerdict = { lands: false, reopen: true, comment: "red", learn: [], reason: { kind: "check" } };
  test("red (check) once, then green: the batch lands whole on the second gate, nothing bisected", async () => {
    const pulls: Queued[] = [];
    for (let i = 1; i <= 4; i++) pulls.push(entry("once", i, await headWriting(`a${i}`, `a${i}.txt`)));
    const gates: number[][] = [];
    const gate = async (c: Candidate): Promise<LandVerdict> => {
      gates.push(pullsOf(c));
      return gates.length === 1 ? redCheck : { lands: true };
    };
    const answers = await enqueue("check-retry-once", pulls, gate, { depth: 1, room: () => 3 });
    expect(gates).toEqual([[1, 2, 3, 4], [1, 2, 3, 4]]);
    expect(answers).toEqual([true, true, true, true]);
    expect(told).toEqual([]);
    expect(logged.some((l) => l.includes("red (check) named no test, gating once more"))).toBe(true);
    expect(logged.some((l) => l.includes("gated alone"))).toBe(false);
  });
  test("red (check) twice is the code's: searched, each prefix gated once, the culprit handed back", async () => {
    const pulls: Queued[] = [];
    for (let i = 11; i <= 12; i++) pulls.push(entry("twice", i, await headWriting(`b${i}`, i === 12 ? "bad.txt" : `b${i}.txt`)));
    const gates: number[][] = [];
    const gate = async (c: Candidate): Promise<LandVerdict> => {
      gates.push(pullsOf(c));
      return git("ls-tree", "--name-only", c.tipOid).split("\n").includes("bad.txt") ? redCheck : { lands: true };
    };
    const answers = await enqueue("check-retry-twice", pulls, gate, { depth: 1, room: () => 0 });
    // The batch twice, then the search's front half once: green, it lands, and #12 is the culprit.
    expect(gates).toEqual([[11, 12], [11, 12], [11]]);
    expect(answers).toEqual([true, false]);
    expect(told).toEqual([12]);
  });
  test("a test red is searched at once, as before", async () => {
    const pulls: Queued[] = [];
    for (let i = 21; i <= 22; i++) pulls.push(entry("tests", i, await headWriting(`c${i}`, i === 22 ? "bad.txt" : `c${i}.txt`)));
    const gates: number[][] = [];
    const gate = async (c: Candidate): Promise<LandVerdict> => {
      gates.push(pullsOf(c));
      return git("ls-tree", "--name-only", c.tipOid).split("\n").includes("bad.txt")
        ? { ...redCheck, reason: { kind: "tests", failed: 1 } }
        : { lands: true };
    };
    const answers = await enqueue("check-retry-tests", pulls, gate, { depth: 1, room: () => 0 });
    expect(gates).toEqual([[21, 22], [21]]);
    expect(answers).toEqual([true, false]);
    expect(told).toEqual([22]);
  });
});
@@ -190,6 +190,25 @@
describe("the merge queue across a restart", () => {
    expect(git(repo, "rev-parse", "main^{tree}")).toBe(record.treeOid);
  });
  test("a red (check) the stopped forge's gate came back with is gated once more as the same candidate, nothing searched (#6722)", async () => {
    const { repo, pulls, tip, record } = await killedMidGate("restart-red-check");
    // The roll's drain answers red with no test named: kept, not acted on.
    running.get(tip)!.go({ ok: false, error: "check exited with code 1", lines: [], stage: "check" });
    for (let i = 0; i < 200 && gateRecords(app.db, repo.id)[0]?.verdict === undefined; i++) await Bun.sleep(5);
    expect(gateRecords(app.db, repo.id)[0]?.verdict).toMatchObject({ lands: false, reason: { kind: "check" } });
    lost = true;
    const again = sweepGreen(app, repo);
    await gating(1);
    // The very commit that was gated before, gated once more before any search.
    expect(started).toEqual([tip, tip]);
    running.get(tip)!.go({ ok: true, error: "", lines: [] });
    expect(await again).toBe(3);
    expect(started).toEqual([tip, tip]);
    for (const pull of pulls) expect(merged(repo, pull)).toBe(true);
    expect(git(repo, "rev-parse", "main^{tree}")).toBe(record.treeOid);
    expect(redMarks(app.db, repo.id).size).toBe(0);
  });
  test("a recorded candidate whose base moved is forgotten, and the line is planned afresh", async () => {
    const { repo, pulls, tip } = await killedMidGate("restart-moved");
    running.get(tip)!.go({ ok: false, error: "cut off", lines: [], interrupted: true });
		@@ -32,11 +32,11 @@ spec:
32	32	annotations:
33	33	plat.sh/user-env-rev: {{ .Values.userEnvRev \| default "init" \| quote }}
34	34	spec:
35		# An hour for an SSH session to end on its own before the old pod is
36		# killed anyway.
37	35	# The crew drains for three minutes, then runs are cut and resume on
38		# the next pod; a little more than that here.
39		terminationGracePeriodSeconds: 300
	36	# the next pod. Checks and merge-queue gates under way drain for
	37	# FORGE_CHECKS_DRAIN_S (660 s, a full gate's ten minutes and more), so
	38	# a roll does not cut a gate off; a minute more than that here (#6722).
	39	terminationGracePeriodSeconds: 720
40	40	{{- with .Values.imagePullSecrets }}
41	41	imagePullSecrets:
42	42	{{- toYaml . \| nindent 8 }}
		@@ -1456,7 +1456,7 @@ a link to the milestone (`docs/PLANNING.md`); its progress never does.
1456	1456	\| `FORGE_CREW_CONCURRENCY` \| 1 per 3 GiB above 2 GiB, 2–16 \| agent runs at once (live: Settings → Capacity) \|
1457	1457	\| `FORGE_CHECKS_CONCURRENCY` \| `1` \| checks at once (live: Settings → Capacity) \|
1458	1458	\| `FORGE_CHECKS_PLACEMENT` \| `process` \| `job` runs a check's commands as a cluster Job (see Placement above) \|
1459		\| `FORGE_CHECKS_DRAIN_S` \| `300` \| seconds a roll lets running checks finish first \|
	1459	\| `FORGE_CHECKS_DRAIN_S` \| `660` \| seconds a roll lets running checks finish first \|
1460	1460	\| `FORGE_CHECK_SHARDS` \| the machine's cores \| test shards per check (live: Settings → Capacity) \|
1461	1461	\| `FORGE_CELLS_EGRESS` \| `enforce` \| `report` lets undeclared egress through on 80/443 (live: Settings → Capacity) \|
1462	1462	\| `FORGE_CELL_CGROUP` \| `/sys/fs/cgroup` at a namespace root \| the delegated cgroup v2 whose children cap each cell's memory; `off` watches only \|
		@@ -152,6 +152,26 @@ Nothing is committed: a commit to main would void every gate in flight
152	152	(#6640). `src/core/releases-before-tags.json` keeps the releases cut before
153	153	this, frozen.
154	154
	155	## Gates in flight across a roll (#6722)
	156
	157	A roll does not turn a merge-queue gate red. On SIGTERM the old forge
	158	starts no new gate and lets the gates under way finish for
	159	`FORGE_CHECKS_DRAIN_S` (660 s: a full gate takes about ten minutes). The
	160	chart's `terminationGracePeriodSeconds` is 720 for that reason, a minute
	161	past the drain; lower one and lower the other with it, or the kubelet kills
	162	the pod while its gates still run. The old and new pods share the node for
	163	up to those 12 minutes, which section 1's rule already sizes for.
	164
	165	What the drain does not finish is no verdict, never red:
	166
	167	- a gate cut off at the drain's end, or one whose check a signal from
	168	outside ended (the pod going, the kernel's OOM killer, exit 137 or 143),
	169	is `interrupted`: no red batch, no suspect, and the new pod gates the same
	170	candidate again (`src/checks/queue.ts`, #6611);
	171	- a gate that came back `red (check)` — red with no test and no type error
	172	named — is gated once more before the queue bisects it, as an install red
	173	is. Red again, it is searched as the code's.
	174
155	175	## The gate of record (#2794)
156	176
157	177	A release ships on two proofs, and nothing else holds it:
		@@ -207,6 +207,23 @@ const VERSION_BANNER_RE = /^\S+ v[\d.]+ *\(/;
207	207	*/
208	208	const ERROR_LINE_RE = /^[A-Za-z]*Error\b/i;
209	209	const TAIL_LINES = 20;
	210	/** A shell's code for a child a signal ended: 128 + the signal's number. */
	211	const SIGNAL_CODES: Readonly<Record<number, NodeJS.Signals>> = {
	212	129: "SIGHUP",
	213	130: "SIGINT",
	214	137: "SIGKILL",
	215	143: "SIGTERM",
	216	};
	217	/** What `runStep` says of a step a signal no one here sent ended (#6722). */
	218	const KILLED = / was killed by SIG[A-Z]+$/;
	219
	220	/**
	221	* Whether a step's error is a signal from outside ending it — the pod
	222	* going, the kernel's OOM killer — not the step's own exit (#6722).
	223	*/
	224	export function killedStep(error: string): boolean {
	225	return KILLED.test(error);
	226	}
210	227	const REASON_MAX = 300;
211	228
212	229	/**
		@@ -330,6 +347,9 @@ export async function runStep(
330	347	// The kernel's OOM kill at the cell's cap is a breach no watch tick saw.
331	348	const kernel = code === 0 ? undefined : cell?.breach();
332	349	if (kernel !== undefined) return err(`${name} stopped: it ${kernel}`);
	350	// Ended by a signal this run never sent: no verdict of the step's own.
	351	const killer = code === 0 ? undefined : (proc.signalCode ?? SIGNAL_CODES[code]);
	352	if (killer !== undefined) return err(`${name} was killed by ${killer}`);
333	353	if (code !== 0) {
334	354	const reason = recent.length > 0 ? ` — ${failureReason(recent)}` : "";
335	355	return err(`${name} exited with code ${code}${reason}`);
		@@ -46,8 +46,13 @@
46	46	* `typecheck` or `check` is the code's and searched as above; `install`,
47	47	* `setup` or `timeout` names no pull's code, so the gate runs once more and
48	48	* red that way twice is infrastructure — no verdict, nothing handed back,
49		* nothing bisected. A pull that conflicts with an earlier one of its batch
50		* is never red: it is left out, the pair named, and the rest gate at once.
	49	* nothing bisected. A `check` red at the line's own gate (a batch or a
	50	* suspect, not a search's prefix) is gated once more before it is searched
	51	* (#6722): a check cut short that printed no reason is not yet a red. A
	52	* check a signal from outside ended — the pod going, the OOM killer — is
	53	* no verdict at all, as a gate cut off is not. A pull that conflicts with
	54	* an earlier one of its batch is never red: it is left out, the pair named,
	55	* and the rest gate at once.
51	56	*
52	57	* What is queued is not stored: it is the pulls `landIfGreen` admitted, and
53	58	* every sweep admits them again. The queue only orders them and remembers
		@@ -649,6 +654,15 @@ function machines(verdict: LandVerdict): verdict is Red {
649	654	return !verdict.lands && !("interrupted" in verdict) && !searched(verdict.reason);
650	655	}
651	656
	657	/**
	658	* A red that names neither a test nor a type error (#6722): the check
	659	* exited red and said nothing a pull could be searched for — a check cut
	660	* short by a reason it did not print. Gated once more before it is searched.
	661	*/
	662	function unproven(verdict: LandVerdict): verdict is Red {
	663	return !verdict.lands && !("interrupted" in verdict) && verdict.reason?.kind === "check";
	664	}
	665
652	666	/** Gate runs a repository's line has made since boot. */
653	667	export function gateRuns(repoId: string): number {
654	668	return lines.get(repoId)?.gates ?? 0;
		@@ -658,13 +672,26 @@ export function gateRuns(repoId: string): number {
658	672	* The line's gate (`attempt`), and once more when it was red for a reason
659	673	* that names no pull's code (#5276, #5093): an install, the gate's own
660	674	* setup, a timeout. Red that way twice is infrastructure — no verdict, as a
661		* gate cut off is not: no pull is handed back and nothing is searched.
	675	* gate cut off is not: no pull is handed back and nothing is searched. A
	676	* line's own `red (check)` — no test named, no type error — is gated once
	677	* more too (#6722), and red again it is searched; a `search` gate is not.
662	678	*/
663		function counted(line: Line, gate: GateCandidate, adopt: QueueOptions["adopt"]): Gated {
	679	function counted(line: Line, gate: GateCandidate, adopt: QueueOptions["adopt"], search: boolean): Gated {
664	680	const once = attempt(line, gate, adopt);
665	681	return async (candidate, signal, resumed) => {
666	682	const first = await once(candidate, signal, resumed);
667		if (!machines(first) \|\| signal?.aborted === true) return first;
	683	// A stopped line gates nothing more: its verdict is kept for the next.
	684	if (signal?.aborted === true \|\| line.stopped) return first;
	685	// The line's own red (check) is gated once more before the search
	686	// spends a gate on it (#6722); a search's prefix is not.
	687	if (!search && unproven(first)) {
	688	log.info(`gated ${pullsWord(candidate.steps.length)}: red (check) named no test, gating once more`, {
	689	repo: line.repoId,
	690	pulls: candidate.steps.map((s) => `#${s.entry.pull.number}`).join(","),
		junipercrew 18d ago This retry costs a whole second gate for every real check red. Consider gating the retry fast, or only when the first run ended under a second or so, since a check cut short with no reason is usually quick to fail.
	691	});
	692	return once(candidate, signal);
	693	}
	694	if (!machines(first)) return first;
668	695	const again = await once(candidate, signal);
669	696	if (!machines(again)) return again;
670	697	log.warn(`gated ${pullsWord(candidate.steps.length)}: infrastructure (${reasonWords(again.reason)}, twice)`, {
		@@ -796,7 +823,8 @@ const ARRIVED = Symbol("arrived");
796	823	*/
797	824	async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Promise<void> {
798	825	line.draining = true;
799		const gated = counted(line, gate, options.adopt);
	826	const gated = counted(line, gate, options.adopt, false);
	827	const searching = counted(line, gate, options.adopt, true);
800	828	const depth = Math.max(1, Math.floor(options.depth ?? DEFAULT_DEPTH));
801	829	const room: Room = options.room ?? (() => 0);
802	830	const ahead: Flight[] = [];
		@@ -845,7 +873,7 @@ async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
845	873	ahead.shift();
846	874	let left: Waiting[];
847	875	try {
848		left = await settleHead({ line, gated, room, onBatch: options.onBatch }, head, verdict, ahead);
	876	left = await settleHead({ line, gated: searching, room, onBatch: options.onBatch }, head, verdict, ahead);
849	877	} catch (error) {
850	878	log.warn("merge queue round failed", { error: String(error) });
851	879	for (const w of head.steps) w.settle(false);
		@@ -71,7 +71,7 @@ import { exportTree, readFileAt } from "../apps/checkout.ts";
71	71	import { parseCheck } from "../apps/manifest.ts";
72	72	import { workerCpus } from "../core/capacity.ts";
73	73	import { accountActive } from "../core/accounts.ts";
74		import { childEnv, runStep } from "../apps/process.ts";
	74	import { childEnv, killedStep, runStep } from "../apps/process.ts";
75	75	import { noteReached } from "../apps/egress_report.ts";
76	76	import {
77	77	capacityCount,
		@@ -737,6 +737,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
737	737	infrastructure?: boolean;
738	738	/** Where it stopped red (#5276); none is before any step ran. */
739	739	stage?: GateStage;
	740	/** The check step was ended by a signal from outside: no verdict (#6722). */
	741	killed?: boolean;
740	742	}
741	743	> {
742	744	const { bare, repo, write, signal } = x;
		@@ -1041,7 +1043,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
1041	1043	checkLines,
1042	1044	ran.error,
1043	1045	);
1044		return { ok: false, error: summary, detail, stage: checkStage(ran.error) };
	1046	const killed = killedStep(ran.error);
	1047	return { ok: false, error: killed ? ran.error : summary, detail, stage: checkStage(ran.error), ...(killed ? { killed } : {}) };
1045	1048	}
1046	1049	// Only a green check of the default branch is kept for the next.
1047	1050	if (
		@@ -1278,13 +1281,15 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
1278	1281	});
1279	1282	if (done.ok) return { ok: true, error: "", lines };
1280	1283	const stage = done.stage ?? "setup";
1281		// Cut off by this forge stopping, the Job lost under it, or an
1282		// install the machine failed twice: the code was not judged, so
1283		// this is no red. A gate the drain lets finish is judged (#6611).
	1284	// Cut off by this forge stopping, the Job lost under it, the check
	1285	// killed by a signal from outside (the pod going, the OOM killer), or
	1286	// an install the machine failed twice: the code was not judged, so
	1287	// this is no red. A gate the drain lets finish is judged (#6611, #6722).
1284	1288	if (
1285	1289	controller.signal.aborted \|\|
1286	1290	done.error === JOB_VANISHED \|\|
1287		done.infrastructure === true
	1291	done.infrastructure === true \|\|
	1292	done.killed === true
1288	1293	)
1289	1294	return { ok: false, error: done.error, lines, interrupted: true, stage };
1290	1295	return { ok: false, error: done.error, lines, stage };
		@@ -358,8 +358,10 @@ const checks =
358	358	concurrency: positive(process.env.FORGE_CHECKS_CONCURRENCY, Number(capacityDefault("checks.concurrency"))),
359	359	redHoldMs: RED_HOLD_MS,
360	360	landSettleMs: LAND_SETTLE_MS,
361		// A roll lets running checks finish this long before cutting them off.
362		drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 300) * 1000,
	361	// A roll lets running checks and merge-queue gates finish this long
	362	// before cutting them off: a full gate takes about ten minutes, and
	363	// the pod's grace (charts/app, 720 s) is longer still (#6722).
	364	drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 660) * 1000,
363	365	});
364	366	// Migrated repositories kept as a read-only mirror push to their source.
365	367	const mirrorOut = startMirrorOut(app);
		@@ -0,0 +1,103 @@
	1	/**
	2	* A gate whose check a signal from outside ended — the pod going, the
	3	* kernel's OOM killer — is no verdict (#6722): the check never said red,
	4	* so the gate answers `interrupted`, as one the forge stopping cut off
	5	* does, and the merge queue hands no one back and marks no suspect. A
	6	* check that exits red on its own is still red.
	7	*/
	8	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import { openPull } from "../src/actions/pulls.ts";
	14	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	15	import { killedStep, runStep } from "../src/apps/process.ts";
	16	import { gateCandidate } from "../src/checks/landing.ts";
	17	import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
	18	import { getPull } from "../src/core/pulls.ts";
	19	import { createUser, type User } from "../src/core/users.ts";
	20	import { repoDir } from "../src/git/repos.ts";
	21
	22	describe("a step a signal from outside ended", () => {
	23	const step = (command: string) =>
	24	runStep("check", command, { cwd: tmpdir(), env: { PATH: process.env["PATH"] ?? "" }, log: () => {}, signal: new AbortController().signal });
	25
	26	test("says it was killed, by which signal, whether the shell or its child took it", async () => {
	27	const own = await step("kill -KILL $$");
	28	expect(own).toEqual({ ok: false, error: "check was killed by SIGKILL" });
	29	const child = await step("sh -c 'kill -TERM $$'; exit $?");
	30	expect(child).toEqual({ ok: false, error: "check was killed by SIGTERM" });
	31	expect(killedStep(own.ok ? "" : own.error)).toBe(true);
	32	});
	33
	34	test("a step that exits red on its own was not killed", async () => {
	35	const red = await step("exit 1");
	36	expect(red.ok).toBe(false);
	37	expect(killedStep(red.ok ? "" : red.error)).toBe(false);
	38	});
	39	});
	40
	41	describe("a gate on land whose check was killed", () => {
	42	let dataDir: string;
	43	let app: App;
	44	let checks: ChecksHandle;
	45	let alice: User;
	46	const cells = process.env["FORGE_CELLS"];
	47
	48	beforeAll(async () => {
	49	process.env["FORGE_CELLS"] = "off";
	50	dataDir = await mkdtemp(join(tmpdir(), "forge-gate-killed-"));
	51	app = createApp({ dataDir, port: 0 });
	52	checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 2 });
	53	const a = await createUser(app.db, {
	54	origin: "seed",
	55	trusted: true,
	56	username: "alice",
	57	email: "[email protected]",
	58	password: "a-good-passphrase",
	59	adminIfFirst: true,
	60	});
	61	if (!a.ok) throw new Error(a.error);
	62	alice = a.value;
	63	});
	64
	65	afterAll(async () => {
	66	await checks.stop({ drain: false });
	67	app.close();
	68	if (cells === undefined) delete process.env["FORGE_CELLS"];
	69	else process.env["FORGE_CELLS"] = cells;
	70	await rm(dataDir, { recursive: true, force: true });
	71	});
	72
	73	/** Gate a pull of a repository whose full check is `full`. */
	74	async function gateWith(name: string, full: string) {
	75	const made = await createRepo(app, alice, { name, visibility: "public" });
	76	if (!made.ok) throw new Error(String(made.error));
	77	const repo = made.value;
	78	const seeded = await commitToBranch(app, alice, repo, {
	79	message: "Start",
	80	files: [{ path: "forge.json", content: JSON.stringify({ check: { run: "true", fast: "true", full } }) }],
	81	});
	82	if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
	83	const branched = await commitToBranch(app, alice, repo, {
	84	branch: "feature",
	85	message: "Touch b",
	86	files: [{ path: "b.txt", content: "b\n" }],
	87	});
	88	if (!branched.ok) throw new Error(JSON.stringify(branched.error));
	89	const opened = await openPull(app, alice, repo, { title: "Feature", baseBranch: "main", headBranch: "feature" });
	90	if (!opened.ok) throw new Error(String(opened.error));
	91	const pull = getPull(app.db, repo.id, opened.value.number)!;
	92	return gateCandidate(app, repo, pull, repoDir(app.reposRoot, repo.ownerName, repo.name), pull.headOid);
	93	}
	94
	95	test("answers interrupted, not red (check)", async () => {
	96	expect(await gateWith("killed", "echo test/slow.test.ts:; kill -KILL $$")).toEqual({ lands: false, interrupted: true });
	97	}, 30_000);
	98
	99	test("a check that exits red on its own is red (check)", async () => {
	100	const verdict = await gateWith("exited", "echo test/slow.test.ts:; exit 1");
	101	expect(verdict).toMatchObject({ lands: false, reason: { kind: "check" } });
	102	}, 30_000);
	103	});
		@@ -0,0 +1,117 @@
	1	/**
	2	* A `red (check)` — the check exited red and named no test and no type
	3	* error — is gated once more before the queue searches it (#6722): a check
	4	* cut short that printed no reason is not yet a red. Green the second time,
	5	* the batch lands whole with nothing bisected; red again, it is searched,
	6	* and the search's own prefixes are not gated twice.
	7	*/
	8	import { afterAll, afterEach, beforeAll, describe, expect, spyOn, test } from "bun:test";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { type Candidate, enqueue, type Expect, type Queued } from "../src/checks/queue.ts";
	13	import type { LandVerdict } from "../src/checks/tiers.ts";
	14	import type { Pull } from "../src/core/pulls.ts";
	15
	16	describe("the merge queue gates a red (check) once more before it searches", () => {
	17	let dir: string;
	18	const told: number[] = [];
	19	let logged: string[] = [];
	20	const spies: { mockRestore(): void }[] = [];
	21
	22	const git = (...args: string[]) =>
	23	Bun.spawnSync(["git", "-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", ...args]).stdout.toString().trim();
	24
	25	beforeAll(async () => {
	26	dir = await mkdtemp(join(tmpdir(), "forge-merge-queue-check-retry-"));
	27	Bun.spawnSync(["git", "init", "-q", "-b", "main", dir]);
	28	git("commit", "-q", "--allow-empty", "-m", "start");
	29	for (const level of ["info", "warn"] as const)
	30	spies.push(spyOn(console, level).mockImplementation((...args: unknown[]) => void logged.push(args.join(" "))));
	31	});
	32
	33	afterEach(() => {
	34	logged = [];
	35	told.length = 0;
	36	});
	37
	38	afterAll(async () => {
	39	for (const s of spies) s.mockRestore();
	40	await rm(dir, { recursive: true, force: true });
	41	});
	42
	43	async function headWriting(branch: string, file: string): Promise<string> {
	44	git("checkout", "-q", "-B", branch, "main");
	45	git("reset", "-q", "--hard", "main");
	46	await Bun.write(join(dir, file), `${file}\n`);
	47	git("add", file);
	48	git("commit", "-q", "-m", file);
	49	git("checkout", "-q", "--detach");
	50	return git("rev-parse", branch);
	51	}
	52
	53	function entry(prefix: string, number: number, headOid: string): Queued {
	54	return {
	55	pull: { id: `${prefix}-${number}`, number, baseBranch: "main", headOid } as Pull,
	56	dir,
	57	land: async (expect: Expect) => {
	58	if (git("rev-parse", "refs/heads/main") !== expect.baseOid) return undefined;
	59	const tip = git("commit-tree", expect.treeOid, "-p", expect.baseOid, "-m", `#${number}`);
	60	git("update-ref", "refs/heads/main", tip);
	61	return tip;
	62	},
	63	release: () => {},
	64	handBack: () => told.push(number),
	65	};
	66	}
	67
	68	const pullsOf = (c: Candidate) => c.steps.map((s) => s.entry.pull.number);
	69	const redCheck: LandVerdict = { lands: false, reopen: true, comment: "red", learn: [], reason: { kind: "check" } };
	70
	71	test("red (check) once, then green: the batch lands whole on the second gate, nothing bisected", async () => {
	72	const pulls: Queued[] = [];
	73	for (let i = 1; i <= 4; i++) pulls.push(entry("once", i, await headWriting(`a${i}`, `a${i}.txt`)));
	74	const gates: number[][] = [];
	75	const gate = async (c: Candidate): Promise<LandVerdict> => {
	76	gates.push(pullsOf(c));
	77	return gates.length === 1 ? redCheck : { lands: true };
	78	};
	79	const answers = await enqueue("check-retry-once", pulls, gate, { depth: 1, room: () => 3 });
	80	expect(gates).toEqual([[1, 2, 3, 4], [1, 2, 3, 4]]);
	81	expect(answers).toEqual([true, true, true, true]);
	82	expect(told).toEqual([]);
	83	expect(logged.some((l) => l.includes("red (check) named no test, gating once more"))).toBe(true);
	84	expect(logged.some((l) => l.includes("gated alone"))).toBe(false);
	85	});
	86
	87	test("red (check) twice is the code's: searched, each prefix gated once, the culprit handed back", async () => {
	88	const pulls: Queued[] = [];
	89	for (let i = 11; i <= 12; i++) pulls.push(entry("twice", i, await headWriting(`b${i}`, i === 12 ? "bad.txt" : `b${i}.txt`)));
	90	const gates: number[][] = [];
	91	const gate = async (c: Candidate): Promise<LandVerdict> => {
	92	gates.push(pullsOf(c));
	93	return git("ls-tree", "--name-only", c.tipOid).split("\n").includes("bad.txt") ? redCheck : { lands: true };
	94	};
	95	const answers = await enqueue("check-retry-twice", pulls, gate, { depth: 1, room: () => 0 });
	96	// The batch twice, then the search's front half once: green, it lands, and #12 is the culprit.
	97	expect(gates).toEqual([[11, 12], [11, 12], [11]]);
	98	expect(answers).toEqual([true, false]);
	99	expect(told).toEqual([12]);
	100	});
	101
	102	test("a test red is searched at once, as before", async () => {
	103	const pulls: Queued[] = [];
	104	for (let i = 21; i <= 22; i++) pulls.push(entry("tests", i, await headWriting(`c${i}`, i === 22 ? "bad.txt" : `c${i}.txt`)));
	105	const gates: number[][] = [];
	106	const gate = async (c: Candidate): Promise<LandVerdict> => {
	107	gates.push(pullsOf(c));
	108	return git("ls-tree", "--name-only", c.tipOid).split("\n").includes("bad.txt")
	109	? { ...redCheck, reason: { kind: "tests", failed: 1 } }
	110	: { lands: true };
	111	};
	112	const answers = await enqueue("check-retry-tests", pulls, gate, { depth: 1, room: () => 0 });
	113	expect(gates).toEqual([[21, 22], [21]]);
	114	expect(answers).toEqual([true, false]);
	115	expect(told).toEqual([22]);
	116	});
	117	});
		@@ -190,6 +190,25 @@ describe("the merge queue across a restart", () => {
190	190	expect(git(repo, "rev-parse", "main^{tree}")).toBe(record.treeOid);
191	191	});
192	192
	193	test("a red (check) the stopped forge's gate came back with is gated once more as the same candidate, nothing searched (#6722)", async () => {
	194	const { repo, pulls, tip, record } = await killedMidGate("restart-red-check");
	195	// The roll's drain answers red with no test named: kept, not acted on.
	196	running.get(tip)!.go({ ok: false, error: "check exited with code 1", lines: [], stage: "check" });
	197	for (let i = 0; i < 200 && gateRecords(app.db, repo.id)[0]?.verdict === undefined; i++) await Bun.sleep(5);
	198	expect(gateRecords(app.db, repo.id)[0]?.verdict).toMatchObject({ lands: false, reason: { kind: "check" } });
	199	lost = true;
	200	const again = sweepGreen(app, repo);
	201	await gating(1);
	202	// The very commit that was gated before, gated once more before any search.
	203	expect(started).toEqual([tip, tip]);
	204	running.get(tip)!.go({ ok: true, error: "", lines: [] });
	205	expect(await again).toBe(3);
	206	expect(started).toEqual([tip, tip]);
	207	for (const pull of pulls) expect(merged(repo, pull)).toBe(true);
	208	expect(git(repo, "rev-parse", "main^{tree}")).toBe(record.treeOid);
	209	expect(redMarks(app.db, repo.id).size).toBe(0);
	210	});
	211
193	212	test("a recorded candidate whose base moved is forgotten, and the line is planned afresh", async () => {
194	213	const { repo, pulls, tip } = await killedMidGate("restart-moved");
195	214	running.get(tip)!.go({ ok: false, error: "cut off", lines: [], interrupted: true });