87d0a11 · 8dd302b · 34bea24
@@ -34,6 +34,10 @@
function readValid(path: string): Map<string, string[]> | undefined {
  }
}
/** The repository's kept map, when there is a valid one. */
export const keptImpact = (dataDir: string, repoId: string): Map<string, string[]> | undefined =>
  readValid(storeOf(dataDir, repoId));
/** Copies the kept map into a checkout; how many tests it maps, or 0 for none. */
export function seedImpact(dataDir: string, repoId: string, checkout: string): number {
  const map = readValid(storeOf(dataDir, repoId));
@@ -0,0 +1,102 @@
/**
 * A pull red only on tests its diff does not reach is checked once more on
 * the same head before one of its builder's tries is spent on it (#7630).
 * An order-dependent test the change never touched parked #7552 twice, a
 * day of the owner's ask each time. So when every failing test file is one
 * the impact map (`impact.ts`) says executes none of the pull's changed
 * files, the check runs again; a second red on that head goes back to the
 * builder as any red does.
 *
 * One fresh re-check per head, recorded: the re-check is a check run with
 * the reason {@link OUTSIDE_DIFF} on that head, so a red on it — or any
 * later red on the head, after a restart too — is never re-checked again.
 *
 * Plain data in, a verdict out ({@link redOutsideDiff}); the forge's reads
 * (the kept map, the run's log, the diff) are {@link outsideDiffCandidate}
 * and {@link pullChanged}.
 */
import type { Database } from "bun:sqlite";
import type { Pull } from "../core/pulls.ts";
import type { Run } from "../core/runs.ts";
import { runGitCapture } from "../git/env.ts";
import { resolveOid } from "../git/merge.ts";
import { readRunLog } from "../runs/log.ts";
import { failuresOf } from "./failures.ts";
import { keptImpact } from "./impact-store.ts";
import { needsFull, reach, type Impact } from "./impact.ts";
import { RULE_TESTS } from "./tiers.ts";
/** The re-check's run reason, which is also its record. */
export const OUTSIDE_DIFF = "outside_diff";
/** What the re-check's log says first. */
export const recheckLine = (files: readonly string[]): string =>
  `re-checked: red outside the diff (${files.join(", ")})`;
/** Files that shape every test: a pull changing one owns any red. */
const SHAPING = /(^|\/)(bun\.lockb?|bunfig\.toml|tsconfig[^/]*\.json|\.env[^/]*)$/;
/**
 * The failing test files when each is outside what the diff reaches, by
 * the map; `undefined` when any may be the change's. Conservative: a test
 * the map has never measured, a repo-wide rule test, a changed test, a
 * changed test helper (the map is of `src/**` only), or a diff the map
 * cannot bound at all, and the red is the pull's.
 */
export function redOutsideDiff(
  failing: readonly string[],
  changed: readonly string[],
  impact: Impact,
): string[] | undefined {
  if (failing.length === 0 || changed.length === 0) return undefined;
  if (needsFull(changed) !== undefined || changed.some((f) => SHAPING.test(f))) return undefined;
  if (changed.some((f) => f.startsWith("test/") && !/\.test\.[cm]?[jt]sx?$/.test(f))) return undefined;
  const reached = reach(changed, impact).tests;
  const inside = (file: string): boolean =>
    !impact.has(file) || RULE_TESTS.includes(file) || changed.includes(file) || reached.has(file);
  return failing.some(inside) ? undefined : [...failing].sort();
}
/** Whether this head of the pull already had its re-check. */
export function recheckedHead(db: Database, issueId: string, commitOid: string): boolean {
  return (
    db
      .query<{ n: number }, [string, string, string]>(
        `SELECT 1 AS n FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2 AND reason = ?3 LIMIT 1`,
      )
      .get(issueId, commitOid, OUTSIDE_DIFF) !== null
  );
}
/**
 * What can be known without git, in the event's own tick: a failed pull
 * check not yet re-checked on its head, whose log names failing test files
 * that the repository's kept map has all measured. The answer is the map
 * and the files; `undefined` is today's path at once.
 */
export function outsideDiffCandidate(
  db: Database,
  dataDir: string,
  run: Run | undefined,
): { impact: Impact; failing: string[] } | undefined {
  if (run === undefined || run.kind !== "check" || run.state !== "failed") return undefined;
  if (run.issueId === null || run.commitOid === "") return undefined;
  if (run.reason === OUTSIDE_DIFF || recheckedHead(db, run.issueId, run.commitOid)) return undefined;
  const impact = keptImpact(dataDir, run.repoId);
  if (impact === undefined) return undefined;
  const { files } = failuresOf(readRunLog(dataDir, run.id).lines);
  if (files.length === 0 || !files.every((f) => impact.has(f))) return undefined;
  return { impact, failing: files };
}
/** The files the pull's head changes from its base; `undefined` when git cannot say. */
export async function pullChanged(dir: string, pull: Pull, head: string): Promise<string[] | undefined> {
  const base = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
  if (base === undefined) return undefined;
  const out = await runGitCapture(
    ["diff", "--name-only", "-z", "--no-renames", `${base}...${head}`, "--"],
    { cwd: dir },
  );
  return out.code === 0 ? out.stdout.split("\0").filter((f) => f !== "") : undefined;
}
@@ -29,7 +29,7 @@
import {
  listComments,
  type Issue,
} from "../core/issues.ts";
import { closesFromBody, getPull, type Pull } from "../core/pulls.ts";
import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
import { claimedClones, sweepOrphanClones } from "./orphans.ts";
import { pruneRunFiles } from "../runs/prune.ts";
import { crewRoot } from "./workdir.ts";
@@ -96,6 +96,15 @@
import { checksFor } from "../checks/service.ts";
import { redMain, sharedWithMain } from "../checks/red_main.ts";
import { diskRecovered, diskRed } from "../checks/disk_red.ts";
import { sharedWithPulls } from "../checks/shared_red.ts";
import type { Impact } from "../checks/impact.ts";
import {
  OUTSIDE_DIFF,
  outsideDiffCandidate,
  pullChanged,
  recheckedHead,
  recheckLine,
  redOutsideDiff,
} from "../checks/outside_diff.ts";
import { certain } from "./decide.ts";
import { readRunLog } from "../runs/log.ts";
import { whenText } from "./limits.ts";
@@ -718,47 +727,24 @@
function route(app: App, event: ForgeEvent): void {
      // the hour (#2266), is not the pull's: no rework, no park, no
      // strike; it waits and is checked again when main is green.
      if (waitOnMain(app, repo, issue, event.payload)) return;
      // Bounded: after a few failed checks in a row the agent stops and a
      // person must speak (docs/APPS.md, rule 7).
      const since = lastHumanWord(app, issue, agent);
      if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
        const pull = getPull(app.db, repo.id, issue.number);
        if (
          pull &&
          parkPull(
            app,
            repo,
            pull,
            `the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries`,
          )
        )
          return;
        log.info("check failed again; waiting for a person", {
          issue: issue.number,
        });
      // A red only on tests the diff does not reach (#7630) is checked once
      // more on the same head before a try is spent on it.
      const failedRun = typeof event.payload["runId"] === "string" ? getRun(app.db, event.payload["runId"]) : undefined;
      const outside = outsideDiffCandidate(app.db, app.config.dataDir, failedRun);
      if (outside === undefined) {
        answerRed(app, repo, issue, agent, event.payload);
        return;
      }
      const runId = event.payload["runId"];
      // Without a decider the rule answers at once, and the rework queues
      // in the same tick the event landed — nothing waits on a promise.
      if (app.config.decider === undefined || typeof runId !== "string") {
        queueAgentRun(app, repo, issue, agent, "check_failed");
        return;
      }
      const commitOid = event.payload["commitOid"];
      const env = event.payload["env"];
      void answerFailedCheck(app, repo, issue, agent, {
        runId,
        kind: String(event.payload["kind"] ?? ""),
        commitOid: typeof commitOid === "string" ? commitOid : undefined,
        env: typeof env === "string" ? env : undefined,
        since,
      }).catch((error) =>
        log.error("a failed check went unanswered", {
          issue: issue.number,
          error: String(error),
        }),
      );
      void recheckOutsideDiff(app, repo, issue, event.payload, outside)
        .then((rechecked) => {
          if (!rechecked) answerRed(app, repo, issue, agent, event.payload);
        })
        .catch((error) =>
          log.error("a red outside the diff went unanswered", {
            issue: issue.number,
            error: String(error),
          }),
        );
      return;
    }
    case "pull.closed":
@@ -800,6 +786,95 @@
function route(app: App, event: ForgeEvent): void {
  }
}
/**
 * A red that is the pull's to answer: parked once the automatic tries are
 * spent, else reworked — through the decider, when there is one.
 */
function answerRed(
  app: App,
  repo: Repo,
  issue: Issue,
  agent: Agent,
  payload: Record<string, unknown>,
): void {
  // Bounded: after a few failed checks in a row the agent stops and a
  // person must speak (docs/APPS.md, rule 7).
  const since = lastHumanWord(app, issue, agent);
  if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
    const pull = getPull(app.db, repo.id, issue.number);
    if (
      pull &&
      parkPull(
        app,
        repo,
        pull,
        `the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries`,
      )
    )
      return;
    log.info("check failed again; waiting for a person", {
      issue: issue.number,
    });
    return;
  }
  const runId = payload["runId"];
  // Without a decider the rule answers at once, and the rework queues
  // in the same tick the event landed — nothing waits on a promise.
  if (app.config.decider === undefined || typeof runId !== "string") {
    queueAgentRun(app, repo, issue, agent, "check_failed");
    return;
  }
  const commitOid = payload["commitOid"];
  const env = payload["env"];
  void answerFailedCheck(app, repo, issue, agent, {
    runId,
    kind: String(payload["kind"] ?? ""),
    commitOid: typeof commitOid === "string" ? commitOid : undefined,
    env: typeof env === "string" ? env : undefined,
    since,
  }).catch((error) =>
    log.error("a failed check went unanswered", {
      issue: issue.number,
      error: String(error),
    }),
  );
}
/**
 * The re-check of a red outside the diff, when the diff says it is one:
 * whether it was asked for (or the head moved on and the red is history).
 * `false` is today's answer: the red is the pull's.
 */
async function recheckOutsideDiff(
  app: App,
  repo: Repo,
  issue: Issue,
  payload: Record<string, unknown>,
  outside: { impact: Impact; failing: string[] },
): Promise<boolean> {
  const { commitOid, env } = payload;
  const checks = checksFor(app);
  const pull = getPull(app.db, repo.id, issue.number);
  if (checks === undefined || !pull || typeof commitOid !== "string" || typeof env !== "string")
    return false;
  const dir = await pullDir(app.db, app.reposRoot, repo, pull);
  const changed = await pullChanged(dir, pull, commitOid);
  const files = changed && redOutsideDiff(outside.failing, changed, outside.impact);
  if (files === undefined) return false;
  const now = getPull(app.db, repo.id, issue.number);
  // A push since then made a newer check; this red is history.
  if (!now || now.state !== "open" || now.headOid !== commitOid) return true;
  if (recheckedHead(app.db, issue.id, commitOid)) return false;
  const run = await checks.request(repo, env, commitOid, OUTSIDE_DIFF, issue.id);
  if (run === undefined) return false;
  if (run.reason === OUTSIDE_DIFF) appendRunLog(app.config.dataDir, run.id, recheckLine(files));
  log.info("check failed only outside the diff; checking the head once more", {
    issue: issue.number,
    files: files.join(", "),
  });
  return true;
}
/**
 * Failures that arrived while the builder's run was still at work on the
 * pull, by pull. Memory only: a restart forgets them, and the pull's next
@@ -0,0 +1,179 @@
/**
 * A pull red only on a test its diff does not reach gets one fresh re-check
 * before a try is spent on it (#7630). Through the crew's event path, with
 * the impact map kept for the repository and a decider that always says
 * stop (which no longer parks: one red goes back to its builder): a red
 * outside the diff is checked once more on the same head, and the re-check
 * says so in its log; green then, the decider is never asked and no rework
 * is queued; red again, it is, and the builder reworks it. A red inside
 * the diff goes to the decider and the builder at once. Each red names its
 * own test, so no two pulls share one more than twice in the hour
 * (`shared_red.ts`).
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { App } from "../src/app.ts";
import { announceRun } from "../src/actions/runs.ts";
import { commitToBranch } from "../src/actions/repos.ts";
import { openPull } from "../src/actions/pulls.ts";
import { formatImpact } from "../src/checks/impact.ts";
import { IMPACT_FILE, keepImpact } from "../src/checks/impact-store.ts";
import { OUTSIDE_DIFF, recheckLine, redOutsideDiff } from "../src/checks/outside_diff.ts";
import { getIssueById } from "../src/core/issues.ts";
import { getPull, type Pull } from "../src/core/pulls.ts";
import type { Repo } from "../src/core/repos.ts";
import { claimRun, createRun, finishRun, listRuns, type Run } from "../src/core/runs.ts";
import type { User } from "../src/core/users.ts";
import type { Decider } from "../src/crew/decide.ts";
import { appendRunLog, readRunLog } from "../src/runs/log.ts";
import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
const IMPACT = new Map([
  ["test/people.test.ts", ["src/people.ts"]],
  ["test/feature.test.ts", ["src/feature.ts", "src/shared.ts"]],
  ["test/flake.test.ts", ["src/shared.ts"]],
]);
describe("which reds are outside the diff", () => {
  test("a mapped test that executes none of the changed files", () => {
    expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts"], IMPACT)).toEqual(["test/people.test.ts"]);
  });
  test("any failing test the diff reaches, changes, or the map has not measured is the pull's", () => {
    expect(redOutsideDiff(["test/people.test.ts", "test/feature.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
    expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts", "test/people.test.ts"], IMPACT)).toBeUndefined();
    expect(redOutsideDiff(["test/new.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
  });
  test("a diff the map cannot bound, a changed helper, or a repo-wide rule test is the pull's", () => {
    for (const unbounded of ["package.json", "bun.lock", "scripts/gate.ts", "test/helpers/x.ts"])
      expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts", unbounded], IMPACT)).toBeUndefined();
    expect(redOutsideDiff(["test/flake.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
    expect(redOutsideDiff([], ["src/feature.ts"], IMPACT)).toBeUndefined();
  });
});
describe("the crew, with a red outside the diff", () => {
  let standIn: StandInCrew;
  let app: App;
  let alice: User;
  let bot: User;
  let repo: Repo;
  let asked: string[] = [];
  let n = 0;
  beforeAll(async () => {
    standIn = await startStandInCrew();
    ({ app, alice, bot, repo } = standIn);
    standIn.checks.request = async (...args: unknown[]) => {
      const [r, env, commitOid, reason, issueId] = args as [Repo, string, string, string, string];
      return createRun(app.db, { repoId: r.id, kind: "check", env, commitOid, reason, issueId });
    };
    const decider: Decider = {
      name: "stand-in",
      async decide(state) {
        asked.push(String((state as { pull?: { number: number } }).pull?.number));
        return {
          retry: { choice: "stop", p: 1, probabilities: { again: 0, stop: 1 } },
          cause: { choice: "change", p: 1, probabilities: { change: 1, environment: 0, base: 0 } },
        } as never;
      },
    };
    (app.config as { decider?: Decider }).decider = decider;
    // Parking is autonomy's: the repository runs on its own, on alice's word.
    app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
    const checkout = mkdtempSync(join(tmpdir(), "forge-outside-diff-"));
    writeFileSync(join(checkout, IMPACT_FILE), formatImpact(IMPACT));
    expect(keepImpact(app.config.dataDir, repo.id, checkout)).toBe("impact map kept: 3 tests");
    rmSync(checkout, { recursive: true, force: true });
  }, 30_000);
  afterAll(() => stopStandInCrew(standIn));
  /** The crew's pull changing src/feature.ts. */
  const featurePull = async (): Promise<Pull> => {
    const branch = `feat-${++n}`;
    const c = await commitToBranch(app, alice, repo, {
      message: branch,
      branch,
      files: [{ path: "src/feature.ts", content: `export const n = ${n};\n` }],
    });
    if (!c.ok) throw new Error(JSON.stringify(c.error));
    const o = await openPull(app, bot, repo, { title: branch, baseBranch: "main", headBranch: branch });
    if (!o.ok) throw new Error(String(o.error));
    app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
    return getPull(app.db, repo.id, o.value.number)!;
  };
  /** A check run on the pull's head, finished and announced; its log names `file` red. */
  const finish = (run: Run, state: "ok" | "failed", file = "test/people.test.ts"): void => {
    claimRun(app.db, run.id);
    if (state === "failed") {
      appendRunLog(app.config.dataDir, run.id, `${file}:`);
      appendRunLog(app.config.dataDir, run.id, `(fail) ${file} > verify [3.00ms]`);
    }
    const done = finishRun(app.db, run.id, {
      state,
      summary: state === "ok" ? "check passed" : "check failed",
      ...(state === "failed" ? { checkDetail: { passed: 9, failed: 1, names: [`${file} > verify`] } } : {}),
    })!;
    announceRun(app, done);
  };
  const pushRed = (pull: Pull, file?: string): void =>
    finish(
      createRun(app.db, { repoId: repo.id, kind: "check", env: `pull-${pull.number}`, commitOid: pull.headOid, reason: "push", issueId: pull.id }),
      "failed",
      file,
    );
  const rechecks = (pull: Pull): Run[] =>
    listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.issueId === pull.id && r.reason === OUTSIDE_DIFF);
  const open = (pull: Pull): boolean => getIssueById(app.db, pull.id)?.state === "open";
  /** The builder's tries spent on the pull's reds. */
  const reworks = (pull: Pull): number =>
    listRuns(app.db, repo.id, { kind: "agent", limit: 50 }).filter((r) => r.issueId === pull.id && r.reason === "check_failed").length;
  const until = async (done: () => boolean): Promise<void> => {
    const deadline = Date.now() + 10_000;
    while (!done() && Date.now() < deadline) await Bun.sleep(20);
    expect(done()).toBe(true);
  };
  test("red outside the diff: one re-check on the same head, which says why; green, no try is spent", async () => {
    asked = [];
    const pull = await featurePull();
    pushRed(pull);
    await until(() => rechecks(pull).length === 1);
    const recheck = rechecks(pull)[0]!;
    expect(recheck.commitOid).toBe(pull.headOid);
    expect(readRunLog(app.config.dataDir, recheck.id).lines).toContain(recheckLine(["test/people.test.ts"]));
    expect(recheckLine(["test/people.test.ts"])).toBe("re-checked: red outside the diff (test/people.test.ts)");
    finish(recheck, "ok");
    await Bun.sleep(100);
    expect(asked).toEqual([]);
    expect(reworks(pull)).toBe(0);
    expect(open(pull)).toBe(true);
    expect(rechecks(pull)).toHaveLength(1);
  });
  test("red outside the diff twice: the second red is the builder's, one try spent, never parked", async () => {
    asked = [];
    const pull = await featurePull();
    pushRed(pull);
    await until(() => rechecks(pull).length === 1);
    expect(asked).toEqual([]);
    expect(reworks(pull)).toBe(0);
    finish(rechecks(pull)[0]!, "failed");
    await until(() => reworks(pull) === 1);
    expect(asked).toEqual([String(pull.number)]);
    expect(open(pull)).toBe(true);
    expect(rechecks(pull)).toHaveLength(1);
  });
  test("red inside the diff: no re-check, the builder's at once", async () => {
    asked = [];
    const pull = await featurePull();
    pushRed(pull, "test/feature.test.ts");
    await until(() => reworks(pull) === 1);
    expect(asked).toEqual([String(pull.number)]);
    expect(open(pull)).toBe(true);
    expect(rechecks(pull)).toEqual([]);
  });
});
		@@ -34,6 +34,10 @@ function readValid(path: string): Map<string, string[]> \| undefined {
34	34	}
35	35	}
36	36
	37	/** The repository's kept map, when there is a valid one. */
	38	export const keptImpact = (dataDir: string, repoId: string): Map<string, string[]> \| undefined =>
	39	readValid(storeOf(dataDir, repoId));
	40
37	41	/** Copies the kept map into a checkout; how many tests it maps, or 0 for none. */
38	42	export function seedImpact(dataDir: string, repoId: string, checkout: string): number {
39	43	const map = readValid(storeOf(dataDir, repoId));
		@@ -0,0 +1,102 @@
	1	/**
	2	* A pull red only on tests its diff does not reach is checked once more on
	3	* the same head before one of its builder's tries is spent on it (#7630).
	4	* An order-dependent test the change never touched parked #7552 twice, a
	5	* day of the owner's ask each time. So when every failing test file is one
	6	* the impact map (`impact.ts`) says executes none of the pull's changed
	7	* files, the check runs again; a second red on that head goes back to the
	8	* builder as any red does.
	9	*
	10	* One fresh re-check per head, recorded: the re-check is a check run with
	11	* the reason {@link OUTSIDE_DIFF} on that head, so a red on it — or any
	12	* later red on the head, after a restart too — is never re-checked again.
	13	*
	14	* Plain data in, a verdict out ({@link redOutsideDiff}); the forge's reads
	15	* (the kept map, the run's log, the diff) are {@link outsideDiffCandidate}
	16	* and {@link pullChanged}.
	17	*/
	18
	19	import type { Database } from "bun:sqlite";
	20	import type { Pull } from "../core/pulls.ts";
	21	import type { Run } from "../core/runs.ts";
	22	import { runGitCapture } from "../git/env.ts";
	23	import { resolveOid } from "../git/merge.ts";
	24	import { readRunLog } from "../runs/log.ts";
	25	import { failuresOf } from "./failures.ts";
	26	import { keptImpact } from "./impact-store.ts";
	27	import { needsFull, reach, type Impact } from "./impact.ts";
	28	import { RULE_TESTS } from "./tiers.ts";
	29
	30	/** The re-check's run reason, which is also its record. */
	31	export const OUTSIDE_DIFF = "outside_diff";
	32
	33	/** What the re-check's log says first. */
	34	export const recheckLine = (files: readonly string[]): string =>
	35	`re-checked: red outside the diff (${files.join(", ")})`;
	36
	37	/** Files that shape every test: a pull changing one owns any red. */
	38	const SHAPING = /(^\|\/)(bun\.lockb?\|bunfig\.toml\|tsconfig[^/]\.json\|\.env[^/])$/;
	39
	40	/**
	41	* The failing test files when each is outside what the diff reaches, by
	42	* the map; `undefined` when any may be the change's. Conservative: a test
	43	* the map has never measured, a repo-wide rule test, a changed test, a
	44	* changed test helper (the map is of `src/**` only), or a diff the map
	45	* cannot bound at all, and the red is the pull's.
	46	*/
	47	export function redOutsideDiff(
	48	failing: readonly string[],
	49	changed: readonly string[],
	50	impact: Impact,
	51	): string[] \| undefined {
	52	if (failing.length === 0 \|\| changed.length === 0) return undefined;
	53	if (needsFull(changed) !== undefined \|\| changed.some((f) => SHAPING.test(f))) return undefined;
	54	if (changed.some((f) => f.startsWith("test/") && !/\.test\.[cm]?[jt]sx?$/.test(f))) return undefined;
	55	const reached = reach(changed, impact).tests;
	56	const inside = (file: string): boolean =>
	57	!impact.has(file) \|\| RULE_TESTS.includes(file) \|\| changed.includes(file) \|\| reached.has(file);
	58	return failing.some(inside) ? undefined : [...failing].sort();
	59	}
	60
	61	/** Whether this head of the pull already had its re-check. */
	62	export function recheckedHead(db: Database, issueId: string, commitOid: string): boolean {
	63	return (
	64	db
	65	.query<{ n: number }, [string, string, string]>(
	66	`SELECT 1 AS n FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2 AND reason = ?3 LIMIT 1`,
	67	)
	68	.get(issueId, commitOid, OUTSIDE_DIFF) !== null
	69	);
	70	}
	71
	72	/**
	73	* What can be known without git, in the event's own tick: a failed pull
	74	* check not yet re-checked on its head, whose log names failing test files
	75	* that the repository's kept map has all measured. The answer is the map
	76	* and the files; `undefined` is today's path at once.
	77	*/
	78	export function outsideDiffCandidate(
	79	db: Database,
	80	dataDir: string,
	81	run: Run \| undefined,
	82	): { impact: Impact; failing: string[] } \| undefined {
	83	if (run === undefined \|\| run.kind !== "check" \|\| run.state !== "failed") return undefined;
	84	if (run.issueId === null \|\| run.commitOid === "") return undefined;
	85	if (run.reason === OUTSIDE_DIFF \|\| recheckedHead(db, run.issueId, run.commitOid)) return undefined;
	86	const impact = keptImpact(dataDir, run.repoId);
	87	if (impact === undefined) return undefined;
	88	const { files } = failuresOf(readRunLog(dataDir, run.id).lines);
	89	if (files.length === 0 \|\| !files.every((f) => impact.has(f))) return undefined;
	90	return { impact, failing: files };
	91	}
	92
	93	/** The files the pull's head changes from its base; `undefined` when git cannot say. */
	94	export async function pullChanged(dir: string, pull: Pull, head: string): Promise<string[] \| undefined> {
	95	const base = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
	96	if (base === undefined) return undefined;
	97	const out = await runGitCapture(
	98	["diff", "--name-only", "-z", "--no-renames", `${base}...${head}`, "--"],
	99	{ cwd: dir },
	100	);
	101	return out.code === 0 ? out.stdout.split("\0").filter((f) => f !== "") : undefined;
	102	}
		@@ -29,7 +29,7 @@ import {
29	29	listComments,
30	30	type Issue,
31	31	} from "../core/issues.ts";
32		import { closesFromBody, getPull, type Pull } from "../core/pulls.ts";
	32	import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
33	33	import { claimedClones, sweepOrphanClones } from "./orphans.ts";
34	34	import { pruneRunFiles } from "../runs/prune.ts";
35	35	import { crewRoot } from "./workdir.ts";
		@@ -96,6 +96,15 @@ import { checksFor } from "../checks/service.ts";
96	96	import { redMain, sharedWithMain } from "../checks/red_main.ts";
97	97	import { diskRecovered, diskRed } from "../checks/disk_red.ts";
98	98	import { sharedWithPulls } from "../checks/shared_red.ts";
	99	import type { Impact } from "../checks/impact.ts";
	100	import {
	101	OUTSIDE_DIFF,
	102	outsideDiffCandidate,
	103	pullChanged,
	104	recheckedHead,
	105	recheckLine,
	106	redOutsideDiff,
	107	} from "../checks/outside_diff.ts";
99	108	import { certain } from "./decide.ts";
100	109	import { readRunLog } from "../runs/log.ts";
101	110	import { whenText } from "./limits.ts";
		@@ -718,47 +727,24 @@ function route(app: App, event: ForgeEvent): void {
718	727	// the hour (#2266), is not the pull's: no rework, no park, no
719	728	// strike; it waits and is checked again when main is green.
720	729	if (waitOnMain(app, repo, issue, event.payload)) return;
721		// Bounded: after a few failed checks in a row the agent stops and a
722		// person must speak (docs/APPS.md, rule 7).
723		const since = lastHumanWord(app, issue, agent);
724		if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
725		const pull = getPull(app.db, repo.id, issue.number);
726		if (
727		pull &&
728		parkPull(
729		app,
730		repo,
731		pull,
732		`the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries`,
733		)
734		)
735		return;
736		log.info("check failed again; waiting for a person", {
737		issue: issue.number,
738		});
	730	// A red only on tests the diff does not reach (#7630) is checked once
	731	// more on the same head before a try is spent on it.
	732	const failedRun = typeof event.payload["runId"] === "string" ? getRun(app.db, event.payload["runId"]) : undefined;
	733	const outside = outsideDiffCandidate(app.db, app.config.dataDir, failedRun);
	734	if (outside === undefined) {
	735	answerRed(app, repo, issue, agent, event.payload);
739	736	return;
740	737	}
741		const runId = event.payload["runId"];
742		// Without a decider the rule answers at once, and the rework queues
743		// in the same tick the event landed — nothing waits on a promise.
744		if (app.config.decider === undefined \|\| typeof runId !== "string") {
745		queueAgentRun(app, repo, issue, agent, "check_failed");
746		return;
747		}
748		const commitOid = event.payload["commitOid"];
749		const env = event.payload["env"];
750		void answerFailedCheck(app, repo, issue, agent, {
751		runId,
752		kind: String(event.payload["kind"] ?? ""),
753		commitOid: typeof commitOid === "string" ? commitOid : undefined,
754		env: typeof env === "string" ? env : undefined,
755		since,
756		}).catch((error) =>
757		log.error("a failed check went unanswered", {
758		issue: issue.number,
759		error: String(error),
760		}),
761		);
	738	void recheckOutsideDiff(app, repo, issue, event.payload, outside)
	739	.then((rechecked) => {
	740	if (!rechecked) answerRed(app, repo, issue, agent, event.payload);
	741	})
	742	.catch((error) =>
	743	log.error("a red outside the diff went unanswered", {
	744	issue: issue.number,
	745	error: String(error),
	746	}),
	747	);
762	748	return;
763	749	}
764	750	case "pull.closed":
		@@ -800,6 +786,95 @@ function route(app: App, event: ForgeEvent): void {
800	786	}
801	787	}
802	788
	789	/**
	790	* A red that is the pull's to answer: parked once the automatic tries are
	791	* spent, else reworked — through the decider, when there is one.
	792	*/
	793	function answerRed(
	794	app: App,
	795	repo: Repo,
	796	issue: Issue,
	797	agent: Agent,
	798	payload: Record<string, unknown>,
	799	): void {
	800	// Bounded: after a few failed checks in a row the agent stops and a
	801	// person must speak (docs/APPS.md, rule 7).
	802	const since = lastHumanWord(app, issue, agent);
	803	if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
	804	const pull = getPull(app.db, repo.id, issue.number);
	805	if (
	806	pull &&
	807	parkPull(
	808	app,
	809	repo,
	810	pull,
	811	`the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries`,
	812	)
	813	)
	814	return;
	815	log.info("check failed again; waiting for a person", {
	816	issue: issue.number,
	817	});
	818	return;
	819	}
	820	const runId = payload["runId"];
	821	// Without a decider the rule answers at once, and the rework queues
	822	// in the same tick the event landed — nothing waits on a promise.
	823	if (app.config.decider === undefined \|\| typeof runId !== "string") {
	824	queueAgentRun(app, repo, issue, agent, "check_failed");
	825	return;
	826	}
	827	const commitOid = payload["commitOid"];
	828	const env = payload["env"];
	829	void answerFailedCheck(app, repo, issue, agent, {
	830	runId,
	831	kind: String(payload["kind"] ?? ""),
	832	commitOid: typeof commitOid === "string" ? commitOid : undefined,
	833	env: typeof env === "string" ? env : undefined,
	834	since,
	835	}).catch((error) =>
	836	log.error("a failed check went unanswered", {
	837	issue: issue.number,
	838	error: String(error),
	839	}),
	840	);
	841	}
	842
	843	/**
	844	* The re-check of a red outside the diff, when the diff says it is one:
	845	* whether it was asked for (or the head moved on and the red is history).
	846	* `false` is today's answer: the red is the pull's.
	847	*/
	848	async function recheckOutsideDiff(
	849	app: App,
	850	repo: Repo,
	851	issue: Issue,
	852	payload: Record<string, unknown>,
	853	outside: { impact: Impact; failing: string[] },
	854	): Promise<boolean> {
	855	const { commitOid, env } = payload;
	856	const checks = checksFor(app);
	857	const pull = getPull(app.db, repo.id, issue.number);
	858	if (checks === undefined \|\| !pull \|\| typeof commitOid !== "string" \|\| typeof env !== "string")
	859	return false;
	860	const dir = await pullDir(app.db, app.reposRoot, repo, pull);
	861	const changed = await pullChanged(dir, pull, commitOid);
	862	const files = changed && redOutsideDiff(outside.failing, changed, outside.impact);
	863	if (files === undefined) return false;
	864	const now = getPull(app.db, repo.id, issue.number);
	865	// A push since then made a newer check; this red is history.
	866	if (!now \|\| now.state !== "open" \|\| now.headOid !== commitOid) return true;
	867	if (recheckedHead(app.db, issue.id, commitOid)) return false;
	868	const run = await checks.request(repo, env, commitOid, OUTSIDE_DIFF, issue.id);
	869	if (run === undefined) return false;
	870	if (run.reason === OUTSIDE_DIFF) appendRunLog(app.config.dataDir, run.id, recheckLine(files));
	871	log.info("check failed only outside the diff; checking the head once more", {
	872	issue: issue.number,
	873	files: files.join(", "),
	874	});
	875	return true;
	876	}
	877
803	878	/**
804	879	* Failures that arrived while the builder's run was still at work on the
805	880	* pull, by pull. Memory only: a restart forgets them, and the pull's next
		@@ -0,0 +1,179 @@
	1	/**
	2	* A pull red only on a test its diff does not reach gets one fresh re-check
	3	* before a try is spent on it (#7630). Through the crew's event path, with
	4	* the impact map kept for the repository and a decider that always says
	5	* stop (which no longer parks: one red goes back to its builder): a red
	6	* outside the diff is checked once more on the same head, and the re-check
	7	* says so in its log; green then, the decider is never asked and no rework
	8	* is queued; red again, it is, and the builder reworks it. A red inside
	9	* the diff goes to the decider and the builder at once. Each red names its
	10	* own test, so no two pulls share one more than twice in the hour
	11	* (`shared_red.ts`).
	12	*/
	13
	14	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	15	import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
	16	import { tmpdir } from "node:os";
	17	import { join } from "node:path";
	18	import type { App } from "../src/app.ts";
	19	import { announceRun } from "../src/actions/runs.ts";
	20	import { commitToBranch } from "../src/actions/repos.ts";
	21	import { openPull } from "../src/actions/pulls.ts";
	22	import { formatImpact } from "../src/checks/impact.ts";
	23	import { IMPACT_FILE, keepImpact } from "../src/checks/impact-store.ts";
	24	import { OUTSIDE_DIFF, recheckLine, redOutsideDiff } from "../src/checks/outside_diff.ts";
	25	import { getIssueById } from "../src/core/issues.ts";
	26	import { getPull, type Pull } from "../src/core/pulls.ts";
	27	import type { Repo } from "../src/core/repos.ts";
	28	import { claimRun, createRun, finishRun, listRuns, type Run } from "../src/core/runs.ts";
	29	import type { User } from "../src/core/users.ts";
	30	import type { Decider } from "../src/crew/decide.ts";
	31	import { appendRunLog, readRunLog } from "../src/runs/log.ts";
	32	import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
	33
	34	const IMPACT = new Map([
	35	["test/people.test.ts", ["src/people.ts"]],
	36	["test/feature.test.ts", ["src/feature.ts", "src/shared.ts"]],
	37	["test/flake.test.ts", ["src/shared.ts"]],
	38	]);
	39
	40	describe("which reds are outside the diff", () => {
	41	test("a mapped test that executes none of the changed files", () => {
	42	expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts"], IMPACT)).toEqual(["test/people.test.ts"]);
	43	});
	44	test("any failing test the diff reaches, changes, or the map has not measured is the pull's", () => {
	45	expect(redOutsideDiff(["test/people.test.ts", "test/feature.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
	46	expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts", "test/people.test.ts"], IMPACT)).toBeUndefined();
	47	expect(redOutsideDiff(["test/new.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
	48	});
	49	test("a diff the map cannot bound, a changed helper, or a repo-wide rule test is the pull's", () => {
	50	for (const unbounded of ["package.json", "bun.lock", "scripts/gate.ts", "test/helpers/x.ts"])
	51	expect(redOutsideDiff(["test/people.test.ts"], ["src/feature.ts", unbounded], IMPACT)).toBeUndefined();
	52	expect(redOutsideDiff(["test/flake.test.ts"], ["src/feature.ts"], IMPACT)).toBeUndefined();
	53	expect(redOutsideDiff([], ["src/feature.ts"], IMPACT)).toBeUndefined();
	54	});
	55	});
	56
	57	describe("the crew, with a red outside the diff", () => {
	58	let standIn: StandInCrew;
	59	let app: App;
	60	let alice: User;
	61	let bot: User;
	62	let repo: Repo;
	63	let asked: string[] = [];
	64	let n = 0;
	65
	66	beforeAll(async () => {
	67	standIn = await startStandInCrew();
	68	({ app, alice, bot, repo } = standIn);
	69	standIn.checks.request = async (...args: unknown[]) => {
	70	const [r, env, commitOid, reason, issueId] = args as [Repo, string, string, string, string];
	71	return createRun(app.db, { repoId: r.id, kind: "check", env, commitOid, reason, issueId });
	72	};
	73	const decider: Decider = {
	74	name: "stand-in",
	75	async decide(state) {
	76	asked.push(String((state as { pull?: { number: number } }).pull?.number));
	77	return {
	78	retry: { choice: "stop", p: 1, probabilities: { again: 0, stop: 1 } },
	79	cause: { choice: "change", p: 1, probabilities: { change: 1, environment: 0, base: 0 } },
	80	} as never;
	81	},
	82	};
	83	(app.config as { decider?: Decider }).decider = decider;
	84	// Parking is autonomy's: the repository runs on its own, on alice's word.
	85	app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
	86	const checkout = mkdtempSync(join(tmpdir(), "forge-outside-diff-"));
	87	writeFileSync(join(checkout, IMPACT_FILE), formatImpact(IMPACT));
	88	expect(keepImpact(app.config.dataDir, repo.id, checkout)).toBe("impact map kept: 3 tests");
	89	rmSync(checkout, { recursive: true, force: true });
	90	}, 30_000);
	91	afterAll(() => stopStandInCrew(standIn));
	92
	93	/** The crew's pull changing src/feature.ts. */
	94	const featurePull = async (): Promise<Pull> => {
	95	const branch = `feat-${++n}`;
	96	const c = await commitToBranch(app, alice, repo, {
	97	message: branch,
	98	branch,
	99	files: [{ path: "src/feature.ts", content: `export const n = ${n};\n` }],
	100	});
	101	if (!c.ok) throw new Error(JSON.stringify(c.error));
	102	const o = await openPull(app, bot, repo, { title: branch, baseBranch: "main", headBranch: branch });
	103	if (!o.ok) throw new Error(String(o.error));
	104	app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
	105	return getPull(app.db, repo.id, o.value.number)!;
	106	};
	107	/** A check run on the pull's head, finished and announced; its log names `file` red. */
	108	const finish = (run: Run, state: "ok" \| "failed", file = "test/people.test.ts"): void => {
	109	claimRun(app.db, run.id);
	110	if (state === "failed") {
	111	appendRunLog(app.config.dataDir, run.id, `${file}:`);
	112	appendRunLog(app.config.dataDir, run.id, `(fail) ${file} > verify [3.00ms]`);
	113	}
	114	const done = finishRun(app.db, run.id, {
	115	state,
	116	summary: state === "ok" ? "check passed" : "check failed",
	117	...(state === "failed" ? { checkDetail: { passed: 9, failed: 1, names: [`${file} > verify`] } } : {}),
	118	})!;
	119	announceRun(app, done);
	120	};
	121	const pushRed = (pull: Pull, file?: string): void =>
	122	finish(
	123	createRun(app.db, { repoId: repo.id, kind: "check", env: `pull-${pull.number}`, commitOid: pull.headOid, reason: "push", issueId: pull.id }),
	124	"failed",
	125	file,
	126	);
	127	const rechecks = (pull: Pull): Run[] =>
	128	listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.issueId === pull.id && r.reason === OUTSIDE_DIFF);
	129	const open = (pull: Pull): boolean => getIssueById(app.db, pull.id)?.state === "open";
	130	/** The builder's tries spent on the pull's reds. */
	131	const reworks = (pull: Pull): number =>
	132	listRuns(app.db, repo.id, { kind: "agent", limit: 50 }).filter((r) => r.issueId === pull.id && r.reason === "check_failed").length;
	133	const until = async (done: () => boolean): Promise<void> => {
	134	const deadline = Date.now() + 10_000;
	135	while (!done() && Date.now() < deadline) await Bun.sleep(20);
	136	expect(done()).toBe(true);
	137	};
	138
	139	test("red outside the diff: one re-check on the same head, which says why; green, no try is spent", async () => {
	140	asked = [];
	141	const pull = await featurePull();
	142	pushRed(pull);
	143	await until(() => rechecks(pull).length === 1);
	144	const recheck = rechecks(pull)[0]!;
	145	expect(recheck.commitOid).toBe(pull.headOid);
	146	expect(readRunLog(app.config.dataDir, recheck.id).lines).toContain(recheckLine(["test/people.test.ts"]));
	147	expect(recheckLine(["test/people.test.ts"])).toBe("re-checked: red outside the diff (test/people.test.ts)");
	148	finish(recheck, "ok");
	149	await Bun.sleep(100);
	150	expect(asked).toEqual([]);
	151	expect(reworks(pull)).toBe(0);
	152	expect(open(pull)).toBe(true);
	153	expect(rechecks(pull)).toHaveLength(1);
	154	});
	155
	156	test("red outside the diff twice: the second red is the builder's, one try spent, never parked", async () => {
	157	asked = [];
	158	const pull = await featurePull();
	159	pushRed(pull);
	160	await until(() => rechecks(pull).length === 1);
	161	expect(asked).toEqual([]);
	162	expect(reworks(pull)).toBe(0);
	163	finish(rechecks(pull)[0]!, "failed");
	164	await until(() => reworks(pull) === 1);
	165	expect(asked).toEqual([String(pull.number)]);
	166	expect(open(pull)).toBe(true);
	167	expect(rechecks(pull)).toHaveLength(1);
	168	});
	169
	170	test("red inside the diff: no re-check, the builder's at once", async () => {
	171	asked = [];
	172	const pull = await featurePull();
	173	pushRed(pull, "test/feature.test.ts");
	174	await until(() => reworks(pull) === 1);
	175	expect(asked).toEqual([String(pull.number)]);
	176	expect(open(pull)).toBe(true);
	177	expect(rechecks(pull)).toEqual([]);
	178	});
	179	});