22e3ce0 · b4ad64e · 2a1ebab
@@ -496,7 +496,11 @@
outbox with a sweep behind the wake-up:
  lands as a new pull that names the one it continues.
- **A failed check.** A deploy that fails on the agent's pull, or the
  repository's own check failing on it, starts a follow-up with the log's
  tail. Bounded by rule 7.
  tail. Bounded by rule 7. A failure the crew reads as the place it ran —
  a timeout, a missing tool, a flaky runner — runs the check again once
  for that commit instead, and spends none of the pull's tries; a second
  red on the same commit is taken at face value, and what the pull is
  parked with says which of the two failed.
Forks follow their parents. A fork's default branch is a copy of the
parent's until someone commits on it, and while that is so the parent
@@ -234,11 +234,15 @@
Where it decides today:
  lands by rule. The decider reads the pull against the direction and may
  say _show_: the pull stays open and appears in the studio as a look,
  saying the decider asked. One answer per head; a new push is asked again.
- **Try again or stop** (`crew/dispatch.ts`, `retryOrStop`): a failed check
  gets another automatic run by rule, up to the limit. The decider reads
  the failure, the log's tail and the count, and may stop sooner, before
  the run's money is spent; the pull then waits for a person as a stopped
  ask.
- **Whose fault, and try again or stop** (`crew/dispatch.ts`,
  `judgeFailedCheck`): a failed check gets another automatic run by rule,
  up to the limit. The decider reads the failure, the log's tail and the
  count and answers both at once — whether another try is worth its money,
  and whether the change, the environment or the base branch is what
  failed. The cause is settled before anything is parked: an environmental
  failure runs the check again (below) whatever the verdict, so a flaky
  runner never throws a change away. Otherwise _stop_ waits for a person as
  a stopped ask, and the parked reason says which of the two it was.
**The ledger.** Each of these is written to the `decision` table
(`core/decisions.ts`): what the decider saw, what it answered and with what
@@ -248,10 +252,13 @@
or red, a picked issue shipped. `GET /api/v1/repos/:owner/:name/decisions`
lists it with the week summed per kind. It is the value function's raw
material: whether the decider beats the rule is a query, not an opinion.
- **A red check** (`crew/dispatch.ts`): from the tail of the check's log,
  whether the change, the environment or the base branch failed. When it
  is the environment's fault, at 70% or better, the check runs again once
  instead of the agent reworking a change that is fine.
- **A red check** (`crew/dispatch.ts`): the `cause` half of the decision
  above — from the tail of the check's log, whether the change, the
  environment or the base branch failed. At 70% or better the
  environment's fault runs the check again, once per head commit, instead
  of the agent reworking a change that is fine or the pull being parked
  for someone else's bad luck. A second red on the same commit is real:
  the pull parks, saying the environment.
- **Cards** (`web/studio/leans.ts`): which option on an "asks you" card is
  likely right, so the card leads with it — _likely_, _probably_ or _maybe_
  beside the button, the probability on hover. The person still decides.
@@ -50,7 +50,7 @@
import {
  spentToday,
} from "./autonomy.ts";
import { checksFor } from "../checks/service.ts";
import { certain, decideOr } from "./decide.ts";
import { certain } from "./decide.ts";
import { readRunLog } from "../runs/log.ts";
import { whenText } from "./limits.ts";
import {
@@ -391,9 +391,6 @@
function route(app: App, event: ForgeEvent): void {
        });
        return;
      }
      // Whose fault: the change, or the environment the check ran in? A
      // flaky runner or a broken base is not the agent's to rework; the
      // check runs again instead, once, and the agent's tries are not spent.
      const runId = event.payload["runId"];
      // Without a decider the rule answers at once, and the rework queues
      // in the same tick the event landed — nothing waits on a promise.
@@ -401,44 +398,20 @@
function route(app: App, event: ForgeEvent): void {
        queueAgentRun(app, repo, issue, agent, "check_failed");
        return;
      }
      void retryOrStop(app, repo, issue, runId, since).then((verdict) => {
        if (verdict === "stop") {
          // The decider's stop is the crew's stop: parked, never waiting,
          // in a repository on its own.
          const pull = getPull(app.db, repo.id, issue.number);
          if (pull)
            parkPull(
              app,
              repo,
              pull,
              "the check failed and the decider saw no point in another try",
            );
          return;
        }
        return whoseFault(app, runId).then((fault) => {
          const checks = checksFor(app);
          const kind = event.payload["kind"];
          const commit = event.payload["commitOid"];
          const env = event.payload["env"];
          if (
            fault === "environment" &&
            kind === "check" &&
            checks !== undefined &&
            typeof commit === "string" &&
            typeof env === "string" &&
            !retriedChecks.has(`${issue.id}:${commit}`)
          ) {
            retriedChecks.add(`${issue.id}:${commit}`);
            log.info(
              "check looked like the environment's fault; running it again",
              { issue: issue.number },
            );
            void checks.request(repo, env, commit, "retry", issue.id);
            return;
          }
          queueAgentRun(app, repo, issue, agent, "check_failed");
        });
      });
      const commitOid = event.payload["commitOid"];
      const env = event.payload["env"];
      void answerFailedCheck(app, repo, issue, agent, {
        runId,
        kind: String(event.payload["kind"] ?? ""),
        commitOid: typeof commitOid === "string" ? commitOid : undefined,
        env: typeof env === "string" ? env : undefined,
        since,
      }).catch((error) =>
        log.error("a failed check went unanswered", {
          issue: issue.number,
          error: String(error),
        }),
      );
      return;
    }
    case "pull.closed":
@@ -480,32 +453,178 @@
function route(app: App, event: ForgeEvent): void {
  }
}
/** The agent an issue or pull belongs to: the assignee, or the pull's author. */
/** Checks already run again once for a head; a second red is real. */
const retriedChecks = new Set<string>();
/** What a failed check led to, once the crew had read it. */
export type CheckAnswer = "reran" | "reworked" | "parked" | "waiting";
/**
 * Whose fault a red check is, from the tail of its log: the change
 * (the agent reworks it), the environment (the check runs again), or the
 * base branch (a person's). The rule says the change; a decider may know
 * better from the log — a network timeout, a missing tool, a flake.
 * A failed check, answered. The cause is established *before* the pull is
 * parked, because the two questions are one decision (`judgeFailedCheck`):
 *
 * - the environment's fault — a timeout, a flaky runner, a full disk — and
 *   the check runs again, once for that commit, without spending the
 *   agent's tries. A stop on that would throw away a change nobody has
 *   found fault with.
 * - `stop` — the pull is parked, with a reason that says which of the two
 *   it was, so the next reader (and the planner) can tell a change that
 *   failed from a change that was unlucky. The re-run is bounded by
 *   `retriedChecks`: a permanently broken runner parks on the second red.
 * - `again` — the agent reworks it, as ever.
 */
async function whoseFault(
export async function answerFailedCheck(
  app: App,
  runId: string | undefined,
): Promise<"change" | "environment" | "base"> {
  const decider = app.config.decider;
  if (decider === undefined || runId === undefined) return "change";
  const tail = readRunLog(app.config.dataDir, runId)
    .lines.join("\n")
    .slice(-4_000);
  const options = ["change", "environment", "base"] as const;
  const decided = await decideOr(
    decider,
    { checkLogTail: tail },
  repo: Repo,
  issue: Issue,
  agent: Agent,
  failed: {
    readonly runId: string;
    readonly kind: string;
    readonly commitOid?: string | undefined;
    readonly env?: string | undefined;
    /** When a person last spoke: what the tries are counted from. */
    readonly since: string;
  },
): Promise<CheckAnswer> {
  const judged = await judgeFailedCheck(
    app,
    repo,
    issue,
    failed.runId,
    failed.since,
  );
  const head = `${issue.id}:${failed.commitOid ?? ""}`;
  const checks = checksFor(app);
  if (
    judged.cause === "environment" &&
    failed.kind === "check" &&
    checks !== undefined &&
    failed.commitOid !== undefined &&
    failed.env !== undefined &&
    !retriedChecks.has(head)
  ) {
    retriedChecks.add(head);
    log.info("check looked like the environment's fault; running it again", {
      issue: issue.number,
      retry: judged.retry,
    });
    await checks
      .request(repo, failed.env, failed.commitOid, "retry", issue.id)
      .catch((error) =>
        log.warn("the check could not be run again", {
          issue: issue.number,
          error: String(error),
        }),
      );
    return "reran";
  }
  if (judged.retry === "stop") {
    // The decider's stop is the crew's stop: parked, never waiting, in a
    // repository on its own.
    const pull = getPull(app.db, repo.id, issue.number);
    if (
      pull &&
      parkPull(app, repo, pull, parkedReason(judged, failed.kind, head))
    )
      return "parked";
    log.info("check failed; waiting for a person", { issue: issue.number });
    return "waiting";
  }
  queueAgentRun(app, repo, issue, agent, "check_failed");
  return "reworked";
}
/**
 * Why a parked pull was parked, in the two cases a reader must tell apart:
 * a change that failed, and a change that was unlucky. It names what
 * failed too — a preview that would not boot is not a red test.
 */
function parkedReason(
  judged: CheckJudgement,
  kind: string,
  head: string,
): string {
  const what = kind === "deploy" ? "the preview's deploy" : "the check";
  if (judged.cause !== "environment")
    return `${what} failed and the decider saw no point in another try`;
  return retriedChecks.has(head)
    ? `${what} failed in the place it ran, not in the change, and failed the same way when it was run again`
    : `${what} failed in the place it ran, not in the change, and it could not be run again`;
}
export function agentFor(app: App, issue: Issue): Agent | undefined {
  if (issue.assigneeId !== null) {
    const assigned = getAgent(app.db, issue.assigneeId);
    if (assigned) return assigned;
  }
  return issue.kind === "pull" ? getAgent(app.db, issue.authorId) : undefined;
}
/** What the crew makes of a red check: whose fault, and whether to try again. */
export interface CheckJudgement {
  /** Whether another automatic try by the agent is worth its cost. */
  readonly retry: "again" | "stop";
  /** Whose fault the failure is, from the tail of the check's log. */
  readonly cause: "change" | "environment" | "base";
}
const RETRY_OPTIONS = ["again", "stop"];
const CAUSE_OPTIONS = ["change", "environment", "base"];
/**
 * Whose fault the failure is, and whether to try again — asked together,
 * of one log tail, in one round trip, because the answers are only useful
 * together: a `stop` on a flaky runner throws away a change that is fine,
 * and an `again` on a broken environment reworks code that is not at
 * fault. The rule says the change's fault and another try, up to the
 * limit; the decider reads the failure, the log's tail and the count and
 * may know better. `retry` is the answer the ledger keeps (the next
 * check's result is its outcome); `cause` rides along.
 */
export async function judgeFailedCheck(
  app: App,
  repo: Repo,
  issue: Issue,
  runId: string,
  since: string,
): Promise<CheckJudgement> {
  if (app.config.decider === undefined)
    return { retry: "again", cause: "change" };
  const run = getRun(app.db, runId);
  const tail = readRunLog(app.config.dataDir, runId, 8 * 1024).lines.slice(-25);
  const decided = await decideAndRecord(
    app,
    { kind: "retry", repo, issue },
    {
      direction: repo.autonomyDirection,
      pull: { number: issue.number, title: issue.title },
      tries: automaticRunsSince(app.db, issue.id, since),
      limit: CHECK_FAILED_LIMIT,
      check: {
        kind: run?.kind ?? "check",
        summary: run?.summary ?? "",
        // What the check's own output said, parsed once when it finished
        // (`checks/bun-test.ts`): one red name beside 274 green ones reads
        // very differently from a suite that is red all through, and that
        // difference is most of what `cause` turns on.
        ...(run?.checkDetail == null ? {} : { failing: run.checkDetail }),
      },
      log: tail,
    },
    {
      retry: {
        options: RETRY_OPTIONS,
        about:
          "The check failed. Is another automatic try by the agent worth its cost, or should a person look?",
        describe: {
          again:
            "Try again: the failure reads as something the agent can fix from the log.",
          stop: "Stop: the failure repeats an earlier one, or is beyond what another run would fix.",
        },
      },
      cause: {
        options,
        options: CAUSE_OPTIONS,
        about:
          "Why the repository's check failed, judging from the end of its log.",
        describe: {
@@ -517,70 +636,26 @@
async function whoseFault(
        },
      },
    },
    () => ({ cause: certain(options, "change") }),
    () => ({
      retry: certain(RETRY_OPTIONS, "again"),
      cause: certain(CAUSE_OPTIONS, "change"),
    }),
  );
  return decided.answers.cause.p >= 0.7
    ? (decided.answers.cause.choice as "change" | "environment" | "base")
    : "change";
}
export function agentFor(app: App, issue: Issue): Agent | undefined {
  if (issue.assigneeId !== null) {
    const assigned = getAgent(app.db, issue.assigneeId);
    if (assigned) return assigned;
  }
  return issue.kind === "pull" ? getAgent(app.db, issue.authorId) : undefined;
}
/**
 * Try again, or stop and wait for a person. The rule tries again up to the
 * limit; the decider reads the failure and the count and may stop sooner,
 * before another run's money is spent. The next check's result is the
 * outcome, written back on the ledger row.
 */
export async function retryOrStop(
  app: App,
  repo: Repo,
  issue: Issue,
  runId: string,
  since: string,
): Promise<"again" | "stop"> {
  if (app.config.decider === undefined) return "again";
  const run = getRun(app.db, runId);
  const tail = readRunLog(app.config.dataDir, runId, 8 * 1024).lines.slice(-25);
  const options = ["again", "stop"];
  const decided = await decideAndRecord(
    app,
    { kind: "retry", repo, issue },
    {
      direction: repo.autonomyDirection,
      pull: { number: issue.number, title: issue.title },
      tries: automaticRunsSince(app.db, issue.id, since),
      limit: CHECK_FAILED_LIMIT,
      check: { kind: run?.kind ?? "check", summary: run?.summary ?? "" },
      log: tail,
    },
    {
      retry: {
        options,
        about:
          "The check failed. Is another automatic try by the agent worth its cost, or should a person look?",
        describe: {
          again:
            "Try again: the failure reads as something the agent can fix from the log.",
          stop: "Stop: the failure is environmental, repeats an earlier one, or is beyond what another run would fix.",
        },
      },
    },
    () => ({ retry: certain(options, "again") }),
  );
  if (decided.answers.retry.choice === "stop")
  const retry = decided.answers.retry.choice === "stop" ? "stop" : "again";
  // The change is what a half-sure answer means: an environment called
  // that way on a hunch is how a broken change escapes a rework.
  const cause =
    decided.answers.cause.p >= 0.7
      ? (decided.answers.cause.choice as CheckJudgement["cause"])
      : "change";
  if (retry === "stop")
    log.info("check failed; the decider stopped here for a person", {
      issue: issue.number,
      by: decided.by,
      p: decided.answers.retry.p.toFixed(2),
      cause,
    });
  return decided.answers.retry.choice === "stop" ? "stop" : "again";
  return { retry, cause };
}
/** When a person last said anything on the conversation, or its start. */
@@ -31,7 +31,8 @@
import { createUser, getUserByName, type User } from "../src/core/users.ts";
import { setCrewRole } from "../src/core/agents.ts";
import type { Answer, Decider, Questions } from "../src/crew/decide.ts";
import { driveRepo, landIfGreen } from "../src/crew/autonomy.ts";
import { retryOrStop } from "../src/crew/dispatch.ts";
import { answerFailedCheck, judgeFailedCheck } from "../src/crew/dispatch.ts";
import { startChecks } from "../src/checks/service.ts";
import { serve, type Servers } from "../src/http/server.ts";
/** A decider that answers from a script: question name → choice. */
@@ -238,7 +239,8 @@
describe("the decision ledger", () => {
    });
    script["retry"] = "stop";
    expect(
      await retryOrStop(app, fresh(), pull, failed.id, pull.createdAt),
      (await judgeFailedCheck(app, fresh(), pull, failed.id, pull.createdAt))
        .retry,
    ).toBe("stop");
    const seen = decider.seen.at(-1) as {
      tries: number;
@@ -261,6 +263,157 @@
describe("the decision ledger", () => {
    expect(resolveOpen(app.db, pull.id, "retry", "green")).toBe(false);
  });
  test("stop is not the last word on a red check: the environment's fault runs it again, once per commit, before anything is parked", async () => {
    const { getAgentByName } = await import("../src/core/agents.ts");
    const { listComments } = await import("../src/core/issues.ts");
    const scoutAgent = getAgentByName(app.db, "scout")!;
    const checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
    try {
      const branch = await commitToBranch(app, alice, repo, {
        branch: "feature/flaky",
        base: "main",
        message: "Flaky",
        files: [
          {
            path: "forge.json",
            content: `{"check": {"run": "test -f OK.md", "install": ""}}\n`,
          },
          { path: "OK.md", content: "ok\n" },
        ],
      });
      if (!branch.ok) throw new Error(JSON.stringify(branch.error));
      const opened = await openPull(app, scout, repo, {
        title: "Flaky",
        baseBranch: "main",
        headBranch: "feature/flaky",
      });
      if (!opened.ok) throw new Error(String(opened.error));
      const pull = opened.value;
      const red = (): string => {
        const run = createRun(app.db, {
          repoId: repo.id,
          kind: "check",
          issueId: pull.id,
          env: `pull-${pull.number}`,
          commitOid: pull.headOid,
          reason: "pull",
        });
        claimRun(app.db, run.id);
        finishRun(app.db, run.id, {
          state: "failed",
          summary: "(fail) a pull closed under a running agent [60000.50ms]",
        });
        return run.id;
      };
      const failure = {
        kind: "check",
        commitOid: pull.headOid,
        env: `pull-${pull.number}`,
        since: pull.createdAt,
      };
      const retries = (): number =>
        listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
          (r) => r.issueId === pull.id && r.reason === "retry",
        ).length;
      // A timeout in a file the change never touched, and the decider says
      // stop. Today that parked the pull on the spot; the cause is asked
      // first now, and it is the environment's — the check runs again.
      script["retry"] = "stop";
      script["cause"] = "environment";
      const asked = decider.seen.length;
      expect(
        await answerFailedCheck(app, fresh(), pull, scoutAgent, {
          runId: red(),
          ...failure,
        }),
      ).toBe("reran");
      // One decision, not two: whose fault and whether to try again come
      // back from the same round trip.
      expect(decider.seen.length).toBe(asked + 1);
      expect(getPull(app.db, repo.id, pull.number)?.state).toBe("open");
      expect(retries()).toBe(1);
      // The agent's tries are not spent: no rework was queued.
      expect(
        listRuns(app.db, repo.id, { kind: "agent" }).some(
          (r) => r.issueId === pull.id && r.reason === "check_failed",
        ),
      ).toBe(false);
      // Once per head commit: red again on the same commit is real, and
      // the pull parks with a reason that says the environment.
      expect(
        await answerFailedCheck(app, fresh(), pull, scoutAgent, {
          runId: red(),
          ...failure,
        }),
      ).toBe("parked");
      expect(retries()).toBe(1);
      expect(getPull(app.db, repo.id, pull.number)?.state).toBe("closed");
      const why = listComments(app.db, pull.id).map((c) => c.body);
      expect(
        why.some((b) => b.includes("in the place it ran, not in the change")),
      ).toBe(true);
    } finally {
      await checks.stop();
      delete script["cause"];
    }
  }, 60_000);
  test("the change's fault parks as it always has, and says so", async () => {
    const { getAgentByName } = await import("../src/core/agents.ts");
    const { listComments } = await import("../src/core/issues.ts");
    const scoutAgent = getAgentByName(app.db, "scout")!;
    const branch = await commitToBranch(app, alice, repo, {
      branch: "feature/broken",
      base: "main",
      message: "Broken",
      files: [{ path: "broken.ts", content: "export const broken = 1;\n" }],
    });
    if (!branch.ok) throw new Error(JSON.stringify(branch.error));
    const opened = await openPull(app, scout, repo, {
      title: "Broken",
      baseBranch: "main",
      headBranch: "feature/broken",
    });
    if (!opened.ok) throw new Error(String(opened.error));
    const pull = opened.value;
    const run = createRun(app.db, {
      repoId: repo.id,
      kind: "check",
      issueId: pull.id,
      env: `pull-${pull.number}`,
      commitOid: pull.headOid,
      reason: "pull",
    });
    claimRun(app.db, run.id);
    finishRun(app.db, run.id, {
      state: "failed",
      summary: "expect(received).toBe(expected): 1 of 274",
    });
    script["retry"] = "stop";
    script["cause"] = "change";
    try {
      expect(
        await answerFailedCheck(app, fresh(), pull, scoutAgent, {
          runId: run.id,
          kind: "check",
          commitOid: pull.headOid,
          env: `pull-${pull.number}`,
          since: pull.createdAt,
        }),
      ).toBe("parked");
      expect(getPull(app.db, repo.id, pull.number)?.state).toBe("closed");
      expect(
        listComments(app.db, pull.id).some((c) =>
          c.body.includes("saw no point in another try"),
        ),
      ).toBe(true);
    } finally {
      delete script["cause"];
    }
  });
  test("conflicts come first: a green pull that no longer merges gets a conflict run before anyone decides", async () => {
    const branch = await commitToBranch(app, alice, repo, {
      branch: "feature/readme",
		@@ -496,7 +496,11 @@ outbox with a sweep behind the wake-up:
496	496	lands as a new pull that names the one it continues.
497	497	- A failed check. A deploy that fails on the agent's pull, or the
498	498	repository's own check failing on it, starts a follow-up with the log's
499		tail. Bounded by rule 7.
	499	tail. Bounded by rule 7. A failure the crew reads as the place it ran —
	500	a timeout, a missing tool, a flaky runner — runs the check again once
	501	for that commit instead, and spends none of the pull's tries; a second
	502	red on the same commit is taken at face value, and what the pull is
	503	parked with says which of the two failed.
500	504
501	505	Forks follow their parents. A fork's default branch is a copy of the
502	506	parent's until someone commits on it, and while that is so the parent
		@@ -234,11 +234,15 @@ Where it decides today:
234	234	lands by rule. The decider reads the pull against the direction and may
235	235	say _show_: the pull stays open and appears in the studio as a look,
236	236	saying the decider asked. One answer per head; a new push is asked again.
237		- Try again or stop (`crew/dispatch.ts`, `retryOrStop`): a failed check
238		gets another automatic run by rule, up to the limit. The decider reads
239		the failure, the log's tail and the count, and may stop sooner, before
240		the run's money is spent; the pull then waits for a person as a stopped
241		ask.
	237	- Whose fault, and try again or stop (`crew/dispatch.ts`,
	238	`judgeFailedCheck`): a failed check gets another automatic run by rule,
	239	up to the limit. The decider reads the failure, the log's tail and the
	240	count and answers both at once — whether another try is worth its money,
	241	and whether the change, the environment or the base branch is what
	242	failed. The cause is settled before anything is parked: an environmental
	243	failure runs the check again (below) whatever the verdict, so a flaky
	244	runner never throws a change away. Otherwise _stop_ waits for a person as
	245	a stopped ask, and the parked reason says which of the two it was.
242	246
243	247	The ledger. Each of these is written to the `decision` table
244	248	(`core/decisions.ts`): what the decider saw, what it answered and with what
		@@ -248,10 +252,13 @@ or red, a picked issue shipped. `GET /api/v1/repos/:owner/:name/decisions`
248	252	lists it with the week summed per kind. It is the value function's raw
249	253	material: whether the decider beats the rule is a query, not an opinion.
250	254
251		- A red check (`crew/dispatch.ts`): from the tail of the check's log,
252		whether the change, the environment or the base branch failed. When it
253		is the environment's fault, at 70% or better, the check runs again once
254		instead of the agent reworking a change that is fine.
	255	- A red check (`crew/dispatch.ts`): the `cause` half of the decision
	256	above — from the tail of the check's log, whether the change, the
	257	environment or the base branch failed. At 70% or better the
	258	environment's fault runs the check again, once per head commit, instead
	259	of the agent reworking a change that is fine or the pull being parked
	260	for someone else's bad luck. A second red on the same commit is real:
	261	the pull parks, saying the environment.
255	262	- Cards (`web/studio/leans.ts`): which option on an "asks you" card is
256	263	likely right, so the card leads with it — _likely_, _probably_ or _maybe_
257	264	beside the button, the probability on hover. The person still decides.
		@@ -50,7 +50,7 @@ import {
50	50	spentToday,
51	51	} from "./autonomy.ts";
52	52	import { checksFor } from "../checks/service.ts";
53		import { certain, decideOr } from "./decide.ts";
	53	import { certain } from "./decide.ts";
54	54	import { readRunLog } from "../runs/log.ts";
55	55	import { whenText } from "./limits.ts";
56	56	import {
		@@ -391,9 +391,6 @@ function route(app: App, event: ForgeEvent): void {
391	391	});
392	392	return;
393	393	}
394		// Whose fault: the change, or the environment the check ran in? A
395		// flaky runner or a broken base is not the agent's to rework; the
396		// check runs again instead, once, and the agent's tries are not spent.
397	394	const runId = event.payload["runId"];
398	395	// Without a decider the rule answers at once, and the rework queues
399	396	// in the same tick the event landed — nothing waits on a promise.
		@@ -401,44 +398,20 @@ function route(app: App, event: ForgeEvent): void {
401	398	queueAgentRun(app, repo, issue, agent, "check_failed");
402	399	return;
403	400	}
404		void retryOrStop(app, repo, issue, runId, since).then((verdict) => {
405		if (verdict === "stop") {
406		// The decider's stop is the crew's stop: parked, never waiting,
407		// in a repository on its own.
408		const pull = getPull(app.db, repo.id, issue.number);
409		if (pull)
410		parkPull(
411		app,
412		repo,
413		pull,
414		"the check failed and the decider saw no point in another try",
415		);
416		return;
417		}
418		return whoseFault(app, runId).then((fault) => {
419		const checks = checksFor(app);
420		const kind = event.payload["kind"];
421		const commit = event.payload["commitOid"];
422		const env = event.payload["env"];
423		if (
424		fault === "environment" &&
425		kind === "check" &&
426		checks !== undefined &&
427		typeof commit === "string" &&
428		typeof env === "string" &&
429		!retriedChecks.has(`${issue.id}:${commit}`)
430		) {
431		retriedChecks.add(`${issue.id}:${commit}`);
432		log.info(
433		"check looked like the environment's fault; running it again",
434		{ issue: issue.number },
435		);
436		void checks.request(repo, env, commit, "retry", issue.id);
437		return;
438		}
439		queueAgentRun(app, repo, issue, agent, "check_failed");
440		});
441		});
	401	const commitOid = event.payload["commitOid"];
	402	const env = event.payload["env"];
	403	void answerFailedCheck(app, repo, issue, agent, {
	404	runId,
	405	kind: String(event.payload["kind"] ?? ""),
	406	commitOid: typeof commitOid === "string" ? commitOid : undefined,
	407	env: typeof env === "string" ? env : undefined,
	408	since,
	409	}).catch((error) =>
	410	log.error("a failed check went unanswered", {
	411	issue: issue.number,
	412	error: String(error),
	413	}),
	414	);
442	415	return;
443	416	}
444	417	case "pull.closed":
		@@ -480,32 +453,178 @@ function route(app: App, event: ForgeEvent): void {
480	453	}
481	454	}
482	455
483		/** The agent an issue or pull belongs to: the assignee, or the pull's author. */
484	456	/** Checks already run again once for a head; a second red is real. */
485	457	const retriedChecks = new Set<string>();
486	458
	459	/** What a failed check led to, once the crew had read it. */
	460	export type CheckAnswer = "reran" \| "reworked" \| "parked" \| "waiting";
	461
487	462	/**
488		* Whose fault a red check is, from the tail of its log: the change
489		* (the agent reworks it), the environment (the check runs again), or the
490		* base branch (a person's). The rule says the change; a decider may know
491		* better from the log — a network timeout, a missing tool, a flake.
	463	* A failed check, answered. The cause is established before the pull is
	464	* parked, because the two questions are one decision (`judgeFailedCheck`):
	465	*
	466	* - the environment's fault — a timeout, a flaky runner, a full disk — and
	467	* the check runs again, once for that commit, without spending the
	468	* agent's tries. A stop on that would throw away a change nobody has
	469	* found fault with.
	470	* - `stop` — the pull is parked, with a reason that says which of the two
	471	* it was, so the next reader (and the planner) can tell a change that
	472	* failed from a change that was unlucky. The re-run is bounded by
	473	* `retriedChecks`: a permanently broken runner parks on the second red.
	474	* - `again` — the agent reworks it, as ever.
492	475	*/
493		async function whoseFault(
	476	export async function answerFailedCheck(
494	477	app: App,
495		runId: string \| undefined,
496		): Promise<"change" \| "environment" \| "base"> {
497		const decider = app.config.decider;
498		if (decider === undefined \|\| runId === undefined) return "change";
499		const tail = readRunLog(app.config.dataDir, runId)
500		.lines.join("\n")
501		.slice(-4_000);
502		const options = ["change", "environment", "base"] as const;
503		const decided = await decideOr(
504		decider,
505		{ checkLogTail: tail },
	478	repo: Repo,
	479	issue: Issue,
	480	agent: Agent,
	481	failed: {
	482	readonly runId: string;
	483	readonly kind: string;
	484	readonly commitOid?: string \| undefined;
	485	readonly env?: string \| undefined;
	486	/** When a person last spoke: what the tries are counted from. */
	487	readonly since: string;
	488	},
	489	): Promise<CheckAnswer> {
	490	const judged = await judgeFailedCheck(
	491	app,
	492	repo,
	493	issue,
	494	failed.runId,
	495	failed.since,
	496	);
	497	const head = `${issue.id}:${failed.commitOid ?? ""}`;
	498	const checks = checksFor(app);
	499	if (
	500	judged.cause === "environment" &&
	501	failed.kind === "check" &&
	502	checks !== undefined &&
	503	failed.commitOid !== undefined &&
	504	failed.env !== undefined &&
	505	!retriedChecks.has(head)
	506	) {
	507	retriedChecks.add(head);
	508	log.info("check looked like the environment's fault; running it again", {
	509	issue: issue.number,
	510	retry: judged.retry,
	511	});
	512	await checks
	513	.request(repo, failed.env, failed.commitOid, "retry", issue.id)
	514	.catch((error) =>
	515	log.warn("the check could not be run again", {
	516	issue: issue.number,
	517	error: String(error),
	518	}),
	519	);
	520	return "reran";
	521	}
	522	if (judged.retry === "stop") {
	523	// The decider's stop is the crew's stop: parked, never waiting, in a
	524	// repository on its own.
	525	const pull = getPull(app.db, repo.id, issue.number);
	526	if (
	527	pull &&
	528	parkPull(app, repo, pull, parkedReason(judged, failed.kind, head))
	529	)
	530	return "parked";
	531	log.info("check failed; waiting for a person", { issue: issue.number });
	532	return "waiting";
	533	}
	534	queueAgentRun(app, repo, issue, agent, "check_failed");
	535	return "reworked";
	536	}
	537
	538	/**
	539	* Why a parked pull was parked, in the two cases a reader must tell apart:
	540	* a change that failed, and a change that was unlucky. It names what
	541	* failed too — a preview that would not boot is not a red test.
	542	*/
	543	function parkedReason(
	544	judged: CheckJudgement,
	545	kind: string,
	546	head: string,
	547	): string {
	548	const what = kind === "deploy" ? "the preview's deploy" : "the check";
	549	if (judged.cause !== "environment")
	550	return `${what} failed and the decider saw no point in another try`;
	551	return retriedChecks.has(head)
	552	? `${what} failed in the place it ran, not in the change, and failed the same way when it was run again`
	553	: `${what} failed in the place it ran, not in the change, and it could not be run again`;
	554	}
	555
	556	export function agentFor(app: App, issue: Issue): Agent \| undefined {
	557	if (issue.assigneeId !== null) {
	558	const assigned = getAgent(app.db, issue.assigneeId);
	559	if (assigned) return assigned;
	560	}
	561	return issue.kind === "pull" ? getAgent(app.db, issue.authorId) : undefined;
	562	}
	563
	564	/** What the crew makes of a red check: whose fault, and whether to try again. */
	565	export interface CheckJudgement {
	566	/** Whether another automatic try by the agent is worth its cost. */
	567	readonly retry: "again" \| "stop";
	568	/** Whose fault the failure is, from the tail of the check's log. */
	569	readonly cause: "change" \| "environment" \| "base";
	570	}
	571
	572	const RETRY_OPTIONS = ["again", "stop"];
	573	const CAUSE_OPTIONS = ["change", "environment", "base"];
	574
	575	/**
	576	* Whose fault the failure is, and whether to try again — asked together,
	577	* of one log tail, in one round trip, because the answers are only useful
	578	* together: a `stop` on a flaky runner throws away a change that is fine,
	579	* and an `again` on a broken environment reworks code that is not at
	580	* fault. The rule says the change's fault and another try, up to the
	581	* limit; the decider reads the failure, the log's tail and the count and
	582	* may know better. `retry` is the answer the ledger keeps (the next
	583	* check's result is its outcome); `cause` rides along.
	584	*/
	585	export async function judgeFailedCheck(
	586	app: App,
	587	repo: Repo,
	588	issue: Issue,
	589	runId: string,
	590	since: string,
	591	): Promise<CheckJudgement> {
	592	if (app.config.decider === undefined)
	593	return { retry: "again", cause: "change" };
	594	const run = getRun(app.db, runId);
	595	const tail = readRunLog(app.config.dataDir, runId, 8 * 1024).lines.slice(-25);
	596	const decided = await decideAndRecord(
	597	app,
	598	{ kind: "retry", repo, issue },
506	599	{
	600	direction: repo.autonomyDirection,
	601	pull: { number: issue.number, title: issue.title },
	602	tries: automaticRunsSince(app.db, issue.id, since),
	603	limit: CHECK_FAILED_LIMIT,
	604	check: {
	605	kind: run?.kind ?? "check",
	606	summary: run?.summary ?? "",
	607	// What the check's own output said, parsed once when it finished
	608	// (`checks/bun-test.ts`): one red name beside 274 green ones reads
	609	// very differently from a suite that is red all through, and that
	610	// difference is most of what `cause` turns on.
	611	...(run?.checkDetail == null ? {} : { failing: run.checkDetail }),
	612	},
	613	log: tail,
	614	},
	615	{
	616	retry: {
	617	options: RETRY_OPTIONS,
	618	about:
	619	"The check failed. Is another automatic try by the agent worth its cost, or should a person look?",
	620	describe: {
	621	again:
	622	"Try again: the failure reads as something the agent can fix from the log.",
	623	stop: "Stop: the failure repeats an earlier one, or is beyond what another run would fix.",
	624	},
	625	},
507	626	cause: {
508		options,
	627	options: CAUSE_OPTIONS,
509	628	about:
510	629	"Why the repository's check failed, judging from the end of its log.",
511	630	describe: {
		@@ -517,70 +636,26 @@ async function whoseFault(
517	636	},
518	637	},
519	638	},
520		() => ({ cause: certain(options, "change") }),
	639	() => ({
	640	retry: certain(RETRY_OPTIONS, "again"),
	641	cause: certain(CAUSE_OPTIONS, "change"),
	642	}),
521	643	);
522		return decided.answers.cause.p >= 0.7
523		? (decided.answers.cause.choice as "change" \| "environment" \| "base")
524		: "change";
525		}
526
527		export function agentFor(app: App, issue: Issue): Agent \| undefined {
528		if (issue.assigneeId !== null) {
529		const assigned = getAgent(app.db, issue.assigneeId);
530		if (assigned) return assigned;
531		}
532		return issue.kind === "pull" ? getAgent(app.db, issue.authorId) : undefined;
533		}
534
535		/**
536		* Try again, or stop and wait for a person. The rule tries again up to the
537		* limit; the decider reads the failure and the count and may stop sooner,
538		* before another run's money is spent. The next check's result is the
539		* outcome, written back on the ledger row.
540		*/
541		export async function retryOrStop(
542		app: App,
543		repo: Repo,
544		issue: Issue,
545		runId: string,
546		since: string,
547		): Promise<"again" \| "stop"> {
548		if (app.config.decider === undefined) return "again";
549		const run = getRun(app.db, runId);
550		const tail = readRunLog(app.config.dataDir, runId, 8 * 1024).lines.slice(-25);
551		const options = ["again", "stop"];
552		const decided = await decideAndRecord(
553		app,
554		{ kind: "retry", repo, issue },
555		{
556		direction: repo.autonomyDirection,
557		pull: { number: issue.number, title: issue.title },
558		tries: automaticRunsSince(app.db, issue.id, since),
559		limit: CHECK_FAILED_LIMIT,
560		check: { kind: run?.kind ?? "check", summary: run?.summary ?? "" },
561		log: tail,
562		},
563		{
564		retry: {
565		options,
566		about:
567		"The check failed. Is another automatic try by the agent worth its cost, or should a person look?",
568		describe: {
569		again:
570		"Try again: the failure reads as something the agent can fix from the log.",
571		stop: "Stop: the failure is environmental, repeats an earlier one, or is beyond what another run would fix.",
572		},
573		},
574		},
575		() => ({ retry: certain(options, "again") }),
576		);
577		if (decided.answers.retry.choice === "stop")
	644	const retry = decided.answers.retry.choice === "stop" ? "stop" : "again";
	645	// The change is what a half-sure answer means: an environment called
	646	// that way on a hunch is how a broken change escapes a rework.
	647	const cause =
	648	decided.answers.cause.p >= 0.7
	649	? (decided.answers.cause.choice as CheckJudgement["cause"])
	650	: "change";
	651	if (retry === "stop")
578	652	log.info("check failed; the decider stopped here for a person", {
579	653	issue: issue.number,
580	654	by: decided.by,
581	655	p: decided.answers.retry.p.toFixed(2),
	656	cause,
582	657	});
583		return decided.answers.retry.choice === "stop" ? "stop" : "again";
	658	return { retry, cause };
584	659	}
585	660
586	661	/** When a person last said anything on the conversation, or its start. */
		@@ -31,7 +31,8 @@ import { createUser, getUserByName, type User } from "../src/core/users.ts";
31	31	import { setCrewRole } from "../src/core/agents.ts";
32	32	import type { Answer, Decider, Questions } from "../src/crew/decide.ts";
33	33	import { driveRepo, landIfGreen } from "../src/crew/autonomy.ts";
34		import { retryOrStop } from "../src/crew/dispatch.ts";
	34	import { answerFailedCheck, judgeFailedCheck } from "../src/crew/dispatch.ts";
	35	import { startChecks } from "../src/checks/service.ts";
35	36	import { serve, type Servers } from "../src/http/server.ts";
36	37
37	38	/** A decider that answers from a script: question name → choice. */
		@@ -238,7 +239,8 @@ describe("the decision ledger", () => {
238	239	});
239	240	script["retry"] = "stop";
240	241	expect(
241		await retryOrStop(app, fresh(), pull, failed.id, pull.createdAt),
	242	(await judgeFailedCheck(app, fresh(), pull, failed.id, pull.createdAt))
	243	.retry,
242	244	).toBe("stop");
243	245	const seen = decider.seen.at(-1) as {
244	246	tries: number;
		@@ -261,6 +263,157 @@ describe("the decision ledger", () => {
261	263	expect(resolveOpen(app.db, pull.id, "retry", "green")).toBe(false);
262	264	});
263	265
	266	test("stop is not the last word on a red check: the environment's fault runs it again, once per commit, before anything is parked", async () => {
	267	const { getAgentByName } = await import("../src/core/agents.ts");
	268	const { listComments } = await import("../src/core/issues.ts");
	269	const scoutAgent = getAgentByName(app.db, "scout")!;
	270	const checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
	271	try {
	272	const branch = await commitToBranch(app, alice, repo, {
	273	branch: "feature/flaky",
	274	base: "main",
	275	message: "Flaky",
	276	files: [
	277	{
	278	path: "forge.json",
	279	content: `{"check": {"run": "test -f OK.md", "install": ""}}\n`,
	280	},
	281	{ path: "OK.md", content: "ok\n" },
	282	],
	283	});
	284	if (!branch.ok) throw new Error(JSON.stringify(branch.error));
	285	const opened = await openPull(app, scout, repo, {
	286	title: "Flaky",
	287	baseBranch: "main",
	288	headBranch: "feature/flaky",
	289	});
	290	if (!opened.ok) throw new Error(String(opened.error));
	291	const pull = opened.value;
	292	const red = (): string => {
	293	const run = createRun(app.db, {
	294	repoId: repo.id,
	295	kind: "check",
	296	issueId: pull.id,
	297	env: `pull-${pull.number}`,
	298	commitOid: pull.headOid,
	299	reason: "pull",
	300	});
	301	claimRun(app.db, run.id);
	302	finishRun(app.db, run.id, {
	303	state: "failed",
	304	summary: "(fail) a pull closed under a running agent [60000.50ms]",
	305	});
	306	return run.id;
	307	};
	308	const failure = {
	309	kind: "check",
	310	commitOid: pull.headOid,
	311	env: `pull-${pull.number}`,
	312	since: pull.createdAt,
	313	};
	314	const retries = (): number =>
	315	listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
	316	(r) => r.issueId === pull.id && r.reason === "retry",
	317	).length;
	318
	319	// A timeout in a file the change never touched, and the decider says
	320	// stop. Today that parked the pull on the spot; the cause is asked
	321	// first now, and it is the environment's — the check runs again.
	322	script["retry"] = "stop";
	323	script["cause"] = "environment";
	324	const asked = decider.seen.length;
	325	expect(
	326	await answerFailedCheck(app, fresh(), pull, scoutAgent, {
	327	runId: red(),
	328	...failure,
	329	}),
	330	).toBe("reran");
	331	// One decision, not two: whose fault and whether to try again come
	332	// back from the same round trip.
	333	expect(decider.seen.length).toBe(asked + 1);
	334	expect(getPull(app.db, repo.id, pull.number)?.state).toBe("open");
	335	expect(retries()).toBe(1);
	336	// The agent's tries are not spent: no rework was queued.
	337	expect(
	338	listRuns(app.db, repo.id, { kind: "agent" }).some(
	339	(r) => r.issueId === pull.id && r.reason === "check_failed",
	340	),
	341	).toBe(false);
	342
	343	// Once per head commit: red again on the same commit is real, and
	344	// the pull parks with a reason that says the environment.
	345	expect(
	346	await answerFailedCheck(app, fresh(), pull, scoutAgent, {
	347	runId: red(),
	348	...failure,
	349	}),
	350	).toBe("parked");
	351	expect(retries()).toBe(1);
	352	expect(getPull(app.db, repo.id, pull.number)?.state).toBe("closed");
	353	const why = listComments(app.db, pull.id).map((c) => c.body);
	354	expect(
	355	why.some((b) => b.includes("in the place it ran, not in the change")),
	356	).toBe(true);
	357	} finally {
	358	await checks.stop();
	359	delete script["cause"];
	360	}
	361	}, 60_000);
	362
	363	test("the change's fault parks as it always has, and says so", async () => {
	364	const { getAgentByName } = await import("../src/core/agents.ts");
	365	const { listComments } = await import("../src/core/issues.ts");
	366	const scoutAgent = getAgentByName(app.db, "scout")!;
	367	const branch = await commitToBranch(app, alice, repo, {
	368	branch: "feature/broken",
	369	base: "main",
	370	message: "Broken",
	371	files: [{ path: "broken.ts", content: "export const broken = 1;\n" }],
	372	});
	373	if (!branch.ok) throw new Error(JSON.stringify(branch.error));
	374	const opened = await openPull(app, scout, repo, {
	375	title: "Broken",
	376	baseBranch: "main",
	377	headBranch: "feature/broken",
	378	});
	379	if (!opened.ok) throw new Error(String(opened.error));
	380	const pull = opened.value;
	381	const run = createRun(app.db, {
	382	repoId: repo.id,
	383	kind: "check",
	384	issueId: pull.id,
	385	env: `pull-${pull.number}`,
	386	commitOid: pull.headOid,
	387	reason: "pull",
	388	});
	389	claimRun(app.db, run.id);
	390	finishRun(app.db, run.id, {
	391	state: "failed",
	392	summary: "expect(received).toBe(expected): 1 of 274",
	393	});
	394	script["retry"] = "stop";
	395	script["cause"] = "change";
	396	try {
	397	expect(
	398	await answerFailedCheck(app, fresh(), pull, scoutAgent, {
	399	runId: run.id,
	400	kind: "check",
	401	commitOid: pull.headOid,
	402	env: `pull-${pull.number}`,
	403	since: pull.createdAt,
	404	}),
	405	).toBe("parked");
	406	expect(getPull(app.db, repo.id, pull.number)?.state).toBe("closed");
	407	expect(
	408	listComments(app.db, pull.id).some((c) =>
	409	c.body.includes("saw no point in another try"),
	410	),
	411	).toBe(true);
	412	} finally {
	413	delete script["cause"];
	414	}
	415	});
	416
264	417	test("conflicts come first: a green pull that no longer merges gets a conflict run before anyone decides", async () => {
265	418	const branch = await commitToBranch(app, alice, repo, {
266	419	branch: "feature/readme",