6a4f2d6 · dda4d1e
@@ -117,3 +117,9 @@
toward no file's heat and is named on the feed (`too wide to count: #4840
## Flow reads, in one page
`bun scripts/flow-report.ts [--db <path>] [--public] [--json]` (database read-only, default `forge.db` under `FORGE_DATA`; `--json` for raw values) prints one line per read of `src/flow/report.ts`: `build_yield` (agent runs finished in 24 h: how many pushed, landed, closed), `review_latency` (median hours from opening to first verdict), `outsiders` (open pulls by non-writers and non-crew, oldest in hours), `checks_pile` (open pulls red or waiting for a check slot), `gate_minutes` (check-slot minutes in 1 h and 6 h), `oldest_approved` (the longest-waiting approved pull), `idle_pulls` (crew pulls untouched for 72 h), `lead_time` (opening to landing, 24 h) and `openers` (who opened the last hour's pulls). The script reads everything; `--public` cuts it to public repositories.
## Red pulls that sit (#8245)
`src/crew/stale_reds.ts` `sweepStaleReds(app, repo, now, room)` runs each autonomy tick and reads only the database, so a restart or roll loses nothing. An open crew pull whose latest check on its head failed, with no run in 30 minutes and no person holding or assigned to it, is re-checked (reason `sweep`); still red, its builder is queued with the failing test named, within the free slots and 3 a tick. Past its tries it is parked and its issue is ready again. Red for over 24 h and conflicting with main, or with its issue closed, it is closed with one line. The flow report's `stale_reds` line (`src/flow/stale_reds.ts`) counts the ones not yet handled, and the keeper names `stale_reds` as a cause from 3.
How a person sees stale reds: `bun scripts/flow-report.ts` prints the `stale_reds:` line (count and oldest hours), and the keeper names `stale_reds` as the cause when they are what holds the flow; the terminal Flow dialog draws the keeper's line (`keeperLine`), so it shows there when the keeper picks that cause. It has no line of its own on the dialog.
@@ -3,8 +3,8 @@
The forge watches its own landing rate, names the one thing binding it, and acts where the act is reversible and its own. `src/flow/keeper.ts` ticks once a minute. It reads the database and asks no model, so it still speaks when the subscription is out.
1. **Measure** (`keeper_rate.ts`): pulls landed per hour over the last 3 h against the best 3 h rate of the week before, and the share of the last 24 hours that ran (a landing or a builder run started). Low is under half the best and under one an hour.
2. **Find** (`keeper_cause.ts`): the first of `ceiling`, `usage` ("resets at HH:MM UTC"), `unassigned`, `empty_pillar`, `red_checks`, `stale_pile`, `capacity`. It asks the dispatcher's own questions (`heldNow`, `atCeiling`, `pace`).
3. **Act**: `unassigned` sweeps assigned issues with no run. `empty_pillar`, `red_checks` and `stale_pile` become one filed issue in the forge's own repository, once a week per cause. `ceiling`, `usage` and `capacity` are only said.
2. **Find** (`keeper_cause.ts`): the first of `ceiling`, `usage` ("resets at HH:MM UTC"), `unassigned`, `empty_pillar`, `stale_reds`, `red_checks`, `stale_pile`, `capacity`. It asks the dispatcher's own questions (`heldNow`, `atCeiling`, `pace`).
3. **Act**: `unassigned` sweeps assigned issues with no run. `empty_pillar`, `red_checks` and `stale_pile` become one filed issue in the forge's own repository, once a week per cause. `ceiling`, `usage`, `capacity` and `stale_reds` (the crew's red-pull sweep is already at it) are only said.
4. **Check** after 2 h: landings rose and the act stands, or it is undone, the cause is left alone for 6 h and the owner is asked one question with one button (`POST /flow/keeper/answer`).
Where it shows: one line on home and on the boards panel, `keeper` in the MCP `flow` report, the terminal's Flow dialog (the line, the question, and `l` for its one button after a confirm) and its crew scorecard ("Forge per day" rows), and "Forge per day" (hours running, landed) on the crew scorecard and in `scorecard` (`keeperDays`). Every attempt is a row in `keeper_attempt`.
@@ -14,7 +14,6 @@
import { onlyListDocs, resolveListConflicts } from "../actions/list-conflicts.ts";
import { closedToday, closeStale } from "./stale_pulls.ts";
import { authorsInRepoPull } from "./authors_pull.ts";
import type { Database } from "bun:sqlite";
import type { App } from "../app.ts";
import { ceilingReached, forgeCeiling } from "../flow/ceiling.ts";
@@ -116,13 +115,8 @@
import { ALREADY_NOTE, alreadyDoneOids } from "./already.ts";
import { removedBy, retiredNames } from "./retired.ts";
import { interruptionOf } from "./interrupted.ts";
import {
  PARK_NOTE,
  PARK_STRIKES,
  PLANNER_HINT,
  parkedCount as countParks,
  parkedTwice,
  parkReason,
  parkReasons,
} from "./parks.ts";
import { reserveOf, reserveWords, short } from "./reserve.ts";
import { createLog } from "../kernel/log.ts";
@@ -135,6 +129,8 @@
import {
import { crewOffInPreview } from "./preview.ts";
import { approvedOnHead, finishApproved, finishFields } from "./finishing.ts";
import { recheckRedApproved } from "./red_approved.ts";
import { sweepStaleReds } from "./stale_reds.ts";
import { parkPull, strikeNote } from "./park_pull.ts";
import { keepBatch } from "../flow/red_batches.ts";
import { recordQueueFlakes } from "../checks/flake.ts";
import { readVitals, vitalsMisses, vitalsWords } from "../flow/vitals.ts";
@@ -145,6 +141,8 @@
import { bootCounts } from "../kernel/boots.ts";
const log = createLog("autonomy");
export { parkPull };
export const AUTONOMY_TICK_MS = 60_000;
/**
 * How soon a planner whose last pass did not end well — a decline a person
@@ -938,39 +936,6 @@
export function parkedCount(app: App, repo: Repo, issueId: string): number {
  return countParks(app.db, repo, issueId);
}
/** What happens to an issue the crew hands over, said once on the issue. */
const HANDED_OVER =
  "The planner splits it, replaces it with a fresh issue, or defers it; a person can also say what to try. Nothing starts here until then, and the board lists it.";
/**
 * What a park says on its issue. A first strike is the reason — and, when a
 * pull closed with it, a word to the planner. The strike that takes the
 * issue out of the crew's hands also says every reason since someone who
 * directs the crew last spoke, and whose it is now: an issue parked twice
 * is handed over, never dropped.
 */
function strikeNote(
  db: Database,
  repo: Repo,
  issue: Issue,
  reason: string,
  hint = "",
): string {
  const note = `${PARK_NOTE}${reason}`;
  const reasons = [...parkReasons(db, repo, issue.id), parkReason(note)];
  if (reasons.length < PARK_STRIKES)
    return hint === "" ? note : `${note} ${hint}`;
  return [
    note,
    "",
    `Parked ${reasons.length === 2 ? "twice" : `${reasons.length} times`} since a person last spoke here, so it is the planner's now. Every reason:`,
    "",
    ...reasons.map((r, n) => `${n + 1}. ${r}`),
    "",
    HANDED_OVER,
  ].join("\n");
}
/**
 * Park an issue: a note with the reason and the assignee cleared, so the
 * planner rewrites it on its next pass and the crew tries it fresh once.
@@ -1117,62 +1082,6 @@
function answeredWithoutChange(
  );
}
/**
 * Park a pull the crew cannot finish: close it with the reason, put its
 * issues back on the map unassigned with the same reason, and move on. A
 * repository on its own never waits for a person; a parked issue is the
 * planner's to split or drop on its next pass, and one parked twice is
 * handed to the planner with every reason (`strikeNote`). A pull a person
 * holds is not parked: closing it would hand its issue back to the crew, to
 * build the same change again beside the one that was told to wait. A
 * person's pull from a branch of the repository itself is not parked
 * either: closing it is its author's, and closed it would stop following
 * the pushes that answer its review (#7572).
 */
export function parkPull(
  app: App,
  repo: Repo,
  pull: Pull,
  why: string,
): boolean {
  if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return false;
  if (getHold(app.db, pull.id) !== undefined) return false;
  if (authorsInRepoPull(pull)) return false;
  const by = getUserById(app.db, repo.autonomyBy);
  if (!by) return false;
  const issue = getIssueById(app.db, pull.id);
  if (!issue || issue.state !== "open") return false;
  addComment(app, by, repo, issue, `${PARK_NOTE}${why}`);
  const closed = setIssueState(app, by, repo, issue, "closed", {
    reason: "parked",
  });
  if (!closed.ok) return false;
  for (const number of closesFromBody(pull.body)) {
    const about = getIssue(app.db, repo.id, number);
    if (!about || about.state !== "open") continue;
    if (about.assigneeId !== null) assignIssue(app, by, repo, about, null);
    addComment(
      app,
      by,
      repo,
      about,
      strikeNote(
        app.db,
        repo,
        about,
        `${why} (pull #${pull.number} closed).`,
        PLANNER_HINT,
      ),
    );
  }
  log.info("parked", {
    repo: `${repo.ownerName}/${repo.name}`,
    pull: pull.number,
    why,
  });
  return true;
}
/**
 * Green pulls that have not landed — a check that passed before autonomy
 * was turned on, or one the event missed — land on the tick, so nothing
@@ -1330,6 +1239,13 @@
export async function autonomyTick(
        }),
      );
      const current = getRepoById(app.db, repo.id) ?? repo;
      // A red pull nobody is working is checked again, reworked, parked or closed (#8245).
      await sweepStaleReds(app, current, now, freeSlots(app, current, now)).catch((error) => {
        log.error("stale red sweep failed", {
          repo: `${repo.ownerName}/${repo.name}`,
          error: String(error),
        });
      });
      // Before the picks: a pull a reviewer answered is finished before new
      // work takes the slots it needs (#2172).
      const requested = await moveRequested(app, current, freeSlots(app, current, now), now);
@@ -0,0 +1,114 @@
/**
 * Parking a pull the crew cannot finish, and the note a park leaves. Its own
 * module so `autonomy.ts` and the stale-red sweep both reach it without
 * importing each other.
 */
import type { Database } from "bun:sqlite";
import type { App } from "../app.ts";
import { addComment, assignIssue, setIssueState } from "../actions/issues.ts";
import { getHold } from "../core/holds.ts";
import { getIssue, getIssueById, type Issue } from "../core/issues.ts";
import { closesFromBody, type Pull } from "../core/pulls.ts";
import type { Repo } from "../core/repos.ts";
import { getUserById } from "../core/users.ts";
import { createLog } from "../kernel/log.ts";
import { authorsInRepoPull } from "./authors_pull.ts";
import {
  PARK_NOTE,
  PARK_STRIKES,
  PLANNER_HINT,
  parkReason,
  parkReasons,
} from "./parks.ts";
const log = createLog("autonomy");
/** What happens to an issue the crew hands over, said once on the issue. */
const HANDED_OVER =
  "The planner splits it, replaces it with a fresh issue, or defers it; a person can also say what to try. Nothing starts here until then, and the board lists it.";
/**
 * What a park says on its issue. A first strike is the reason — and, when a
 * pull closed with it, a word to the planner. The strike that takes the
 * issue out of the crew's hands also says every reason since someone who
 * directs the crew last spoke, and whose it is now: an issue parked twice
 * is handed over, never dropped.
 */
export function strikeNote(
  db: Database,
  repo: Repo,
  issue: Issue,
  reason: string,
  hint = "",
): string {
  const note = `${PARK_NOTE}${reason}`;
  const reasons = [...parkReasons(db, repo, issue.id), parkReason(note)];
  if (reasons.length < PARK_STRIKES)
    return hint === "" ? note : `${note} ${hint}`;
  return [
    note,
    "",
    `Parked ${reasons.length === 2 ? "twice" : `${reasons.length} times`} since a person last spoke here, so it is the planner's now. Every reason:`,
    "",
    ...reasons.map((r, n) => `${n + 1}. ${r}`),
    "",
    HANDED_OVER,
  ].join("\n");
}
/**
 * Park a pull the crew cannot finish: close it with the reason, put its
 * issues back on the map unassigned with the same reason, and move on. A
 * repository on its own never waits for a person; a parked issue is the
 * planner's to split or drop on its next pass, and one parked twice is
 * handed to the planner with every reason (`strikeNote`). A pull a person
 * holds is not parked: closing it would hand its issue back to the crew, to
 * build the same change again beside the one that was told to wait. A
 * person's pull from a branch of the repository itself is not parked
 * either: closing it is its author's, and closed it would stop following
 * the pushes that answer its review (#7572).
 */
export function parkPull(
  app: App,
  repo: Repo,
  pull: Pull,
  why: string,
): boolean {
  if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return false;
  if (getHold(app.db, pull.id) !== undefined) return false;
  if (authorsInRepoPull(pull)) return false;
  const by = getUserById(app.db, repo.autonomyBy);
  if (!by) return false;
  const issue = getIssueById(app.db, pull.id);
  if (!issue || issue.state !== "open") return false;
  addComment(app, by, repo, issue, `${PARK_NOTE}${why}`);
  const closed = setIssueState(app, by, repo, issue, "closed", {
    reason: "parked",
  });
  if (!closed.ok) return false;
  for (const number of closesFromBody(pull.body)) {
    const about = getIssue(app.db, repo.id, number);
    if (!about || about.state !== "open") continue;
    if (about.assigneeId !== null) assignIssue(app, by, repo, about, null);
    addComment(
      app,
      by,
      repo,
      about,
      strikeNote(
        app.db,
        repo,
        about,
        `${why} (pull #${pull.number} closed).`,
        PLANNER_HINT,
      ),
    );
  }
  log.info("parked", {
    repo: `${repo.ownerName}/${repo.name}`,
    pull: pull.number,
    why,
  });
  return true;
}
@@ -0,0 +1,195 @@
/**
 * A red pull never sits (#8245). Reworking a red is event-driven
 * (`dispatch.ts`): a missed event, a restart or a roll leaves it red and
 * assigned to a builder with no run, holding its issue. This sweep reads the
 * database each tick, so it needs no memory of what it did:
 *
 * - an open crew pull whose latest check on its head failed, with no run of
 *   its own in the last {@link QUIET_MS} and no person holding it, is checked
 *   again (reason `sweep`); still red, its builder is queued with the failing
 *   test named, within the free slots and {@link PER_TICK} a tick;
 * - past its tries it is parked (`parkPull`): closed with the reason, its
 *   issue unassigned and ready again;
 * - red for over {@link STALE_MS} and conflicting with main, or with no
 *   issue left open to close, it is closed with one line saying why.
 *
 * Left alone: a pull a person holds or is assigned, a person's pull from a
 * branch of the repository, one approved on its head (`red_approved.ts`),
 * a red main shares (`dispatch.ts` waits those out) and a machine failure.
 */
import type { App } from "../app.ts";
import { addComment, assignIssue, setIssueState } from "../actions/issues.ts";
import { checksFor } from "../checks/service.ts";
import { redMain, sharedWithMain } from "../checks/red_main.ts";
import { getHold } from "../core/holds.ts";
import { getIssue, getIssueById } from "../core/issues.ts";
import { closesFromBody, listOpenPulls, pullDir, type Pull } from "../core/pulls.ts";
import type { Repo } from "../core/repos.ts";
import { automaticRunsSince, getRun, INFRA_SUMMARY, type Run } from "../core/runs.ts";
import { getUserById } from "../core/users.ts";
import { QUIET_MS, SWEEP_REASON } from "../flow/stale_reds.ts";
import { mergeable, resolveOid } from "../git/merge.ts";
import { createLog } from "../kernel/log.ts";
import { authorsInRepoPull } from "./authors_pull.ts";
import { agentFor, CHECK_FAILED_LIMIT, lastHumanWord, queueAgentRun } from "./dispatch.ts";
import { approvedOnHead } from "./finishing.ts";
      
        junipercrew
        12d ago
        
      
      Circular import with autonomy.ts (it imports this file). Extract parkPull to its own module.
    
import { parkPull } from "./park_pull.ts";
import { failingLine } from "./red_approved.ts";
const log = createLog("stale-reds");
/** How long a pull may stay red before a conflict or a closed issue closes it. */
export const STALE_MS = 24 * 3_600_000;
/** Pulls the sweep re-checks or hands to a builder in one tick: paced, not a burst. */
export const PER_TICK = 3;
/** What the sweep did with one pull. */
export interface StaleMove {
  readonly pull: number;
  readonly did: "re-check queued" | "builder queued" | "parked" | "closed";
}
/** The line a closed stale red carries. */
export const closedLine = (why: string): string =>
  `Closed: red for over 24 h and ${why}; the pull cannot land as it is.`;
interface Row {
  id: string;
  kind: string;
  state: string;
  reason: string;
  created_at: string;
  finished_at: string | null;
  commit_oid: string;
}
/**
 * The red check this head sits on: its latest non-sweep check failed (not a
 * machine failure), and nothing but the sweep has run on the pull since
 * `cutoff`. Undefined for a pull that is not sitting.
 */
function sittingRed(app: App, pull: Pull, now: Date): { run: Run; at: string } | undefined {
  const cutoff = new Date(now.getTime() - QUIET_MS).toISOString();
  const rows = app.db
    .query<Row, [string]>(
      `SELECT id, kind, state, reason, created_at, finished_at, commit_oid FROM run
        WHERE issue_id = ?1 AND kind IN ('agent','check','deploy') AND reason != '${SWEEP_REASON}'
        ORDER BY id DESC LIMIT 20`,
    )
    .all(pull.id);
  for (const r of rows)
    if (r.state === "queued" || r.state === "running" || (r.finished_at ?? r.created_at) > cutoff)
      return undefined;
  const last = rows.find((r) => r.kind === "check" && r.commit_oid === pull.headOid);
  if (last?.state !== "failed" || last.finished_at === null) return undefined;
  const run = getRun(app.db, last.id);
  if (run === undefined || run.summary.startsWith(INFRA_SUMMARY)) return undefined;
  return { run, at: last.finished_at };
}
/** The sweep's own re-check of this head, newest, if one was queued. */
function sweepRun(app: App, pull: Pull): Run | undefined {
      
        junipercrew
        12d ago
        outdated
      
      Redundant wrapper around readRun; import and call it directly.
    
  const row = app.db
    .query<{ id: string }, [string, string]>(
      `SELECT id FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2
          AND reason = '${SWEEP_REASON}' ORDER BY id DESC LIMIT 1`,
    )
    .get(pull.id, pull.headOid);
  return row === null ? undefined : getRun(app.db, row.id);
}
/** Close a stale red with one line; the issue it was for goes back to ready, unassigned. */
function closeRed(app: App, repo: Repo, pull: Pull, why: string): boolean {
  const by = repo.autonomyBy === null ? undefined : getUserById(app.db, repo.autonomyBy);
  const issue = getIssueById(app.db, pull.id);
  if (!by || !issue || issue.state !== "open") return false;
  addComment(app, by, repo, issue, closedLine(why));
  if (!setIssueState(app, by, repo, issue, "closed", { reason: "parked", stale: true }).ok) return false;
  for (const n of closesFromBody(pull.body)) {
    const about = getIssue(app.db, repo.id, n);
    if (about?.state === "open" && about.assigneeId !== null) assignIssue(app, by, repo, about, null);
  }
  log.info("stale red closed", { repo: `${repo.ownerName}/${repo.name}`, pull: pull.number, why });
  return true;
}
/**
 * One pass over a repository's open pulls. `room` is the free builder slots;
 * the re-checks and rework runs together stay within it and {@link PER_TICK}.
 */
export async function sweepStaleReds(
  app: App,
  repo: Repo,
  now: Date,
  room: number,
): Promise<StaleMove[]> {
  const moves: StaleMove[] = [];
  if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return moves;
  let budget = Math.min(PER_TICK, Math.max(room, 0));
  const main = redMain(app.db, repo.id);
  for (const pull of listOpenPulls(app.db, repo.id)) {
    if (!pull.authorIsAgent || pull.mergedAt !== null || pull.headOid === "") continue;
    if (getHold(app.db, pull.id) !== undefined || authorsInRepoPull(pull)) continue;
    const issue = getIssueById(app.db, pull.id);
    if (!issue || issue.state !== "open") continue;
    const agent = agentFor(app, issue);
    if (!agent) continue;
    if (approvedOnHead(app.db, pull)) continue;
    const red = sittingRed(app, pull, now);
    if (red === undefined || sharedWithMain(main, red.run)) continue;
    // Stale: red over a day and it conflicts, or its issue is gone.
    if (now.getTime() - Date.parse(red.at) >= STALE_MS) {
      const named = closesFromBody(pull.body);
      const live = named.some((n) => getIssue(app.db, repo.id, n)?.state === "open");
      
        junipercrew
        12d ago
        outdated
      
      The conflict branch is untested. Add a test with a conflicting head and a red older than 24 h.
    
      let why: string | undefined =
        named.length > 0 && !live ? "the issue it was for is no longer open" : undefined;
      if (why === undefined) {
        const dir = await pullDir(app.db, app.reposRoot, repo, pull);
        const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
        if (tip !== undefined && !(await mergeable(dir, tip, pull.headOid)).ok)
          why = "it conflicts with main";
      }
      if (why !== undefined && closeRed(app, repo, pull, why)) {
        moves.push({ pull: pull.number, did: "closed" });
        continue;
      }
    }
    const since = lastHumanWord(app, issue, agent);
    if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
      const why = `the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries: ${failingLine(red.run)}`;
      if (parkPull(app, repo, pull, why)) moves.push({ pull: pull.number, did: "parked" });
      continue;
    }
    if (budget <= 0) continue;
    const again = sweepRun(app, pull);
    if (again === undefined) {
      const checks = checksFor(app);
      if (checks === undefined) continue;
      const env = `pull-${pull.number}`;
      const queued = await checks.request(repo, env, pull.headOid, SWEEP_REASON, pull.id);
      if (queued?.reason !== SWEEP_REASON) continue;
      budget--;
      moves.push({ pull: pull.number, did: "re-check queued" });
      continue;
    }
    if (again.state !== "failed" || again.summary.startsWith(INFRA_SUMMARY)) continue;
    // Said as the builder, not as a person: a person's word restarts the tries.
    addComment(
      app,
      agent.user,
      repo,
      issue,
      `Still red on current main: ${failingLine(again)}. Fix it and push.`,
    );
    if (queueAgentRun(app, repo, issue, agent, "check_failed") !== undefined) {
      budget--;
      moves.push({ pull: pull.number, did: "builder queued" });
    }
  }
  return moves;
}
@@ -210,6 +210,7 @@
const GENERIC: Record<KeeperCause, string> = {
  usage: "Usage is the limit: no subscription has headroom.",
  unassigned: "Ready issues wait and no builder is running.",
  empty_pillar: "Nothing is ready: no pillar has an issue a builder can start.",
  stale_reds: "Open crew pulls are red and no run is working them.",
  red_checks: "Open crew pulls have a red check.",
  stale_pile: "Open crew pulls are waiting on a builder or a reviewer.",
  capacity: "Every builder slot is busy.",
@@ -17,12 +17,14 @@
import { atCeiling } from "../crew/dispatch.ts";
import { heldNow } from "../crew/held.ts";
import { pace } from "../crew/pacing.ts";
import { readPile } from "./pile.ts";
import { readStaleReds } from "./stale_reds.ts";
export const KEEPER_CAUSES = [
  "ceiling",
  "usage",
  "unassigned",
  "empty_pillar",
  "stale_reds",
  "red_checks",
  "stale_pile",
  "capacity",
@@ -51,6 +53,8 @@
export interface RepoFacts {
export interface KeeperFacts {
  readonly repos: readonly RepoFacts[];
  readonly red: number;
  /** Red pulls with no run for 30 min and no person holding them. */
  readonly staleReds?: number;
  readonly stale: number;
}
@@ -65,6 +69,7 @@
export interface Found {
/** The pile that is not moving counts as stale from this many pulls; red from this many. */
export const STALE_PULLS = 20;
export const RED_PULLS = 5;
export const STALE_REDS = 3;
const hhmm = (iso: string): string => `${iso.slice(11, 16)} UTC`;
@@ -101,6 +106,10 @@
export function pickCause(f: KeeperFacts, skip: ReadonlySet<KeeperCause> = new S
    const found = hit({ cause: "empty_pillar", move: "file", repoId: first.repoId, says: "Nothing is ready: no pillar has an issue a builder can start." });
    if (found) return found;
  }
  if (first && (f.staleReds ?? 0) >= STALE_REDS) {
    const found = hit({ cause: "stale_reds", move: "say", repoId: first.repoId, says: `${f.staleReds} open crew pulls are red and no run is working them: the sweep reworks, parks or closes them a few a tick.` });
    if (found) return found;
  }
  if (first && f.red >= RED_PULLS) {
    const found = hit({ cause: "red_checks", move: "file", repoId: first.repoId, says: `${f.red} open crew pulls have a red check.` });
    if (found) return found;
@@ -154,5 +163,5 @@
export function readKeeperFacts(app: App, now: Date): KeeperFacts {
    });
  }
  const pile = readPile(db);
  return { repos, red: pile.red, stale: pile.awaitingReview + pile.changesAsked + pile.approved };
  return { repos, red: pile.red, staleReds: readStaleReds(db, now).count, stale: pile.awaitingReview + pile.changesAsked + pile.approved };
}
@@ -1,5 +1,5 @@
/**
 * Every flow read in one page: the nine reads under src/flow/ that nothing
 * Every flow read in one page: the reads under src/flow/ that nothing
 * else calls yet, each as one plain line, in a fixed order. `flowReadings`
 * is the raw values; `flowReportLines` words them. `publicOnly` is the rule every
 * read keeps: public repositories alone.
@@ -15,6 +15,7 @@
import { readOldestApproved } from "./oldest_approved.ts";
import { readOpeners } from "./openers.ts";
import { readOutsiders } from "./outsiders.ts";
import { readReviewLatency } from "./review_latency.ts";
import { readStaleReds } from "./stale_reds.ts";
/** The raw value of each read, keyed by its name, in report order. */
export function flowReadings(db: Database, now = new Date(), publicOnly = false) {
@@ -23,6 +24,7 @@
export function flowReadings(db: Database, now = new Date(), publicOnly = false)
    review_latency: readReviewLatency(db, now, publicOnly),
    outsiders: readOutsiders(db, now, publicOnly),
    checks_pile: readChecksPile(db, publicOnly),
    stale_reds: readStaleReds(db, now, publicOnly),
    gate_minutes: readGateMinutes(db, now, publicOnly),
    oldest_approved: readOldestApproved(db, now, publicOnly),
    idle_pulls: readIdlePulls(db, now, publicOnly),
@@ -46,6 +48,7 @@
export function flowReportLines(r: FlowReadings): string[] {
    `review_latency: ${r.review_latency ? `median ${hours(r.review_latency.median)} over ${r.review_latency.count} pulls in 24 h` : "no first verdict in 24 h"}`,
    `outsiders: ${r.outsiders.open} open, oldest ${r.outsiders.oldestHours} h`,
    `checks_pile: ${r.checks_pile.red} red, ${r.checks_pile.queued} queued`,
    `stale_reds: ${r.stale_reds.count === 0 ? "none" : `${r.stale_reds.count} red with no run for 30 min, oldest ${hours(r.stale_reds.oldestHours)}`}`,
    `gate_minutes: gates ${g.gate1h} min in 1 h, ${g.gate6h} min in 6 h; pull checks ${g.pull1h} min in 1 h, ${g.pull6h} min in 6 h`,
    `oldest_approved: ${o ? `${o.repo}#${o.pull} approved ${hours(o.hours)} ago` : "none waiting"}`,
    `idle_pulls: ${r.idle_pulls.length === 0 ? "none" : r.idle_pulls.map((p) => `${p.repo}#${p.pull}${p.issue === null ? "" : ` (closes #${p.issue})`} ${hours(p.hours)}`).join(", ")}`,
@@ -0,0 +1,47 @@
/**
 * Open pulls that are red and sit (#8245): the latest check on the current
 * head failed at least `QUIET_MS` ago, no person holds the pull, and no
 * agent or check run touched it since. A sweep (`crew/stale_reds.ts`)
 * reworks, parks or closes these; the report counts the ones it has not yet.
 * A re-check the sweep itself queued (reason `sweep`) is not a touch.
 */
import type { Database } from "bun:sqlite";
/** How long a red pull must be left alone before it counts as sitting. */
export const QUIET_MS = 30 * 60_000;
/** The reason of the check the sweep queues on a red head. */
export const SWEEP_REASON = "sweep";
export interface StaleReds {
  readonly count: number;
  /** Hours the longest-sitting one has been red; null when none is. */
  readonly oldestHours: number | null;
}
/** `publicOnly` counts public repositories alone, as `readVitals` does. */
export function readStaleReds(db: Database, now = new Date(), publicOnly = false): StaleReds {
  const cutoff = new Date(now.getTime() - QUIET_MS).toISOString();
  const rows = db
    .query<{ at: string }, [string, number]>(
      `SELECT l.finished_at AS at FROM issue i
         JOIN pull p ON p.issue_id = i.id JOIN repo r ON r.id = i.repo_id
         JOIN run l ON l.id = (SELECT c.id FROM run c WHERE c.kind = 'check' AND c.issue_id = i.id
                                  AND c.commit_oid = p.head_oid AND c.reason != '${SWEEP_REASON}'
                                ORDER BY c.id DESC LIMIT 1)
        WHERE i.state = 'open' AND p.merged_at IS NULL AND (?2 = 0 OR r.visibility = 'public')
          AND l.state = 'failed' AND l.finished_at IS NOT NULL AND l.finished_at <= ?1
          AND NOT EXISTS (SELECT 1 FROM pull_hold h WHERE h.issue_id = i.id AND h.released_at IS NULL)
          AND NOT EXISTS (SELECT 1 FROM user u WHERE u.id = i.assignee_id AND u.is_agent = 0)
          AND NOT EXISTS (SELECT 1 FROM run t WHERE t.issue_id = i.id AND t.kind IN ('agent','check','deploy')
                            AND t.reason != '${SWEEP_REASON}'
                            AND (t.state IN ('queued','running') OR COALESCE(t.finished_at, t.created_at) > ?1))`,
    )
    .all(cutoff, publicOnly ? 1 : 0);
  const oldest = rows.reduce<string | null>((o, r) => (o === null || r.at < o ? r.at : o), null);
  return {
    count: rows.length,
    oldestHours: oldest === null ? null : (now.getTime() - Date.parse(oldest)) / 3_600_000,
  };
}
@@ -12,7 +12,7 @@
import { flowReadings, flowReportLines } from "../src/flow/report.ts";
const NOW = new Date("2026-10-26T12:00:00.000Z");
const ago = (min: number) => new Date(NOW.getTime() - min * 60_000).toISOString();
const ORDER = [
  "build_yield", "review_latency", "outsiders", "checks_pile", "gate_minutes",
  "build_yield", "review_latency", "outsiders", "checks_pile", "stale_reds", "gate_minutes",
  "oldest_approved", "idle_pulls", "lead_time", "openers",
];
const name = (line: string) => line.split(":")[0]!;
@@ -76,7 +76,7 @@
beforeEach(async () => {
  review(landed, "changes_requested", 2 * 60);
  const outsider = issue(priv, carol, 20); // an outsider's pull, red check, opened in the last hour
  pull(outsider, "h3");
  run(priv, "check", { issue: outsider, oid: "h3", state: "failed", start: 30, end: 10 });
  run(priv, "check", { issue: outsider, oid: "h3", state: "failed", start: 60, end: 40 });
  run(priv, "agent", { state: "ok", start: 180, end: 120, reason: "build" });
});
afterEach(() => {
@@ -102,6 +102,7 @@
describe("flowReportLines", () => {
      "review_latency: no first verdict in 24 h",
      "outsiders: 0 open, oldest 0 h",
      "checks_pile: 0 red, 0 queued",
      "stale_reds: none",
      "gate_minutes: gates 0 min in 1 h, 0 min in 6 h; pull checks 0 min in 1 h, 0 min in 6 h",
      "oldest_approved: none waiting",
      "idle_pulls: none",
@@ -116,6 +117,7 @@
describe("flowReportLines", () => {
    expect(all.review_latency).toContain("over 1 pulls");
    expect(all.outsiders).toBe("outsiders: 1 open, oldest 0 h");
    expect(all.checks_pile).toBe("checks_pile: 1 red, 0 queued");
    expect(all.stale_reds).toBe("stale_reds: 1 red with no run for 30 min, oldest 0.7 h");
    expect(all.gate_minutes).toContain("pull checks 20 min in 1 h");
    expect(all.oldest_approved).toContain("priv#1");
    expect(all.idle_pulls).toContain("priv#1");
@@ -0,0 +1,219 @@
/**
 * A red pull never sits (#8245): the sweep reads the database with `now` given,
 * so a restart in the middle changes nothing; a person's hold is left alone.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { openIssue } from "../src/actions/issues.ts";
import { openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { provideChecksForTest, type ChecksHandle } from "../src/checks/service.ts";
import { setHold } from "../src/core/holds.ts";
import { getIssueById, listComments } from "../src/core/issues.ts";
import { getPull } from "../src/core/pulls.ts";
import { getRepoById, type Repo } from "../src/core/repos.ts";
import { createUser, getUserByName, type User } from "../src/core/users.ts";
import { createRun, listRuns } from "../src/core/runs.ts";
import { sweepStaleReds, PER_TICK } from "../src/crew/stale_reds.ts";
import { readStaleReds, SWEEP_REASON } from "../src/flow/stale_reds.ts";
import { pickCause, STALE_REDS } from "../src/flow/keeper_cause.ts";
import { flowReadings, flowReportLines } from "../src/flow/report.ts";
let dataDir: string;
let app: App;
let alice: User;
let mason: User;
let repo: Repo;
let n = 0;
const requested: string[] = [];
const T0 = new Date("2026-11-02T12:00:00.000Z");
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-stale-reds-"));
  app = createApp({ dataDir, port: 0 });
  const a = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-good-passphrase",
    adminIfFirst: true,
  });
  if (!a.ok) throw new Error(a.error);
  alice = a.value;
  const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
  if (!made.ok) throw new Error(String(made.error));
  repo = made.value;
  const seeded = await commitToBranch(app, alice, repo, {
    message: "Start",
    files: [{ path: "shared.ts", content: "export const shared = 0;\n" }],
  });
  if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
  const agent = await createAgent(app, alice, { username: "mason" });
  if (!agent.ok) throw new Error(String(agent.error));
  expect(addToCrew(app, alice, repo, "mason").ok).toBe(true);
  mason = getUserByName(app.db, "mason")!;
  app.db
    .query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?2 WHERE id = ?1")
    .run(repo.id, alice.id);
  repo = getRepoById(app.db, repo.id)!;
  const handle = {
    request: async (r: Repo, env: string, oid: string, reason: string, issueId?: string) => {
      requested.push(env);
      return createRun(app.db, { repoId: r.id, kind: "check", env, commitOid: oid, reason, issueId });
    },
  } as unknown as ChecksHandle;
  provideChecksForTest(app, handle);
});
afterAll(async () => {
  app.db.close();
  await rm(dataDir, { recursive: true, force: true });
});
/** An open crew pull on its own file, closing a fresh issue assigned to mason. */
async function pullBy(who: User, file?: string) {
  const k = ++n;
  const ask = openIssue(app, alice, repo, { title: `Ask ${k}` });
  if (!ask.ok) throw new Error(String(ask.error));
  app.db.query("UPDATE issue SET assignee_id = ?2 WHERE id = ?1").run(ask.value.id, mason.id);
  const branch = await commitToBranch(app, alice, repo, {
    branch: `feature/r${k}`,
    base: "main",
    message: `Red ${k}`,
    files: [{ path: file ?? `r${k}.ts`, content: `export const r = ${k};\n` }],
  });
  if (!branch.ok) throw new Error(JSON.stringify(branch.error));
  const opened = await openPull(app, who, repo, {
    title: `Red ${k}`,
    body: `Closes #${ask.value.number}`,
    baseBranch: "main",
    headBranch: `feature/r${k}`,
  });
  if (!opened.ok) throw new Error(String(opened.error));
  return { pull: getPull(app.db, repo.id, opened.value.number)!, ask: ask.value };
}
/** A finished run on the pull's head, `minutes` after T0. */
function finished(p: { pull: { id: string; headOid: string } }, kind: "check" | "agent", state: string, minutes: number, reason = "", test = "") {
  const run = createRun(app.db, { repoId: repo.id, kind, issueId: p.pull.id, commitOid: p.pull.headOid, reason });
  app.db
    .query("UPDATE run SET state = ?2, created_at = ?3, finished_at = ?3, summary = ?4, check_detail = ?5 WHERE id = ?1")
    .run(run.id, state, at(minutes).toISOString(), state === "failed" ? `failed: ${test}` : "", test === "" ? "" : JSON.stringify({ names: [test] }));
  return run;
}
const runsOf = (id: string) => listRuns(app.db, repo.id, { issueId: id, limit: 50 });
const open = (id: string) => getIssueById(app.db, id)!.state === "open";
test("a red pull quiet for 30 minutes is re-checked, then its builder is queued with the failing test named", async () => {
  const p = await pullBy(mason);
  finished(p, "check", "failed", 0, "", "engine > adds up");
  // 20 minutes after the failure: not yet quiet.
  expect(await sweepStaleReds(app, repo, at(20), 3)).toEqual([]);
  // A restart here loses nothing: the sweep keeps no memory.
  const first = await sweepStaleReds(app, repo, at(31), 3);
  expect(first).toContainEqual({ pull: p.pull.number, did: "re-check queued" });
  expect(requested).toContain(`pull-${p.pull.number}`);
  // The re-check is in flight: nothing more.
  expect((await sweepStaleReds(app, repo, at(32), 3)).filter((m) => m.pull === p.pull.number)).toEqual([]);
  // It ends red.
  const again = runsOf(p.pull.id).find((r) => r.reason === SWEEP_REASON)!;
  app.db
    .query("UPDATE run SET state = 'failed', finished_at = ?2, summary = 'failed', check_detail = ?3 WHERE id = ?1")
    .run(again.id, at(40).toISOString(), JSON.stringify({ names: ["engine > adds up"] }));
  const second = await sweepStaleReds(app, repo, at(41), 3);
  expect(second).toContainEqual({ pull: p.pull.number, did: "builder queued" });
  expect(listComments(app.db, p.pull.id).map((c) => c.body).join("\n")).toContain("engine > adds up");
  expect(runsOf(p.pull.id).some((r) => r.kind === "agent" && r.reason === "check_failed")).toBe(true);
});
test("a pull a person holds, or is assigned to, is left alone", async () => {
  const held = await pullBy(mason);
  const mine = await pullBy(mason);
  finished(held, "check", "failed", 100, "", "engine > held");
  finished(mine, "check", "failed", 100, "", "engine > mine");
  setHold(app.db, held.pull.id, alice.id, "wait for the design");
  app.db.query("UPDATE issue SET assignee_id = ?2 WHERE id = ?1").run(mine.pull.id, alice.id);
  const moves = await sweepStaleReds(app, repo, at(200), 3);
  expect(moves.filter((m) => m.pull === held.pull.number || m.pull === mine.pull.number)).toEqual([]);
  expect(runsOf(held.pull.id).some((r) => r.reason === SWEEP_REASON || r.kind === "agent")).toBe(false);
  expect(runsOf(mine.pull.id).some((r) => r.reason === SWEEP_REASON || r.kind === "agent")).toBe(false);
  expect(open(held.pull.id)).toBe(true);
});
test("a pull past its tries is parked and its issue is ready again", async () => {
  const p = await pullBy(mason);
  finished(p, "check", "failed", 300, "", "engine > stubborn");
  finished(p, "agent", "ok", 301, "check_failed");
  finished(p, "agent", "ok", 302, "check_failed");
  const moves = await sweepStaleReds(app, repo, at(400), 3);
  expect(moves).toContainEqual({ pull: p.pull.number, did: "parked" });
  expect(open(p.pull.id)).toBe(false);
  const freed = getIssueById(app.db, p.ask.id)!;
  expect(freed.state).toBe("open");
  expect(freed.assigneeId).toBeNull();
});
test("a pull red for over a day whose issue is closed is closed with a line saying why", async () => {
  const p = await pullBy(mason);
  finished(p, "check", "failed", 1000, "", "engine > old");
  app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(p.ask.id);
  expect(await sweepStaleReds(app, repo, at(1000 + 23 * 60), 3)).not.toContainEqual({ pull: p.pull.number, did: "closed" });
  const moves = await sweepStaleReds(app, repo, at(1000 + 25 * 60), 3);
  expect(moves).toContainEqual({ pull: p.pull.number, did: "closed" });
  expect(open(p.pull.id)).toBe(false);
  expect(listComments(app.db, p.pull.id).map((c) => c.body).join("\n")).toContain("the issue it was for is no longer open");
});
test("a pull red for over a day that conflicts with main is closed saying so", async () => {
  const p = await pullBy(mason, "shared.ts");
  finished(p, "check", "failed", 2000, "", "engine > conflicting");
  const moved = await commitToBranch(app, alice, repo, {
    message: "Main moves on",
    files: [{ path: "shared.ts", content: "export const shared = 99;\n" }],
  });
  if (!moved.ok) throw new Error(JSON.stringify(moved.error));
  // Red for under a day: it conflicts, but it is reworked, not closed.
  expect(await sweepStaleReds(app, repo, at(2000 + 23 * 60), 3)).not.toContainEqual({ pull: p.pull.number, did: "closed" });
  expect(open(p.pull.id)).toBe(true);
  const moves = await sweepStaleReds(app, repo, at(2000 + 25 * 60), 3);
  expect(moves).toContainEqual({ pull: p.pull.number, did: "closed" });
  expect(open(p.pull.id)).toBe(false);
  expect(listComments(app.db, p.pull.id).map((c) => c.body).join("\n")).toContain("it conflicts with main");
  expect(getIssueById(app.db, p.ask.id)!.assigneeId).toBeNull();
});
test("a burst of reds is paced: no more than PER_TICK start in one tick", async () => {
  const made: Awaited<ReturnType<typeof pullBy>>[] = [];
  for (let i = 0; i < PER_TICK + 2; i++) {
    const p = await pullBy(mason);
    finished(p, "check", "failed", 3000, "", `engine > burst ${i}`);
    made.push(p);
  }
  const moves = await sweepStaleReds(app, repo, at(3100), 99);
  expect(moves.filter((m) => made.some((p) => p.pull.number === m.pull)).length).toBe(PER_TICK);
  // No free slot, no start.
  expect(
    (await sweepStaleReds(app, repo, at(3101), 0)).filter((m) => made.some((p) => p.pull.number === m.pull)),
  ).toEqual([]);
});
test("the flow report counts stale reds and the keeper names them as a cause", () => {
  const now = at(3100 + 600);
  const read = readStaleReds(app.db, now);
  // The burst's five reds, failed at minute 3000, 700 minutes before `now`; the held,
  // person-assigned, parked and closed ones are not counted.
  expect(read.count).toBe(PER_TICK + 2);
  expect(read.oldestHours).toBeCloseTo(700 / 60, 5);
  const line = flowReportLines(flowReadings(app.db, now)).find((l) => l.startsWith("stale_reds:"))!;
  expect(line).toBe("stale_reds: 5 red with no run for 30 min, oldest 11.7 h");
  const found = pickCause({ repos: [{ repoId: "r", slug: "a/b", startable: 1, running: 1, slots: 4, ceilingUsd: 10, atCeiling: false, noHeadroom: false, resetsAt: null }], red: 0, staleReds: STALE_REDS, stale: 0 });
  expect(found?.cause).toBe("stale_reds");
});
		@@ -117,3 +117,9 @@ toward no file's heat and is named on the feed (`too wide to count: #4840
117	117	## Flow reads, in one page
118	118
119	119	`bun scripts/flow-report.ts [--db <path>] [--public] [--json]` (database read-only, default `forge.db` under `FORGE_DATA`; `--json` for raw values) prints one line per read of `src/flow/report.ts`: `build_yield` (agent runs finished in 24 h: how many pushed, landed, closed), `review_latency` (median hours from opening to first verdict), `outsiders` (open pulls by non-writers and non-crew, oldest in hours), `checks_pile` (open pulls red or waiting for a check slot), `gate_minutes` (check-slot minutes in 1 h and 6 h), `oldest_approved` (the longest-waiting approved pull), `idle_pulls` (crew pulls untouched for 72 h), `lead_time` (opening to landing, 24 h) and `openers` (who opened the last hour's pulls). The script reads everything; `--public` cuts it to public repositories.
	120
	121	## Red pulls that sit (#8245)
	122
	123	`src/crew/stale_reds.ts` `sweepStaleReds(app, repo, now, room)` runs each autonomy tick and reads only the database, so a restart or roll loses nothing. An open crew pull whose latest check on its head failed, with no run in 30 minutes and no person holding or assigned to it, is re-checked (reason `sweep`); still red, its builder is queued with the failing test named, within the free slots and 3 a tick. Past its tries it is parked and its issue is ready again. Red for over 24 h and conflicting with main, or with its issue closed, it is closed with one line. The flow report's `stale_reds` line (`src/flow/stale_reds.ts`) counts the ones not yet handled, and the keeper names `stale_reds` as a cause from 3.
	124
	125	How a person sees stale reds: `bun scripts/flow-report.ts` prints the `stale_reds:` line (count and oldest hours), and the keeper names `stale_reds` as the cause when they are what holds the flow; the terminal Flow dialog draws the keeper's line (`keeperLine`), so it shows there when the keeper picks that cause. It has no line of its own on the dialog.
		@@ -3,8 +3,8 @@
3	3	The forge watches its own landing rate, names the one thing binding it, and acts where the act is reversible and its own. `src/flow/keeper.ts` ticks once a minute. It reads the database and asks no model, so it still speaks when the subscription is out.
4	4
5	5	1. Measure (`keeper_rate.ts`): pulls landed per hour over the last 3 h against the best 3 h rate of the week before, and the share of the last 24 hours that ran (a landing or a builder run started). Low is under half the best and under one an hour.
6		2. Find (`keeper_cause.ts`): the first of `ceiling`, `usage` ("resets at HH:MM UTC"), `unassigned`, `empty_pillar`, `red_checks`, `stale_pile`, `capacity`. It asks the dispatcher's own questions (`heldNow`, `atCeiling`, `pace`).
7		3. Act: `unassigned` sweeps assigned issues with no run. `empty_pillar`, `red_checks` and `stale_pile` become one filed issue in the forge's own repository, once a week per cause. `ceiling`, `usage` and `capacity` are only said.
	6	2. Find (`keeper_cause.ts`): the first of `ceiling`, `usage` ("resets at HH:MM UTC"), `unassigned`, `empty_pillar`, `stale_reds`, `red_checks`, `stale_pile`, `capacity`. It asks the dispatcher's own questions (`heldNow`, `atCeiling`, `pace`).
	7	3. Act: `unassigned` sweeps assigned issues with no run. `empty_pillar`, `red_checks` and `stale_pile` become one filed issue in the forge's own repository, once a week per cause. `ceiling`, `usage`, `capacity` and `stale_reds` (the crew's red-pull sweep is already at it) are only said.
8	8	4. Check after 2 h: landings rose and the act stands, or it is undone, the cause is left alone for 6 h and the owner is asked one question with one button (`POST /flow/keeper/answer`).
9	9
10	10	Where it shows: one line on home and on the boards panel, `keeper` in the MCP `flow` report, the terminal's Flow dialog (the line, the question, and `l` for its one button after a confirm) and its crew scorecard ("Forge per day" rows), and "Forge per day" (hours running, landed) on the crew scorecard and in `scorecard` (`keeperDays`). Every attempt is a row in `keeper_attempt`.
		@@ -14,7 +14,6 @@
14	14
15	15	import { onlyListDocs, resolveListConflicts } from "../actions/list-conflicts.ts";
16	16	import { closedToday, closeStale } from "./stale_pulls.ts";
17		import { authorsInRepoPull } from "./authors_pull.ts";
18	17	import type { Database } from "bun:sqlite";
19	18	import type { App } from "../app.ts";
20	19	import { ceilingReached, forgeCeiling } from "../flow/ceiling.ts";
		@@ -116,13 +115,8 @@ import { ALREADY_NOTE, alreadyDoneOids } from "./already.ts";
116	115	import { removedBy, retiredNames } from "./retired.ts";
117	116	import { interruptionOf } from "./interrupted.ts";
118	117	import {
119		PARK_NOTE,
120		PARK_STRIKES,
121		PLANNER_HINT,
122	118	parkedCount as countParks,
123	119	parkedTwice,
124		parkReason,
125		parkReasons,
126	120	} from "./parks.ts";
127	121	import { reserveOf, reserveWords, short } from "./reserve.ts";
128	122	import { createLog } from "../kernel/log.ts";
		@@ -135,6 +129,8 @@ import {
135	129	import { crewOffInPreview } from "./preview.ts";
136	130	import { approvedOnHead, finishApproved, finishFields } from "./finishing.ts";
137	131	import { recheckRedApproved } from "./red_approved.ts";
	132	import { sweepStaleReds } from "./stale_reds.ts";
	133	import { parkPull, strikeNote } from "./park_pull.ts";
138	134	import { keepBatch } from "../flow/red_batches.ts";
139	135	import { recordQueueFlakes } from "../checks/flake.ts";
140	136	import { readVitals, vitalsMisses, vitalsWords } from "../flow/vitals.ts";
		@@ -145,6 +141,8 @@ import { bootCounts } from "../kernel/boots.ts";
145	141
146	142	const log = createLog("autonomy");
147	143
	144	export { parkPull };
	145
148	146	export const AUTONOMY_TICK_MS = 60_000;
149	147	/**
150	148	* How soon a planner whose last pass did not end well — a decline a person
		@@ -938,39 +936,6 @@ export function parkedCount(app: App, repo: Repo, issueId: string): number {
938	936	return countParks(app.db, repo, issueId);
939	937	}
940	938
941		/** What happens to an issue the crew hands over, said once on the issue. */
942		const HANDED_OVER =
943		"The planner splits it, replaces it with a fresh issue, or defers it; a person can also say what to try. Nothing starts here until then, and the board lists it.";
944
945		/**
946		* What a park says on its issue. A first strike is the reason — and, when a
947		* pull closed with it, a word to the planner. The strike that takes the
948		* issue out of the crew's hands also says every reason since someone who
949		* directs the crew last spoke, and whose it is now: an issue parked twice
950		* is handed over, never dropped.
951		*/
952		function strikeNote(
953		db: Database,
954		repo: Repo,
955		issue: Issue,
956		reason: string,
957		hint = "",
958		): string {
959		const note = `${PARK_NOTE}${reason}`;
960		const reasons = [...parkReasons(db, repo, issue.id), parkReason(note)];
961		if (reasons.length < PARK_STRIKES)
962		return hint === "" ? note : `${note} ${hint}`;
963		return [
964		note,
965		"",
966		`Parked ${reasons.length === 2 ? "twice" : `${reasons.length} times`} since a person last spoke here, so it is the planner's now. Every reason:`,
967		"",
968		...reasons.map((r, n) => `${n + 1}. ${r}`),
969		"",
970		HANDED_OVER,
971		].join("\n");
972		}
973
974	939	/**
975	940	* Park an issue: a note with the reason and the assignee cleared, so the
976	941	* planner rewrites it on its next pass and the crew tries it fresh once.
		@@ -1117,62 +1082,6 @@ function answeredWithoutChange(
1117	1082	);
1118	1083	}
1119	1084
1120		/**
1121		* Park a pull the crew cannot finish: close it with the reason, put its
1122		* issues back on the map unassigned with the same reason, and move on. A
1123		* repository on its own never waits for a person; a parked issue is the
1124		* planner's to split or drop on its next pass, and one parked twice is
1125		* handed to the planner with every reason (`strikeNote`). A pull a person
1126		* holds is not parked: closing it would hand its issue back to the crew, to
1127		* build the same change again beside the one that was told to wait. A
1128		* person's pull from a branch of the repository itself is not parked
1129		* either: closing it is its author's, and closed it would stop following
1130		* the pushes that answer its review (#7572).
1131		*/
1132		export function parkPull(
1133		app: App,
1134		repo: Repo,
1135		pull: Pull,
1136		why: string,
1137		): boolean {
1138		if (repo.autonomy !== "autonomous" \|\| repo.autonomyBy === null) return false;
1139		if (getHold(app.db, pull.id) !== undefined) return false;
1140		if (authorsInRepoPull(pull)) return false;
1141		const by = getUserById(app.db, repo.autonomyBy);
1142		if (!by) return false;
1143		const issue = getIssueById(app.db, pull.id);
1144		if (!issue \|\| issue.state !== "open") return false;
1145		addComment(app, by, repo, issue, `${PARK_NOTE}${why}`);
1146		const closed = setIssueState(app, by, repo, issue, "closed", {
1147		reason: "parked",
1148		});
1149		if (!closed.ok) return false;
1150		for (const number of closesFromBody(pull.body)) {
1151		const about = getIssue(app.db, repo.id, number);
1152		if (!about \|\| about.state !== "open") continue;
1153		if (about.assigneeId !== null) assignIssue(app, by, repo, about, null);
1154		addComment(
1155		app,
1156		by,
1157		repo,
1158		about,
1159		strikeNote(
1160		app.db,
1161		repo,
1162		about,
1163		`${why} (pull #${pull.number} closed).`,
1164		PLANNER_HINT,
1165		),
1166		);
1167		}
1168		log.info("parked", {
1169		repo: `${repo.ownerName}/${repo.name}`,
1170		pull: pull.number,
1171		why,
1172		});
1173		return true;
1174		}
1175
1176	1085	/**
1177	1086	* Green pulls that have not landed — a check that passed before autonomy
1178	1087	* was turned on, or one the event missed — land on the tick, so nothing
		@@ -1330,6 +1239,13 @@ export async function autonomyTick(
1330	1239	}),
1331	1240	);
1332	1241	const current = getRepoById(app.db, repo.id) ?? repo;
	1242	// A red pull nobody is working is checked again, reworked, parked or closed (#8245).
	1243	await sweepStaleReds(app, current, now, freeSlots(app, current, now)).catch((error) => {
	1244	log.error("stale red sweep failed", {
	1245	repo: `${repo.ownerName}/${repo.name}`,
	1246	error: String(error),
	1247	});
	1248	});
1333	1249	// Before the picks: a pull a reviewer answered is finished before new
1334	1250	// work takes the slots it needs (#2172).
1335	1251	const requested = await moveRequested(app, current, freeSlots(app, current, now), now);
		@@ -0,0 +1,114 @@
	1	/**
	2	* Parking a pull the crew cannot finish, and the note a park leaves. Its own
	3	* module so `autonomy.ts` and the stale-red sweep both reach it without
	4	* importing each other.
	5	*/
	6
	7	import type { Database } from "bun:sqlite";
	8	import type { App } from "../app.ts";
	9	import { addComment, assignIssue, setIssueState } from "../actions/issues.ts";
	10	import { getHold } from "../core/holds.ts";
	11	import { getIssue, getIssueById, type Issue } from "../core/issues.ts";
	12	import { closesFromBody, type Pull } from "../core/pulls.ts";
	13	import type { Repo } from "../core/repos.ts";
	14	import { getUserById } from "../core/users.ts";
	15	import { createLog } from "../kernel/log.ts";
	16	import { authorsInRepoPull } from "./authors_pull.ts";
	17	import {
	18	PARK_NOTE,
	19	PARK_STRIKES,
	20	PLANNER_HINT,
	21	parkReason,
	22	parkReasons,
	23	} from "./parks.ts";
	24
	25	const log = createLog("autonomy");
	26
	27	/** What happens to an issue the crew hands over, said once on the issue. */
	28	const HANDED_OVER =
	29	"The planner splits it, replaces it with a fresh issue, or defers it; a person can also say what to try. Nothing starts here until then, and the board lists it.";
	30
	31	/**
	32	* What a park says on its issue. A first strike is the reason — and, when a
	33	* pull closed with it, a word to the planner. The strike that takes the
	34	* issue out of the crew's hands also says every reason since someone who
	35	* directs the crew last spoke, and whose it is now: an issue parked twice
	36	* is handed over, never dropped.
	37	*/
	38	export function strikeNote(
	39	db: Database,
	40	repo: Repo,
	41	issue: Issue,
	42	reason: string,
	43	hint = "",
	44	): string {
	45	const note = `${PARK_NOTE}${reason}`;
	46	const reasons = [...parkReasons(db, repo, issue.id), parkReason(note)];
	47	if (reasons.length < PARK_STRIKES)
	48	return hint === "" ? note : `${note} ${hint}`;
	49	return [
	50	note,
	51	"",
	52	`Parked ${reasons.length === 2 ? "twice" : `${reasons.length} times`} since a person last spoke here, so it is the planner's now. Every reason:`,
	53	"",
	54	...reasons.map((r, n) => `${n + 1}. ${r}`),
	55	"",
	56	HANDED_OVER,
	57	].join("\n");
	58	}
	59
	60	/**
	61	* Park a pull the crew cannot finish: close it with the reason, put its
	62	* issues back on the map unassigned with the same reason, and move on. A
	63	* repository on its own never waits for a person; a parked issue is the
	64	* planner's to split or drop on its next pass, and one parked twice is
	65	* handed to the planner with every reason (`strikeNote`). A pull a person
	66	* holds is not parked: closing it would hand its issue back to the crew, to
	67	* build the same change again beside the one that was told to wait. A
	68	* person's pull from a branch of the repository itself is not parked
	69	* either: closing it is its author's, and closed it would stop following
	70	* the pushes that answer its review (#7572).
	71	*/
	72	export function parkPull(
	73	app: App,
	74	repo: Repo,
	75	pull: Pull,
	76	why: string,
	77	): boolean {
	78	if (repo.autonomy !== "autonomous" \|\| repo.autonomyBy === null) return false;
	79	if (getHold(app.db, pull.id) !== undefined) return false;
	80	if (authorsInRepoPull(pull)) return false;
	81	const by = getUserById(app.db, repo.autonomyBy);
	82	if (!by) return false;
	83	const issue = getIssueById(app.db, pull.id);
	84	if (!issue \|\| issue.state !== "open") return false;
	85	addComment(app, by, repo, issue, `${PARK_NOTE}${why}`);
	86	const closed = setIssueState(app, by, repo, issue, "closed", {
	87	reason: "parked",
	88	});
	89	if (!closed.ok) return false;
	90	for (const number of closesFromBody(pull.body)) {
	91	const about = getIssue(app.db, repo.id, number);
	92	if (!about \|\| about.state !== "open") continue;
	93	if (about.assigneeId !== null) assignIssue(app, by, repo, about, null);
	94	addComment(
	95	app,
	96	by,
	97	repo,
	98	about,
	99	strikeNote(
	100	app.db,
	101	repo,
	102	about,
	103	`${why} (pull #${pull.number} closed).`,
	104	PLANNER_HINT,
	105	),
	106	);
	107	}
	108	log.info("parked", {
	109	repo: `${repo.ownerName}/${repo.name}`,
	110	pull: pull.number,
	111	why,
	112	});
	113	return true;
	114	}
		@@ -0,0 +1,195 @@
	1	/**
	2	* A red pull never sits (#8245). Reworking a red is event-driven
	3	* (`dispatch.ts`): a missed event, a restart or a roll leaves it red and
	4	* assigned to a builder with no run, holding its issue. This sweep reads the
	5	* database each tick, so it needs no memory of what it did:
	6	*
	7	* - an open crew pull whose latest check on its head failed, with no run of
	8	* its own in the last {@link QUIET_MS} and no person holding it, is checked
	9	* again (reason `sweep`); still red, its builder is queued with the failing
	10	* test named, within the free slots and {@link PER_TICK} a tick;
	11	* - past its tries it is parked (`parkPull`): closed with the reason, its
	12	* issue unassigned and ready again;
	13	* - red for over {@link STALE_MS} and conflicting with main, or with no
	14	* issue left open to close, it is closed with one line saying why.
	15	*
	16	* Left alone: a pull a person holds or is assigned, a person's pull from a
	17	* branch of the repository, one approved on its head (`red_approved.ts`),
	18	* a red main shares (`dispatch.ts` waits those out) and a machine failure.
	19	*/
	20
	21	import type { App } from "../app.ts";
	22	import { addComment, assignIssue, setIssueState } from "../actions/issues.ts";
	23	import { checksFor } from "../checks/service.ts";
	24	import { redMain, sharedWithMain } from "../checks/red_main.ts";
	25	import { getHold } from "../core/holds.ts";
	26	import { getIssue, getIssueById } from "../core/issues.ts";
	27	import { closesFromBody, listOpenPulls, pullDir, type Pull } from "../core/pulls.ts";
	28	import type { Repo } from "../core/repos.ts";
	29	import { automaticRunsSince, getRun, INFRA_SUMMARY, type Run } from "../core/runs.ts";
	30	import { getUserById } from "../core/users.ts";
	31	import { QUIET_MS, SWEEP_REASON } from "../flow/stale_reds.ts";
	32	import { mergeable, resolveOid } from "../git/merge.ts";
	33	import { createLog } from "../kernel/log.ts";
	34	import { authorsInRepoPull } from "./authors_pull.ts";
	35	import { agentFor, CHECK_FAILED_LIMIT, lastHumanWord, queueAgentRun } from "./dispatch.ts";
	36	import { approvedOnHead } from "./finishing.ts";
		junipercrew 12d ago Circular import with autonomy.ts (it imports this file). Extract parkPull to its own module.
	37	import { parkPull } from "./park_pull.ts";
	38	import { failingLine } from "./red_approved.ts";
	39
	40	const log = createLog("stale-reds");
	41
	42	/** How long a pull may stay red before a conflict or a closed issue closes it. */
	43	export const STALE_MS = 24 * 3_600_000;
	44
	45	/** Pulls the sweep re-checks or hands to a builder in one tick: paced, not a burst. */
	46	export const PER_TICK = 3;
	47
	48	/** What the sweep did with one pull. */
	49	export interface StaleMove {
	50	readonly pull: number;
	51	readonly did: "re-check queued" \| "builder queued" \| "parked" \| "closed";
	52	}
	53
	54	/** The line a closed stale red carries. */
	55	export const closedLine = (why: string): string =>
	56	`Closed: red for over 24 h and ${why}; the pull cannot land as it is.`;
	57
	58	interface Row {
	59	id: string;
	60	kind: string;
	61	state: string;
	62	reason: string;
	63	created_at: string;
	64	finished_at: string \| null;
	65	commit_oid: string;
	66	}
	67
	68	/**
	69	* The red check this head sits on: its latest non-sweep check failed (not a
	70	* machine failure), and nothing but the sweep has run on the pull since
	71	* `cutoff`. Undefined for a pull that is not sitting.
	72	*/
	73	function sittingRed(app: App, pull: Pull, now: Date): { run: Run; at: string } \| undefined {
	74	const cutoff = new Date(now.getTime() - QUIET_MS).toISOString();
	75	const rows = app.db
	76	.query<Row, [string]>(
	77	`SELECT id, kind, state, reason, created_at, finished_at, commit_oid FROM run
	78	WHERE issue_id = ?1 AND kind IN ('agent','check','deploy') AND reason != '${SWEEP_REASON}'
	79	ORDER BY id DESC LIMIT 20`,
	80	)
	81	.all(pull.id);
	82	for (const r of rows)
	83	if (r.state === "queued" \|\| r.state === "running" \|\| (r.finished_at ?? r.created_at) > cutoff)
	84	return undefined;
	85	const last = rows.find((r) => r.kind === "check" && r.commit_oid === pull.headOid);
	86	if (last?.state !== "failed" \|\| last.finished_at === null) return undefined;
	87	const run = getRun(app.db, last.id);
	88	if (run === undefined \|\| run.summary.startsWith(INFRA_SUMMARY)) return undefined;
	89	return { run, at: last.finished_at };
	90	}
	91
	92	/** The sweep's own re-check of this head, newest, if one was queued. */
	93	function sweepRun(app: App, pull: Pull): Run \| undefined {
		junipercrew 12d ago outdated Redundant wrapper around readRun; import and call it directly.
	94	const row = app.db
	95	.query<{ id: string }, [string, string]>(
	96	`SELECT id FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2
	97	AND reason = '${SWEEP_REASON}' ORDER BY id DESC LIMIT 1`,
	98	)
	99	.get(pull.id, pull.headOid);
	100	return row === null ? undefined : getRun(app.db, row.id);
	101	}
	102
	103	/** Close a stale red with one line; the issue it was for goes back to ready, unassigned. */
	104	function closeRed(app: App, repo: Repo, pull: Pull, why: string): boolean {
	105	const by = repo.autonomyBy === null ? undefined : getUserById(app.db, repo.autonomyBy);
	106	const issue = getIssueById(app.db, pull.id);
	107	if (!by \|\| !issue \|\| issue.state !== "open") return false;
	108	addComment(app, by, repo, issue, closedLine(why));
	109	if (!setIssueState(app, by, repo, issue, "closed", { reason: "parked", stale: true }).ok) return false;
	110	for (const n of closesFromBody(pull.body)) {
	111	const about = getIssue(app.db, repo.id, n);
	112	if (about?.state === "open" && about.assigneeId !== null) assignIssue(app, by, repo, about, null);
	113	}
	114	log.info("stale red closed", { repo: `${repo.ownerName}/${repo.name}`, pull: pull.number, why });
	115	return true;
	116	}
	117
	118	/**
	119	* One pass over a repository's open pulls. `room` is the free builder slots;
	120	* the re-checks and rework runs together stay within it and {@link PER_TICK}.
	121	*/
	122	export async function sweepStaleReds(
	123	app: App,
	124	repo: Repo,
	125	now: Date,
	126	room: number,
	127	): Promise<StaleMove[]> {
	128	const moves: StaleMove[] = [];
	129	if (repo.autonomy !== "autonomous" \|\| repo.autonomyBy === null) return moves;
	130	let budget = Math.min(PER_TICK, Math.max(room, 0));
	131	const main = redMain(app.db, repo.id);
	132	for (const pull of listOpenPulls(app.db, repo.id)) {
	133	if (!pull.authorIsAgent \|\| pull.mergedAt !== null \|\| pull.headOid === "") continue;
	134	if (getHold(app.db, pull.id) !== undefined \|\| authorsInRepoPull(pull)) continue;
	135	const issue = getIssueById(app.db, pull.id);
	136	if (!issue \|\| issue.state !== "open") continue;
	137	const agent = agentFor(app, issue);
	138	if (!agent) continue;
	139	if (approvedOnHead(app.db, pull)) continue;
	140	const red = sittingRed(app, pull, now);
	141	if (red === undefined \|\| sharedWithMain(main, red.run)) continue;
	142
	143	// Stale: red over a day and it conflicts, or its issue is gone.
	144	if (now.getTime() - Date.parse(red.at) >= STALE_MS) {
	145	const named = closesFromBody(pull.body);
	146	const live = named.some((n) => getIssue(app.db, repo.id, n)?.state === "open");
		junipercrew 12d ago outdated The conflict branch is untested. Add a test with a conflicting head and a red older than 24 h.
	147	let why: string \| undefined =
	148	named.length > 0 && !live ? "the issue it was for is no longer open" : undefined;
	149	if (why === undefined) {
	150	const dir = await pullDir(app.db, app.reposRoot, repo, pull);
	151	const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
	152	if (tip !== undefined && !(await mergeable(dir, tip, pull.headOid)).ok)
	153	why = "it conflicts with main";
	154	}
	155	if (why !== undefined && closeRed(app, repo, pull, why)) {
	156	moves.push({ pull: pull.number, did: "closed" });
	157	continue;
	158	}
	159	}
	160
	161	const since = lastHumanWord(app, issue, agent);
	162	if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
	163	const why = `the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries: ${failingLine(red.run)}`;
	164	if (parkPull(app, repo, pull, why)) moves.push({ pull: pull.number, did: "parked" });
	165	continue;
	166	}
	167
	168	if (budget <= 0) continue;
	169	const again = sweepRun(app, pull);
	170	if (again === undefined) {
	171	const checks = checksFor(app);
	172	if (checks === undefined) continue;
	173	const env = `pull-${pull.number}`;
	174	const queued = await checks.request(repo, env, pull.headOid, SWEEP_REASON, pull.id);
	175	if (queued?.reason !== SWEEP_REASON) continue;
	176	budget--;
	177	moves.push({ pull: pull.number, did: "re-check queued" });
	178	continue;
	179	}
	180	if (again.state !== "failed" \|\| again.summary.startsWith(INFRA_SUMMARY)) continue;
	181	// Said as the builder, not as a person: a person's word restarts the tries.
	182	addComment(
	183	app,
	184	agent.user,
	185	repo,
	186	issue,
	187	`Still red on current main: ${failingLine(again)}. Fix it and push.`,
	188	);
	189	if (queueAgentRun(app, repo, issue, agent, "check_failed") !== undefined) {
	190	budget--;
	191	moves.push({ pull: pull.number, did: "builder queued" });
	192	}
	193	}
	194	return moves;
	195	}
		@@ -210,6 +210,7 @@ const GENERIC: Record<KeeperCause, string> = {
210	210	usage: "Usage is the limit: no subscription has headroom.",
211	211	unassigned: "Ready issues wait and no builder is running.",
212	212	empty_pillar: "Nothing is ready: no pillar has an issue a builder can start.",
	213	stale_reds: "Open crew pulls are red and no run is working them.",
213	214	red_checks: "Open crew pulls have a red check.",
214	215	stale_pile: "Open crew pulls are waiting on a builder or a reviewer.",
215	216	capacity: "Every builder slot is busy.",
		@@ -17,12 +17,14 @@ import { atCeiling } from "../crew/dispatch.ts";
17	17	import { heldNow } from "../crew/held.ts";
18	18	import { pace } from "../crew/pacing.ts";
19	19	import { readPile } from "./pile.ts";
	20	import { readStaleReds } from "./stale_reds.ts";
20	21
21	22	export const KEEPER_CAUSES = [
22	23	"ceiling",
23	24	"usage",
24	25	"unassigned",
25	26	"empty_pillar",
	27	"stale_reds",
26	28	"red_checks",
27	29	"stale_pile",
28	30	"capacity",
		@@ -51,6 +53,8 @@ export interface RepoFacts {
51	53	export interface KeeperFacts {
52	54	readonly repos: readonly RepoFacts[];
53	55	readonly red: number;
	56	/** Red pulls with no run for 30 min and no person holding them. */
	57	readonly staleReds?: number;
54	58	readonly stale: number;
55	59	}
56	60
		@@ -65,6 +69,7 @@ export interface Found {
65	69	/** The pile that is not moving counts as stale from this many pulls; red from this many. */
66	70	export const STALE_PULLS = 20;
67	71	export const RED_PULLS = 5;
	72	export const STALE_REDS = 3;
68	73
69	74	const hhmm = (iso: string): string => `${iso.slice(11, 16)} UTC`;
70	75
		@@ -101,6 +106,10 @@ export function pickCause(f: KeeperFacts, skip: ReadonlySet<KeeperCause> = new S
101	106	const found = hit({ cause: "empty_pillar", move: "file", repoId: first.repoId, says: "Nothing is ready: no pillar has an issue a builder can start." });
102	107	if (found) return found;
103	108	}
	109	if (first && (f.staleReds ?? 0) >= STALE_REDS) {
	110	const found = hit({ cause: "stale_reds", move: "say", repoId: first.repoId, says: `${f.staleReds} open crew pulls are red and no run is working them: the sweep reworks, parks or closes them a few a tick.` });
	111	if (found) return found;
	112	}
104	113	if (first && f.red >= RED_PULLS) {
105	114	const found = hit({ cause: "red_checks", move: "file", repoId: first.repoId, says: `${f.red} open crew pulls have a red check.` });
106	115	if (found) return found;
		@@ -154,5 +163,5 @@ export function readKeeperFacts(app: App, now: Date): KeeperFacts {
154	163	});
155	164	}
156	165	const pile = readPile(db);
157		return { repos, red: pile.red, stale: pile.awaitingReview + pile.changesAsked + pile.approved };
	166	return { repos, red: pile.red, staleReds: readStaleReds(db, now).count, stale: pile.awaitingReview + pile.changesAsked + pile.approved };
158	167	}
		@@ -1,5 +1,5 @@
1	1	/**
2		* Every flow read in one page: the nine reads under src/flow/ that nothing
	2	* Every flow read in one page: the reads under src/flow/ that nothing
3	3	* else calls yet, each as one plain line, in a fixed order. `flowReadings`
4	4	* is the raw values; `flowReportLines` words them. `publicOnly` is the rule every
5	5	* read keeps: public repositories alone.
		@@ -15,6 +15,7 @@ import { readOldestApproved } from "./oldest_approved.ts";
15	15	import { readOpeners } from "./openers.ts";
16	16	import { readOutsiders } from "./outsiders.ts";
17	17	import { readReviewLatency } from "./review_latency.ts";
	18	import { readStaleReds } from "./stale_reds.ts";
18	19
19	20	/** The raw value of each read, keyed by its name, in report order. */
20	21	export function flowReadings(db: Database, now = new Date(), publicOnly = false) {
		@@ -23,6 +24,7 @@ export function flowReadings(db: Database, now = new Date(), publicOnly = false)
23	24	review_latency: readReviewLatency(db, now, publicOnly),
24	25	outsiders: readOutsiders(db, now, publicOnly),
25	26	checks_pile: readChecksPile(db, publicOnly),
	27	stale_reds: readStaleReds(db, now, publicOnly),
26	28	gate_minutes: readGateMinutes(db, now, publicOnly),
27	29	oldest_approved: readOldestApproved(db, now, publicOnly),
28	30	idle_pulls: readIdlePulls(db, now, publicOnly),
		@@ -46,6 +48,7 @@ export function flowReportLines(r: FlowReadings): string[] {
46	48	`review_latency: ${r.review_latency ? `median ${hours(r.review_latency.median)} over ${r.review_latency.count} pulls in 24 h` : "no first verdict in 24 h"}`,
47	49	`outsiders: ${r.outsiders.open} open, oldest ${r.outsiders.oldestHours} h`,
48	50	`checks_pile: ${r.checks_pile.red} red, ${r.checks_pile.queued} queued`,
	51	`stale_reds: ${r.stale_reds.count === 0 ? "none" : `${r.stale_reds.count} red with no run for 30 min, oldest ${hours(r.stale_reds.oldestHours)}`}`,
49	52	`gate_minutes: gates ${g.gate1h} min in 1 h, ${g.gate6h} min in 6 h; pull checks ${g.pull1h} min in 1 h, ${g.pull6h} min in 6 h`,
50	53	`oldest_approved: ${o ? `${o.repo}#${o.pull} approved ${hours(o.hours)} ago` : "none waiting"}`,
51	54	`idle_pulls: ${r.idle_pulls.length === 0 ? "none" : r.idle_pulls.map((p) => `${p.repo}#${p.pull}${p.issue === null ? "" : ` (closes #${p.issue})`} ${hours(p.hours)}`).join(", ")}`,
		@@ -0,0 +1,47 @@
	1	/**
	2	* Open pulls that are red and sit (#8245): the latest check on the current
	3	* head failed at least `QUIET_MS` ago, no person holds the pull, and no
	4	* agent or check run touched it since. A sweep (`crew/stale_reds.ts`)
	5	* reworks, parks or closes these; the report counts the ones it has not yet.
	6	* A re-check the sweep itself queued (reason `sweep`) is not a touch.
	7	*/
	8
	9	import type { Database } from "bun:sqlite";
	10
	11	/** How long a red pull must be left alone before it counts as sitting. */
	12	export const QUIET_MS = 30 * 60_000;
	13
	14	/** The reason of the check the sweep queues on a red head. */
	15	export const SWEEP_REASON = "sweep";
	16
	17	export interface StaleReds {
	18	readonly count: number;
	19	/** Hours the longest-sitting one has been red; null when none is. */
	20	readonly oldestHours: number \| null;
	21	}
	22
	23	/** `publicOnly` counts public repositories alone, as `readVitals` does. */
	24	export function readStaleReds(db: Database, now = new Date(), publicOnly = false): StaleReds {
	25	const cutoff = new Date(now.getTime() - QUIET_MS).toISOString();
	26	const rows = db
	27	.query<{ at: string }, [string, number]>(
	28	`SELECT l.finished_at AS at FROM issue i
	29	JOIN pull p ON p.issue_id = i.id JOIN repo r ON r.id = i.repo_id
	30	JOIN run l ON l.id = (SELECT c.id FROM run c WHERE c.kind = 'check' AND c.issue_id = i.id
	31	AND c.commit_oid = p.head_oid AND c.reason != '${SWEEP_REASON}'
	32	ORDER BY c.id DESC LIMIT 1)
	33	WHERE i.state = 'open' AND p.merged_at IS NULL AND (?2 = 0 OR r.visibility = 'public')
	34	AND l.state = 'failed' AND l.finished_at IS NOT NULL AND l.finished_at <= ?1
	35	AND NOT EXISTS (SELECT 1 FROM pull_hold h WHERE h.issue_id = i.id AND h.released_at IS NULL)
	36	AND NOT EXISTS (SELECT 1 FROM user u WHERE u.id = i.assignee_id AND u.is_agent = 0)
	37	AND NOT EXISTS (SELECT 1 FROM run t WHERE t.issue_id = i.id AND t.kind IN ('agent','check','deploy')
	38	AND t.reason != '${SWEEP_REASON}'
	39	AND (t.state IN ('queued','running') OR COALESCE(t.finished_at, t.created_at) > ?1))`,
	40	)
	41	.all(cutoff, publicOnly ? 1 : 0);
	42	const oldest = rows.reduce<string \| null>((o, r) => (o === null \|\| r.at < o ? r.at : o), null);
	43	return {
	44	count: rows.length,
	45	oldestHours: oldest === null ? null : (now.getTime() - Date.parse(oldest)) / 3_600_000,
	46	};
	47	}
		@@ -12,7 +12,7 @@ import { flowReadings, flowReportLines } from "../src/flow/report.ts";
12	12	const NOW = new Date("2026-10-26T12:00:00.000Z");
13	13	const ago = (min: number) => new Date(NOW.getTime() - min * 60_000).toISOString();
14	14	const ORDER = [
15		"build_yield", "review_latency", "outsiders", "checks_pile", "gate_minutes",
	15	"build_yield", "review_latency", "outsiders", "checks_pile", "stale_reds", "gate_minutes",
16	16	"oldest_approved", "idle_pulls", "lead_time", "openers",
17	17	];
18	18	const name = (line: string) => line.split(":")[0]!;
		@@ -76,7 +76,7 @@ beforeEach(async () => {
76	76	review(landed, "changes_requested", 2 * 60);
77	77	const outsider = issue(priv, carol, 20); // an outsider's pull, red check, opened in the last hour
78	78	pull(outsider, "h3");
79		run(priv, "check", { issue: outsider, oid: "h3", state: "failed", start: 30, end: 10 });
	79	run(priv, "check", { issue: outsider, oid: "h3", state: "failed", start: 60, end: 40 });
80	80	run(priv, "agent", { state: "ok", start: 180, end: 120, reason: "build" });
81	81	});
82	82	afterEach(() => {
		@@ -102,6 +102,7 @@ describe("flowReportLines", () => {
102	102	"review_latency: no first verdict in 24 h",
103	103	"outsiders: 0 open, oldest 0 h",
104	104	"checks_pile: 0 red, 0 queued",
	105	"stale_reds: none",
105	106	"gate_minutes: gates 0 min in 1 h, 0 min in 6 h; pull checks 0 min in 1 h, 0 min in 6 h",
106	107	"oldest_approved: none waiting",
107	108	"idle_pulls: none",
		@@ -116,6 +117,7 @@ describe("flowReportLines", () => {
116	117	expect(all.review_latency).toContain("over 1 pulls");
117	118	expect(all.outsiders).toBe("outsiders: 1 open, oldest 0 h");
118	119	expect(all.checks_pile).toBe("checks_pile: 1 red, 0 queued");
	120	expect(all.stale_reds).toBe("stale_reds: 1 red with no run for 30 min, oldest 0.7 h");
119	121	expect(all.gate_minutes).toContain("pull checks 20 min in 1 h");
120	122	expect(all.oldest_approved).toContain("priv#1");
121	123	expect(all.idle_pulls).toContain("priv#1");