d2d5008 · 81035fd · 80b7ed3 · 5c09248 · b353b64 · e1bbdcb
@@ -0,0 +1,10 @@
# The lane has stopped (#11507)
On 2026-11-12 nothing merged for 12 h 51 min and nothing said so: every check was being stopped for CPU, and main was not ahead of production, so the main-ahead alarm stayed quiet. The flow alarm `lane_stopped` (`src/flow/alarm_lane.ts`) fires for the forge's own repository in two states, as one alarm:
- **The lane has stopped:** pulls are open (or work is ready), the crew is not at its ceiling, and no pull has merged for 90 minutes. Change the limit with the `flow.lane_minutes` row of `forge_setting` (a whole number, 1 to 10080; anything else is 90). It names how long, how many checks failed since the last merge and the commonest failure line verbatim.
- **One cause failing many pulls:** the same failure line (numbers masked) failed checks on five different pulls within an hour and since the last merge. The line is the first line of a check's summary, for a check that named no failing test; a named test is `src/checks/shared_red.ts`'s. /flow says it is the forge's failure; `check_failed` is not sent to builders for it (`waitOnMain` in `src/crew/dispatch.ts`) and the stale-red sweep neither parks nor reworks its pulls, only checks them again.
It is an ask for the owner and one Sentinel issue (`src/crew/sentinel_lane.ts`), closed at the next hunt after a pull merges and the cause is gone, unless a person commented on it. `land_past_gate` does not open for it: the break-glass rests on red gates only (`landingStopped` in `src/checks/gate_stopped.ts`), since a merge drought with pulls waiting on review is not a red gate.
Left out: the Sentinel issue's evidence carries the failure count, so it is commented again as that count moves.
@@ -0,0 +1,13 @@
# Release heartbeat and the main-ahead alarm (#11507)
The release tick records a heartbeat each time it runs (`src/keeper/beat.ts`, table `keeper_beat`): when, whether it cut, skipped or failed, and why. The outcome is the keeper's own (`tickStep` in `src/keeper/keeper.ts` says it where it decides), never read back from the line's words; a tick that throws is recorded as failed (`stepWithBeat`). None for 60 minutes is said plainly ("No heartbeat from the release tick for …").
Where to see it:
- `/releases`, in the release keeper section: the last heartbeat (`#keeper-heartbeat`) and, while main is green and ahead of production, how far and for how long (`#keeper-ahead`).
- The terminal: the palette's **Release keeper** screen, gated as /releases is, shows the same two lines under what holds the next cut.
- `release_keeper` (MCP) and `GET /api/v1/keeper`: `heartbeat` (at, outcome, reason, silent, says).
The flow alarm `main_ahead` (`src/flow/alarm_main_ahead.ts`) fires when main's last finished check passed, its head is not the commit production runs, and main has been ahead for more than 90 minutes. Change the limit with the `release.ahead_minutes` row of `forge_setting` (a whole number, 1 to 10080; anything else is 90). It names production's version and commit, the commits and pulls ahead, how long, and when the tick was last heard from. It is an ask for the owner (home) and one Sentinel issue (`src/crew/sentinel_main_ahead.ts`), which closes itself at the next hunt after production catches up, unless a person commented on it. The issue names the tick's last outcome and time, never its reason: the issue may be public, and the reason is shown only to readers of /releases.
`/api/v1/telemetry/home`: `live` is the version and commit this process runs (null when unstamped); `main` is the head of the forge's own default branch and `behind` what main holds that live does not (#11676).
@@ -66,7 +66,10 @@
function oneCause(gates: readonly { reason: string | null; tests: string }[]): b
export function ownLandingStopped(db: Database, repo: { readonly id: string }): Stopped | undefined {
  const [owner, name] = ownRepoPath().split("/");
  const own = owner && name ? getRepo(db, owner, name) : undefined;
  return own !== undefined && own.id === repo.id ? landingStopped(db, own.id) : undefined;
  if (own === undefined || own.id !== repo.id) return undefined;
  // Gate evidence only: the lane alarm (#11507) is a heuristic over merges and
  // run rows, and a merge drought with pulls waiting on review is not a red gate.
  return landingStopped(db, own.id);
}
/** The one sentence home and the pipeline carry. */
      
        sentinelcrew
        1d ago
        outdated
      
      Blocking: do not let the lane alarm make landing past the gate available. A merge drought with open pulls is not a red gate. Return only landingStopped here, or add an explicit, tested condition based on failing gates or checks.
    
@@ -0,0 +1,1 @@
a57e110db188bf5db091597675159f303ad60ac496f0c841a66d62e09a6df833
@@ -0,0 +1,10 @@
// The release tick's last heartbeat (#11507): one row, replaced each time the tick
// reports in — when, what it did (cut, skipped, failed) and why. Additive.
export default `
  CREATE TABLE keeper_beat (
    id      INTEGER PRIMARY KEY CHECK (id = 1),
    at      TEXT NOT NULL,
    outcome TEXT NOT NULL CHECK (outcome IN ('cut', 'skipped', 'failed')),
    reason  TEXT NOT NULL
  );
  `;
@@ -48,6 +48,7 @@
import { appendRunLog } from "../runs/log.ts";
import { noteHeld, type HoldCause } from "../runs/held.ts";
import { decideAndRecord } from "./ledger.ts";
import { bugsFirst } from "../flow/bugs.ts";
import { forgeFailed } from "../flow/alarm_lane.ts";
import { byDials } from "./team_share.ts";
import { peopleFirst } from "./people_first.ts";
import { inMaintenance } from "../maintenance/state.ts";
@@ -1099,6 +1100,11 @@
function waitOnMain(
    });
    return true;
  }
  // One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent, no pull parked.
  if (forgeFailed(app.db, run, Date.now())) {
    log.info("check failed on a line five pulls share; the forge's failure, not the pull's", { issue: issue.number });
    return true;
  }
  const main = sharedWithMain(redMain(app.db, repo.id), run);
  if (!main && !sharedWithPulls(app.db, run)) return false;
  if (typeof commitOid === "string" && typeof env === "string") {
@@ -0,0 +1,13 @@
/** Whether a person, not an agent, has commented on an issue: then they decide, not the sentinel. */
import type { Database } from "bun:sqlite";
export function personSpoke(db: Database, issueId: string): boolean {
  const spoke = db
    .query<{ n: number }, [string]>(
      `SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id
        WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`,
    )
    .get(issueId);
  return spoke !== null && spoke.n > 0;
}
@@ -47,6 +47,10 @@
import {
} from "../checks/shared_red.ts";
import { readRunLogTail } from "../runs/log.ts";
import { clip } from "../text/clip.ts";
import { forgeCeiling } from "../flow/ceiling.ts";
import { personSpoke } from "./person_spoke.ts";
import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts";
import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts";
import type { FindingKind } from "../hunt/finding.ts";
import { addLabels, kindLabels } from "../hunt/label.ts";
@@ -400,7 +404,7 @@
export function repeatedErrors(now: number): Finding[] {
 * Each cause whose spans total more than ten minutes in the day ending at
 * `now`, quoting its sentence. Shorter totals stay in the history.
 */
export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string): Finding[] {
export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] {
  const from = now - DAY_MS;
  const byCause = new Map<
    string,
@@ -479,7 +483,7 @@
export function ownCause(cause: string, forgeRepoId: string | undefined): boolea
 * at once, not when the day has added up. The finding shares its key with
 * `degradedCauses`, so one cause is one issue.
 */
export function budgetBreaches(db: Database, now: number, forgeRepoId?: string, dataDir?: string): Finding[] {
export function budgetBreaches(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] {
  const out: Finding[] = [];
  for (const s of spans(db, now - DAY_MS, now)) {
    if (s.closedAt !== null || !BREACH_CAUSE.test(s.cause) || !ownCause(s.cause, forgeRepoId)) continue;
@@ -569,8 +573,10 @@
function sharedLines(
}
/** Everything the sentinel reads, in one pass. */
export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string): Finding[] {
export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] {
  const found = [
    ...mainAhead(db, now),
    ...laneStopped(db, now, ceilingUsd),
    ...budgetBreaches(db, now, forgeRepoId, dataDir),
    ...heldSlots(db, now),
    ...strayTriggers(db),
@@ -770,17 +776,6 @@
function sinceFix(app: App, repo: Repo, actor: User, f: Finding, evidence: strin
  return 1;
}
/** Whether a person, not an agent, has commented on the issue: then they decide, not the sentinel. */
function personSpoke(db: Database, issueId: string): boolean {
  const spoke = db
    .query<{ n: number }, [string]>(
      `SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id
        WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`,
    )
    .get(issueId);
  return spoke !== null && spoke.n > 0;
}
/** The test a shared finding names, read back from its evidence. */
const testOf = (evidence: readonly string[]): string | undefined =>
  evidence.find((e) => e.startsWith("test: "))?.slice(6);
@@ -1031,7 +1026,7 @@
export function hunt(
  let commented = 0;
  const [owner = "", name = ""] = repoPath.split("/");
  const repo = getRepo(app.db, owner, name);
  const findings = find(app.db, now, repo?.id, app.config.dataDir);
  const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config));
  const actor = sentinelActor(app.db);
  if (repo !== undefined && actor !== undefined) {
    settleShared(app, repo, actor, now);
@@ -1106,8 +1101,12 @@
export function hunt(
      }
    }
  }
  if (repo !== undefined && actor !== undefined)
    closeHeld(app, repo, actor, new Set(findings.map((f) => f.key)), now);
  if (repo !== undefined && actor !== undefined) {
    const active = new Set(findings.map((f) => f.key));
    closeHeld(app, repo, actor, active, now);
    closeCaughtUp(app, repo, actor, active, now);
    closeLaneMoving(app, repo, actor, active, now);
  }
  // A hunt with nowhere to file is recorded as such, never as a quiet one.
  const unfiled = repo === undefined || actor === undefined;
  app.db
@@ -0,0 +1,34 @@
/**
 * The sentinel's half of the lane-stopped alarm (#11507): nothing merging for
 * the limit, or one cause failing many pulls, is one finding; its issue closes
 * at the next hunt after the lane moves, unless a person spoke on it.
 */
import type { Database } from "bun:sqlite";
import type { App } from "../app.ts";
import type { Repo } from "../core/repos.ts";
import type { User } from "../core/users.ts";
import { laneAlarm } from "../flow/alarm_lane.ts";
import { closeCaughtUp } from "./sentinel_main_ahead.ts";
import type { Finding } from "./sentinel.ts";
export const LANE_KEY = "lane-stopped";
export function laneStopped(db: Database, now: number, ceilingUsd: number | null): Finding[] {
  const a = laneAlarm(db, new Date(now), { ceilingUsd });
  if (!a.fired) return [];
  return [{
    key: LANE_KEY,
    title: a.forge === true ? "One failure is stopping many pulls: the forge's, not theirs" : "The lane has stopped: nothing has merged and nothing said so",
    evidence: a.facts ?? [a.says],
    pillar: "P0 Trust: the forge says when it is not well, and is well again",
    kind: "defect",
    lookFor: "A pull merged since, and no one failure line failing five pulls within an hour.",
    urgent: true,
  }];
}
/** Close the open lane issue once the lane moves; `active` is this hunt's finding keys. */
export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void {
  closeCaughtUp(app, repo, actor, active, now, LANE_KEY, "The lane moved: a pull merged and no one failure is stopping many");
}
@@ -0,0 +1,65 @@
/**
 * The sentinel's half of the main-ahead alarm (#11507): main green and ahead
 * of production past the setting is one finding, and its issue closes itself
 * at the next hunt after production catches up — unless a person spoke on it,
 * then it is theirs to close.
 */
import type { Database } from "bun:sqlite";
import type { App } from "../app.ts";
import { addComment, setIssueState } from "../actions/issues.ts";
import { getIssueById } from "../core/issues.ts";
import type { Repo } from "../core/repos.ts";
import type { User } from "../core/users.ts";
import { mainAheadAlarm } from "../flow/alarm_main_ahead.ts";
import { createLog } from "../kernel/log.ts";
import { personSpoke } from "./person_spoke.ts";
import type { Finding } from "./sentinel.ts";
const log = createLog("sentinel");
export const MAIN_AHEAD_KEY = "main-ahead";
export function mainAhead(db: Database, now: number): Finding[] {
  const a = mainAheadAlarm(db, new Date(now));
  if (!a.fired) return [];
  return [{
    key: MAIN_AHEAD_KEY,
    title: "Main is green and ahead of production: the release tick has not cut",
    evidence: a.facts ?? [a.says],
    pillar: "P0 Trust: the forge says when it is not well, and is well again",
    kind: "defect",
    lookFor: "Production runs main's head, or main has been ahead of it for under the limit.",
    urgent: true,
  }];
}
/** Close the open main-ahead issue once its finding is gone; `active` is this hunt's finding keys. */
export function closeCaughtUp(
  app: App,
  repo: Repo,
  actor: User,
  active: ReadonlySet<string>,
  now: number,
  key: string = MAIN_AHEAD_KEY,
  cleared: string = "Production caught up with main",
): void {
  if (active.has(key)) return;
  const open = app.db
    .query<{ issue_id: string }, [string]>(
      `SELECT t.issue_id FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id WHERE i.state = 'open' AND t.key = ?1`,
    )
    .all(key);
  for (const t of open) {
    try {
      if (personSpoke(app.db, t.issue_id)) continue;
      const issue = getIssueById(app.db, t.issue_id);
      if (issue === undefined || issue.repoId !== repo.id) continue;
      const said = addComment(app, actor, repo, issue, `${cleared} at ${new Date(now).toISOString()}. Closing as done; if it falls behind again it is filed again.`);
      if (!said.ok) continue;
      setIssueState(app, actor, repo, issue, "closed", { reason: "done", system: true });
    } catch (error) {
      log.error("sentinel close failed", { key, error: String(error) });
    }
  }
}
@@ -28,6 +28,7 @@
import { closesFromBody, listOpenPulls, pullDir, type Pull } from "../core/pulls
import type { Repo } from "../core/repos.ts";
import { automaticRunsSince, getRun, INFRA_SUMMARY, type Run } from "../core/runs.ts";
import { getUserById } from "../core/users.ts";
import { type ForgeLines, forgeFailed } from "../flow/alarm_lane.ts";
import { QUIET_MS, SWEEP_REASON } from "../flow/stale_reds.ts";
import { mergeable, resolveOid } from "../git/merge.ts";
import { createLog } from "../kernel/log.ts";
@@ -129,6 +130,7 @@
export async function sweepStaleReds(
  if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return moves;
  let budget = Math.min(PER_TICK, Math.max(room, 0));
  const main = redMain(app.db, repo.id);
  const forgeLines: ForgeLines = new Map();
  for (const pull of listOpenPulls(app.db, repo.id)) {
    if (!pull.authorIsAgent || pull.mergedAt !== null || pull.headOid === "") continue;
    if (getHold(app.db, pull.id) !== undefined || authorsInRepoPull(pull)) continue;
@@ -159,7 +161,9 @@
export async function sweepStaleReds(
    }
    const since = lastHumanWord(app, issue, agent);
    if (automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
    // The forge's failure (#11507) is never a reason to park: the pull is checked again, not reworked.
    const forge = forgeFailed(app.db, red.run, now.getTime(), forgeLines);
    if (!forge && automaticRunsSince(app.db, issue.id, since) >= CHECK_FAILED_LIMIT) {
      const why = `the check stayed red through ${CHECK_FAILED_LIMIT} automatic tries: ${failingLine(red.run)}`;
      if (await parkPullTold(app, repo, pull, why, since)) moves.push({ pull: pull.number, did: "parked" });
      continue;
@@ -178,6 +182,7 @@
export async function sweepStaleReds(
      continue;
    }
    if (again.state !== "failed" || again.summary.startsWith(INFRA_SUMMARY)) continue;
    if (forgeFailed(app.db, again, now.getTime(), forgeLines)) continue;
    // Said as the builder, not as a person: a person's word restarts the tries.
    addComment(
      app,
@@ -0,0 +1,195 @@
/**
 * The "lane has stopped" flow alarm (#11507). On 2026-11-12 nothing merged for
 * 12 h 51 min and nothing said so: main was not ahead of production, so the
 * main-ahead alarm stayed quiet, and every check was being stopped for CPU.
 * Two states raise this one alarm for the forge's own repository:
 *
 * - the lane has stopped: pulls are open or work is ready, the crew is not
 *   at its ceiling, and nothing has merged for `flow.lane_minutes` (90);
 * - one cause fails many pulls: the same failure line (numbers masked) failed
 *   checks on {@link FORGE_PULLS} different pulls within an hour. That is the
 *   forge's failure, not the builders': `forgeFailed` is what the crew asks
 *   before it reworks or parks a pull for it.
 *
 * A failure line is the first line of a check's summary, for a check that
 * named no failing test (a stop for CPU, a budget, a lost machine); a named
 * test is `checks/shared_red.ts`'s. Pure of the clock.
 */
import type { Database } from "bun:sqlite";
import { spentTodayEverywhere } from "../crew/autonomy.ts";
import type { Run } from "../core/runs.ts";
import { getRepo } from "../core/repos.ts";
import { agoWords } from "../keeper/beat.ts";
import { ownRepoPath } from "../kernel/own_repo.ts";
import { ceilingReached } from "./ceiling.ts";
import { readPipeline } from "./pipeline.ts";
export const LANE_SETTING = "flow.lane_minutes";
export const LANE_DEFAULT_MIN = 90;
/** Different pulls failing one line within {@link FORGE_WINDOW_MS} make it the forge's. */
export const FORGE_PULLS = 5;
export const FORGE_WINDOW_MS = 3_600_000;
const LINE_MAX = 200;
export interface LaneAlarm {
  readonly fired: boolean;
  /** When the lane last merged, or the first failure of the shared line; set when fired. */
  readonly since?: string;
  readonly says: string;
  /** The commonest failure line, verbatim, when one exists. */
  readonly line?: string | undefined;
  /** Whether one cause is failing many pulls. */
  readonly forge?: boolean;
  readonly facts?: readonly string[];
}
/** The minutes without a merge before the alarm; a whole number from 1 to 10080 is the owner's, else 90. */
export function laneMinutes(db: Database): number {
  const row = db.query<{ value: string }, [string]>("SELECT value FROM forge_setting WHERE key = ?1").get(LANE_SETTING);
  const n = row === null ? NaN : Number(row.value);
  return Number.isInteger(n) && n >= 1 && n <= 10_080 ? n : LANE_DEFAULT_MIN;
}
/** The same line with every number as `#`, so `16.0 cores for 30s` and `15.2 cores for 31s` are one cause. */
export const maskLine = (line: string): string => line.replace(/\d+(?:\.\d+)?/g, "#");
interface Failure {
  readonly issue: string;
  readonly at: string;
  readonly line: string;
}
/** A check's failure line, or "" when it named a failing test (not this alarm's) or said nothing. */
function lineOf(summary: string, detail: string): string {
  try {
    const names = (JSON.parse(detail) as { names?: unknown } | null)?.names;
    if (Array.isArray(names) && names.length > 0) return "";
  } catch {
    // An unreadable detail names no test.
  }
  const first = summary.trim().split("\n")[0] ?? "";
  return first.replace(/^check failed: /, "").slice(0, LINE_MAX);
}
/** Failed pull checks of the repository finished since `since`, newest first, bounded. */
function failuresSince(db: Database, repoId: string, since: string): Failure[] {
  return db
    .query<{ issue_id: string | null; finished_at: string; summary: string; check_detail: string }, [string, string]>(
      `SELECT issue_id, finished_at, summary, check_detail FROM run
        WHERE repo_id = ?1 AND kind = 'check' AND state = 'failed' AND finished_at >= ?2
        ORDER BY finished_at DESC LIMIT 2000`,
    )
    .all(repoId, since)
    .map((r) => ({ issue: r.issue_id ?? "", at: r.finished_at, line: lineOf(r.summary, r.check_detail) }));
}
interface Group {
  readonly masked: string;
  /** The newest instance, verbatim. */
  readonly line: string;
  count: number;
  readonly pulls: Set<string>;
  since: string;
}
function groups(failures: readonly Failure[]): Group[] {
  const by = new Map<string, Group>();
  for (const f of failures) {
    if (f.line === "") continue;
    const m = maskLine(f.line);
    const g = by.get(m) ?? { masked: m, line: f.line, count: 0, pulls: new Set<string>(), since: f.at };
    g.count++;
    if (f.issue !== "") g.pulls.add(f.issue);
    if (f.at < g.since) g.since = f.at;
    by.set(m, g);
  }
  return [...by.values()].sort((a, b) => b.count - a.count || (a.masked < b.masked ? -1 : 1));
}
/** When a pull of the repository last merged. */
function lastMerge(db: Database, repoId: string): string | null {
  return (
    db
      .query<{ at: string | null }, [string]>(`SELECT MAX(p.merged_at) AS at FROM pull p JOIN issue i ON i.id = p.issue_id WHERE i.repo_id = ?1`)
      .get(repoId)?.at ?? null
  );
}
/** The failure lines that failed {@link FORGE_PULLS} pulls' checks within the hour and since the last merge (a merge proves the lane): the forge's own. */
function forgeGroups(db: Database, repoId: string, now: number): Group[] {
  const hour = new Date(now - FORGE_WINDOW_MS).toISOString();
  const merged = lastMerge(db, repoId);
  const since = merged !== null && merged > hour ? merged : hour;
  return groups(failuresSince(db, repoId, since)).filter((g) => g.pulls.size >= FORGE_PULLS);
}
/** The masked forge lines per repository, read once per sweep: a sweep asks {@link forgeFailed} for every pull. */
export type ForgeLines = Map<string, ReadonlySet<string>>;
/** Whether this failed check's line is one the forge failed on many pulls: the crew neither reworks nor parks for it. */
export function forgeFailed(db: Database, run: Run | undefined, now: number, memo?: ForgeLines): boolean {
  if (run === undefined || run.kind !== "check" || run.state !== "failed") return false;
  const line = lineOf(run.summary, run.checkDetail === null || run.checkDetail === undefined ? "{}" : JSON.stringify(run.checkDetail));
  if (line === "") return false;
  let lines = memo?.get(run.repoId);
  if (lines === undefined) {
    lines = new Set(forgeGroups(db, run.repoId, now).map((g) => g.masked));
    memo?.set(run.repoId, lines);
  }
  return lines.has(maskLine(line));
}
function ownRepoId(db: Database): string | undefined {
  const [owner, name] = ownRepoPath().split("/");
  return owner && name ? getRepo(db, owner, name)?.id : undefined;
}
const count = (n: number, noun: string) => `${n} ${noun}${n === 1 ? "" : "s"}`;
export function laneAlarm(db: Database, now: Date, forge: { readonly ceilingUsd: number | null } = { ceilingUsd: null }): LaneAlarm {
  const quiet = { fired: false, says: "Pulls are landing, or there is nothing to land, or the crew is paused on purpose." } as const;
  const repoId = ownRepoId(db);
  if (repoId === undefined) return quiet;
  const at = now.getTime();
  const shared = forgeGroups(db, repoId, at)[0];
  const last = lastMerge(db, repoId);
  const open = db
    .query<{ n: number }, [string]>(`SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND kind = 'pull' AND state = 'open'`)
    .get(repoId)?.n ?? 0;
  const limit = laneMinutes(db) * 60_000;
  let stopped = false;
  let wait = 0;
  if (last !== null && (at - Date.parse(last) > limit)) {
    wait = at - Date.parse(last);
    const ready = open > 0 ? open : (readPipeline(db, now).find((s) => s.key === "ready")?.waiting ?? 0);
    stopped = ready > 0 && !ceilingReached(spentTodayEverywhere(db, now), forge.ceilingUsd);
  }
  if (!stopped && shared === undefined) return quiet;
  const parts: string[] = [];
  const facts: string[] = [];
  let line: string | undefined;
  let since: string;
  if (stopped) {
    const failed = failuresSince(db, repoId, last!);
    const top = groups(failed)[0];
    line = top?.line;
    since = last!;
    parts.push(
      `Nothing has merged for ${agoWords(wait).replace(" ago", "")} (since ${last}) with ${count(open, "pull")} open; ${failed.length} ${failed.length === 1 ? "check" : "checks"} failed in that span` +
        (top === undefined ? "." : `, ${top.count} of them with: ${top.line}`),
    );
    facts.push(`last merge at ${last}`, `${count(open, "pull")} open`, `${failed.length} checks failed since the last merge`);
    if (top !== undefined) facts.push(`commonest failure, ${top.count} times: ${top.line}`);
  } else since = shared!.since;
  if (shared !== undefined) {
    line = line ?? shared.line;
    if (stopped && shared.masked !== maskLine(line)) facts.push(`failed ${count(shared.pulls.size, "pull")} within the hour: ${shared.line}`);
    parts.push(
      `One cause is failing ${count(shared.pulls.size, "pull")} within the hour: ${shared.line}. That is the forge's failure, not the builders': their reworks and parks for it are held`,
    );
    facts.push(`forge failure on ${count(shared.pulls.size, "pull")} within the hour since ${shared.since}: ${shared.line}`);
  }
  return { fired: true, since, says: `${parts.join(". ")}.`.replace(/\.\.$/, "."), line, forge: shared !== undefined, facts };
}
@@ -0,0 +1,91 @@
/**
 * The "main is ahead of production" flow alarm (#11507): the forge's own main
 * head is proven green (its last finished check passed), differs from the
 * commit production runs, and has for longer than the setting
 * `release.ahead_minutes` (default 90). On 2026-11-11 production stood still
 * for 5 h 50 min with main green and nothing said so. Names production's
 * version and commit, how far main is ahead (commits and pulls), how long, and
 * the last time the release tick was heard from. Pure of the clock.
 */
import type { Database } from "bun:sqlite";
import { getRepo } from "../core/repos.ts";
import { agoWords, lastBeat } from "../keeper/beat.ts";
import { ownRepoPath } from "../kernel/own_repo.ts";
import { running } from "../kernel/version.ts";
import { waitingRows } from "../release/landings.ts";
export const AHEAD_SETTING = "release.ahead_minutes";
export const AHEAD_DEFAULT_MIN = 90;
export interface MainAheadAlarm {
  readonly fired: boolean;
  /** When main first moved past production; while it is ahead. */
  readonly since?: string;
  readonly says: string;
  /** Facts for the issue and the ask; only when fired. */
  readonly facts?: readonly string[];
}
/** The minutes main may be ahead and green before the alarm; a whole number of 1 or more is the owner's, else 90. */
export function aheadMinutes(db: Database): number {
  const row = db.query<{ value: string }, [string]>("SELECT value FROM forge_setting WHERE key = ?1").get(AHEAD_SETTING);
  const n = row === null ? NaN : Number(row.value);
  return Number.isInteger(n) && n >= 1 && n <= 10_080 ? n : AHEAD_DEFAULT_MIN;
}
/** The commit of the last finished check of the forge's default branch, when that check passed. */
function greenHead(db: Database): string | undefined {
  const [owner, name] = ownRepoPath().split("/");
  const repo = owner && name ? getRepo(db, owner, name) : undefined;
  if (repo === undefined) return undefined;
  const last = db
    .query<{ state: string; commit_oid: string }, [string, string]>(
      `SELECT state, commit_oid FROM run
        WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL AND env = ?2 AND state IN ('ok', 'failed')
        ORDER BY id DESC LIMIT 1`,
    )
    .get(repo.id, repo.defaultBranch);
  return last?.state === "ok" ? last.commit_oid : undefined;
}
export function mainAheadAlarm(db: Database, now: Date): MainAheadAlarm {
  const quiet = { fired: false, says: "Production runs main's head, or main's head is not proven green." } as const;
  // The cheap checks first: this runs on every /flow and home request, and the waiting pulls are read only once main's green head is not production's.
  const prod = running();
  if (prod.commit === null) return quiet;
  const head = greenHead(db);
  if (head === undefined || head.startsWith(prod.commit) || prod.commit.startsWith(head)) return quiet;
  const waiting = waitingRows(db);
  const first = waiting[0];
  if (first === undefined) return quiet;
  const wait = now.getTime() - Date.parse(first.at);
  const limit = aheadMinutes(db) * 60_000;
  if (!(wait > limit)) return { fired: false, since: first.at, says: `Main is ahead of production ${agoWords(wait).replace(" ago", "")}; not yet due.` };
  const commits = new Set(waiting.map((r) => r.oid).filter((o) => o !== "")).size || waiting.length;
  const pulls = waiting.length;
  const beat = lastBeat(db);
  const tick = beat === null ? "never" : agoWords(now.getTime() - Date.parse(beat.at));
  const how = `${commits} ${commits === 1 ? "commit" : "commits"} and ${pulls} ${pulls === 1 ? "pull" : "pulls"}`;
  const forHow = agoWords(wait).replace(" ago", "");
  const facts = [
    `production runs ${prod.version} at ${prod.commit.slice(0, 7)}`,
    `main is green at ${head.slice(0, 7)}, ${how} ahead`,
    `ahead since ${first.at}`,
    // Absolute times: the evidence is commented again only when it changes, not as the clock moves.
    // The tick's reason stays out: these facts are a Sentinel issue's, which may be public; /releases shows it to its readers.
    `the release tick was last heard from ${beat === null ? "never" : `${beat.at} (${beat.outcome})`}`,
  ];
  return {
    fired: true,
    since: first.at,
    says: `Main is green and ${how} ahead of production (${prod.version} at ${prod.commit.slice(0, 7)}) for ${forHow}; the release tick was last heard from ${tick}.`,
    facts,
  };
}
/** Main's state against production as one sentence for /releases and the terminal; undefined when main is not ahead and green. */
export function aheadLine(db: Database, now: Date): string | undefined {
  const a = mainAheadAlarm(db, now);
  return a.fired || a.since !== undefined ? a.says : undefined;
}
@@ -16,6 +16,8 @@
import { alarmBackup } from "./alarm_backup.ts";
import { alarmFromApp } from "./alarm_from_app.ts";
import { alarmPaidOrder } from "./alarm_paid_order.ts";
import { alarmIdleCrew } from "./alarm_idle_crew.ts";
import { laneAlarm } from "./alarm_lane.ts";
import { mainAheadAlarm } from "./alarm_main_ahead.ts";
import { noReleaseAlarm } from "./alarm_no_release.ts";
import { redMainAlarm } from "./alarm_red_main.ts";
import { alarmRepeat } from "./alarm_repeat.ts";
@@ -23,7 +25,7 @@
import { alarmSilent } from "./alarm_silent.ts";
import { alarmStuck } from "./alarm_stuck.ts";
export interface FiredAlarm {
  readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release" | "bug_clock" | "silent" | "repeat" | "from_app" | "unreachable" | "backup" | "paid_order";
  readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release" | "main_ahead" | "lane_stopped" | "bug_clock" | "silent" | "repeat" | "from_app" | "unreachable" | "backup" | "paid_order";
  readonly says: string;
  readonly since: string;
  /** A same-origin path to the item the alarm names. */
@@ -70,6 +72,10 @@
export function firedAlarms(
  }
  const release = noReleaseAlarm(db, now);
  if (release.fired) out.push({ key: "no_release", says: release.says, since: release.since!, href: "/releases" });
  const ahead = mainAheadAlarm(db, now);
      
        sentinelcrew
        1d ago
        outdated
      
      Issue #11507 asks /flow to show the last heartbeat; only the alarm is added. Add the heartbeat line here, gated like /releases.
    
  if (ahead.fired) out.push({ key: "main_ahead", says: ahead.says, since: ahead.since!, href: "/releases" });
  const lane = laneAlarm(db, now, { ceilingUsd: opts.ceilingUsd ?? null });
  if (lane.fired) out.push({ key: "lane_stopped", says: lane.says, since: lane.since!, href: "/flow" });
  const bug = bugClockAlarm(db, now, opts.publicOnly ? { publicOnly: true } : {});
  if (bug.fired) out.push({ key: "bug_clock", says: bug.says, since: bug.since!, href: bug.href });
  return out;
@@ -4,7 +4,7 @@
import { bad, good, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "release_keeper",
  doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; for whoever may read /releases",
  doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases",
  description: "The release keeper's releases, holds and what blocks the next cut.",
  inputSchema: { type: "object", properties: {} },
  writes: false,
@@ -0,0 +1,85 @@
/**
 * The release tick's heartbeat (#11507): each time the tick runs it reports
 * what it did — cut, skipped and why, or failed and why — and the forge keeps
 * the last. The tick runs outside the forge's own judgment, so when it stops
 * this is the only trace; `release_keeper`, /releases and the terminal's
 * release keeper screen show it, and a beat older
 * than {@link SILENT_MS} is said plainly. Plain rows in, plain rows out.
 */
import type { Database } from "bun:sqlite";
export type Outcome = "cut" | "skipped" | "failed";
/** No heartbeat for this long is itself said. */
export const SILENT_MS = 60 * 60_000;
const REASON_MAX = 300;
export interface Beat {
  readonly at: string;
  readonly outcome: Outcome;
  readonly reason: string;
}
export function recordBeat(db: Database, outcome: Outcome, reason: string, at: Date = new Date()): Beat {
  const beat: Beat = { at: at.toISOString(), outcome, reason: reason.replace(/\s+/g, " ").trim().slice(0, REASON_MAX) };
  db.query(
    `INSERT INTO keeper_beat (id, at, outcome, reason) VALUES (1, ?1, ?2, ?3)
     ON CONFLICT (id) DO UPDATE SET at = excluded.at, outcome = excluded.outcome, reason = excluded.reason`,
  ).run(beat.at, beat.outcome, beat.reason);
  return beat;
}
export function lastBeat(db: Database): Beat | null {
  return db.query<Beat, []>("SELECT at, outcome, reason FROM keeper_beat WHERE id = 1").get() ?? null;
}
/** What one tick did, as the keeper says it: the line and its outcome, never read back from the words. */
export interface Step {
  readonly line: string;
  readonly outcome: Outcome;
}
/** Run one tick and record its beat; a tick that throws is recorded as failed and the error goes on. */
export async function stepWithBeat(db: Database, step: () => Promise<Step>, now: () => Date = () => new Date()): Promise<string> {
  let done: Step;
  try {
    done = await step();
      
        sentinelcrew
        1d ago
        outdated
      
      If recordBeat throws here the catch below records 'failed' for a tick that succeeded. Separate the step's error from the beat write.
    
  } catch (error) {
    writeBeat(db, "failed", `step failed: ${error instanceof Error ? error.message : String(error)}`, now());
    throw error;
  }
  writeBeat(db, done.outcome, done.line, now());
  return done.line;
}
/** The beat is a trace, not the tick: a write that fails is said and never turns a good tick into a failed one. */
function writeBeat(db: Database, outcome: Outcome, reason: string, at: Date): void {
  try {
    recordBeat(db, outcome, reason, at);
  } catch (error) {
    console.error(`keeper heartbeat not recorded: ${error instanceof Error ? error.message : String(error)}`);
  }
}
/** "never", "4 min ago", "2 h 5 min ago". */
export function agoWords(ms: number): string {
  const min = Math.max(0, Math.floor(ms / 60_000));
  if (min < 60) return `${min} min ago`;
  return `${Math.floor(min / 60)} h ${min % 60} min ago`;
}
/** The heartbeat as one sentence, saying plainly when there is none or it is old. */
export function beatLine(beat: Beat | null, now: Date): string {
  if (beat === null) return "The release tick has never reported a heartbeat.";
  const age = now.getTime() - Date.parse(beat.at);
  const said = `The release tick last reported ${agoWords(age)} (${beat.outcome}): ${beat.reason}`;
  return age > SILENT_MS ? `No heartbeat from the release tick for ${agoWords(age).replace(" ago", "")}. ${said}` : said;
}
/** The heartbeat as a `dl` row's value: outcome, age and reason; "never reported" before the first. */
export function beatFact(beat: Beat | null, now: Date): string {
  if (beat === null) return "never reported";
  const age = now.getTime() - Date.parse(beat.at);
  return `${beat.outcome}, ${agoWords(age)}${age > SILENT_MS ? " (silent)" : ""}: ${beat.reason}`;
}
@@ -37,6 +37,7 @@
import { stallReason, STALL_HOLD_MS } from "../perf/boot_stall.ts";
import { countRollProbe, dropCut, finish, getKeeperRelease, inFlight, keepStall, keepTally, listKeeperReleases, markLive, releaseBefore, recordCut, type KeeperRelease } from "./record.ts";
import { rollBackToNamed } from "./rollback.ts";
import type { Probe, Tally, Verdict } from "./verify.ts";
import type { Step } from "./beat.ts";
/** A release not live this long after its cut never rolled: 3 h (the tag's check may take 2 h, the build some more). */
export const ROLL_DEADLINE_MS = 3 * 3_600_000;
@@ -116,8 +117,8 @@
function fileDark(p: KeeperPorts, r: KeeperRelease): number | undefined {
const verifying = new WeakMap<KeeperPorts, number>();
/** The verify of an in-flight release this process is the new version of, to its end. */
async function verifyAndDecide(p: KeeperPorts, r: KeeperRelease): Promise<string> {
  if (verifying.has(p)) return `verifying ${r.version}`;
async function verifyAndDecide(p: KeeperPorts, r: KeeperRelease): Promise<Step> {
  if (verifying.has(p)) return { line: `verifying ${r.version}`, outcome: "skipped" };
  verifying.set(p, r.id);
  try {
    const verdict = await p.verify((t) => keepTally(p.db, r.id, t, r.rollNon200));
@@ -131,9 +132,9 @@
async function verifyAndDecide(p: KeeperPorts, r: KeeperRelease): Promise<string
      const lifted = liftFixed(p.db, r.version, p.now());
      p.bus.append({ kind: "release.verified_live", repoId: p.repo.id, payload: { ...numbers, holdsLifted: lifted } });
      fileDark(p, getKeeperRelease(p.db, r.id)!);
      return `${r.version} verified live: ${kept.probes} probes, worst ${kept.worstMs} ms, ${kept.secondsDark} s dark${stalled === undefined ? "" : `; ${stalled}`}`;
      return { line: `${r.version} verified live: ${kept.probes} probes, worst ${kept.worstMs} ms, ${kept.secondsDark} s dark${stalled === undefined ? "" : `; ${stalled}`}`, outcome: "cut" };
    }
    return await rollBack(p, kept, verdict.reason);
    return { line: await rollBack(p, kept, verdict.reason), outcome: "failed" };
  } finally {
    verifying.delete(p);
  }
@@ -171,6 +172,16 @@
async function rollBack(p: KeeperPorts, r: KeeperRelease, reason: string): Promi
/** One step of the keeper; the line says what it did or why it did nothing. */
export async function tick(p: KeeperPorts): Promise<string> {
  return (await tickStep(p)).line;
}
/**
 * One step with its outcome, said where it is decided (#11507): `cut` when it
 * moved a release (cut one, or verified one live), `failed` when something
 * refused or broke (a verify, a tag, the deployer's admit, main unreadable),
 * else `skipped` with the reason.
 */
export async function tickStep(p: KeeperPorts): Promise<Step> {
  const holds = standingHolds(p.db);
  findCauses(p, holds);
  const flying = inFlight(p.db);
@@ -182,14 +193,14 @@
export async function tick(p: KeeperPorts): Promise<string> {
    // A cut recorded but never tagged (the process died between the two): tag it now, or drop it.
    if (flying.stage === "rolling" && (await p.newest())?.version !== flying.version) {
      const refused = await tagged(p, flying);
      if (refused !== undefined) return refused;
      if (refused !== undefined) return { line: refused, outcome: "failed" };
    }
    if (flying.stage === "rolling" && p.now() - Date.parse(flying.cutAt) > ROLL_DEADLINE_MS)
      return rollBack(p, flying, `${flying.version} was not live ${ROLL_DEADLINE_MS / 3_600_000} h after its cut`);
      return { line: await rollBack(p, flying, `${flying.version} was not live ${ROLL_DEADLINE_MS / 3_600_000} h after its cut`), outcome: "failed" };
    // A promote that did not take (the API refused, or something set the image back) is sent again.
    const want = refFor(p, flying.knownGood.image, flying.version, flying.sha);
    if (flying.stage === "rolling" && (await p.deployer.image()) !== want) await promote(p, flying.version, want);
    return `${flying.version} is ${flying.stage}`;
    return { line: `${flying.version} is ${flying.stage}`, outcome: "skipped" };
  }
  const main = await p.main();
  const newest = await p.newest();
@@ -207,19 +218,20 @@
export async function tick(p: KeeperPorts): Promise<string> {
    now: p.now(),
    ...(p.minIntervalMs === undefined ? {} : { minIntervalMs: p.minIntervalMs }),
  });
  if (!decision.cut || main === undefined) return decision.cut ? "main is unreadable" : decision.why;
  if (!decision.cut) return { line: decision.why, outcome: "skipped" };
  if (main === undefined) return { line: "main is unreadable", outcome: "failed" };
  // A tree the deployer cannot release (dependencies changed, preflight red) is not cut: no tag, no row.
  const refused = await p.deployer.admits?.(main);
  if (refused !== undefined) return `not cut: ${refused}`;
  if (refused !== undefined) return { line: `not cut: ${refused}`, outcome: "failed" };
  const image = await p.deployer.image();
  const knownGood = { version: p.live(), image };
  // The row first, the tag second: a crash between them leaves a `rolling` row
  // whose tag the next tick sends (`tagged`), never a tag the keeper has no row for.
  const r = recordCut(p.db, { version: decision.version, sha: main, knownGood, at: p.now() });
  const tagRefused = await tagged(p, r);
  if (tagRefused !== undefined) return tagRefused;
  if (tagRefused !== undefined) return { line: tagRefused, outcome: "failed" };
  const sent = await promote(p, r.version, refFor(p, image, r.version, main));
  return `cut ${r.version} at ${main.slice(0, 12)}; ${sent}; known good: ${knownGood.version}`;
  return { line: `cut ${r.version} at ${main.slice(0, 12)}; ${sent}; known good: ${knownGood.version}`, outcome: "cut" };
}
const refFor = (p: KeeperPorts, image: string, version: string, sha: string): string =>
@@ -7,6 +7,7 @@
 */
import type { Database } from "bun:sqlite";
import { beatLine, lastBeat, SILENT_MS, type Beat } from "./beat.ts";
import { blocking, listHolds, standingHolds, type Hold } from "./hold.ts";
import { listKeeperReleases, type KeeperRelease } from "./record.ts";
@@ -55,11 +56,24 @@
export function withLandedToLive(db: Database, releases: readonly KeeperRelease[
  return releases.map((r) => ({ ...r, commit: r.sha.slice(0, 7), landedToLiveSeconds: landedToLive(db, r) ?? null }));
}
/** The tick's last heartbeat with its sentence and whether it has gone quiet (#11507); null before the first. */
export type Heartbeat = Beat & { readonly silent: boolean; readonly says: string };
export function heartbeatOf(db: Database, now: Date = new Date()): Heartbeat | null {
  const beat = lastBeat(db);
  if (beat === null) return null;
  return { ...beat, silent: now.getTime() - Date.parse(beat.at) > SILENT_MS, says: beatLine(beat, now) };
}
/** What the keeper did and what stops the next cut: one answer for MCP `release_keeper` and `GET /api/v1/keeper`. */
export function keeperAnswer(db: Database): { blocked: string | null; releases: LandedRelease[]; holds: Hold[] } {
export function keeperAnswer(
  db: Database,
  now: Date = new Date(),
): { blocked: string | null; releases: LandedRelease[]; holds: Hold[]; heartbeat: Heartbeat | null } {
  return {
    blocked: blocking(standingHolds(db)) ?? null,
    releases: withLandedToLive(db, listKeeperReleases(db)),
    holds: listHolds(db),
    heartbeat: heartbeatOf(db, now),
  };
}
@@ -97,7 +97,8 @@
import { proveRunning } from "./release/prove.ts";
import { CHECK_EVERY_MS, setBehind, watchBehind } from "./release/behind.ts";
import { readLandings } from "./release/landings.ts";
import { ownTagsReader } from "./release/own_tags.ts";
import { rollProbe, tick as keeperTick } from "./keeper/keeper.ts";
import { rollProbe, tickStep as keeperTick } from "./keeper/keeper.ts";
import { stepWithBeat } from "./keeper/beat.ts";
import { keeperEnv, keeperPorts } from "./keeper/wire.ts";
import { snapshotPlace } from "./actions/health.ts";
import { capacityDefault, egressEnvIgnored } from "./core/capacity.ts";
@@ -824,7 +825,7 @@
if (keeperOn.on) {
  else {
    let said = "";
    const step = () =>
      void keeperTick(ports)
      void stepWithBeat(app.db, () => keeperTick(ports))
        .then((line) => {
          if (line !== said) app.log.info("release keeper", { line });
          said = line;
@@ -139,6 +139,14 @@
export function waitingSince(db: Database): string | undefined {
  return onItsWay.at(-1)?.at;
}
/** The merged pulls not yet in the running commit, oldest first, with their merge commits; empty when nothing was read (#11507). */
export function waitingRows(db: Database): MergedRow[] {
  if (kept === undefined) return [];
  const rows = mergedRows(db);
  const way = new Set(split(rows, kept).onItsWay.map((l) => `${l.repo}#${l.number}`));
  return rows.filter((r) => way.has(`${r.repo}#${r.number}`)).reverse();
}
/** How long a miss is believed: a repository can turn public, or merge its first pull, after boot. */
const MISS_MS = 60_000;
@@ -11,6 +11,7 @@
import type { App } from "../app.ts";
import { SOURCES } from "../boards/sources.ts";
import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../flow/conflicted.ts";
import { firedAlarms } from "../flow/alarms.ts";
import { forgeCeiling } from "../flow/ceiling.ts";
import { running } from "../kernel/version.ts";
import { readTalkTurns, talkTurnsWords } from "../flow/talk_turns.ts";
import { readTalkResumes, talkResumesWords } from "../flow/talk_resumes.ts";
@@ -18,6 +19,7 @@
import { readVitals, vitalsFor, vitalsLine, vitalsMisses } from "../flow/vitals.
import { parkedSays, readParked } from "../flow/parked.ts";
import { readDeclines } from "../flow/declines.ts";
import { readKeeper } from "../flow/keeper.ts";
import { heartbeatOf } from "../keeper/landed.ts";
import { readPipeline, readStepStats, statWords, verdict } from "../flow/pipeline.ts";
import { type QueueReading, readQueues } from "../flow/queue.ts";
import { reviewersBusy, reviewingSays } from "../flow/reviewing.ts";
@@ -103,6 +105,8 @@
function flowBody(x: {
    steps: x.steps.map((m) => ({ ...m, says: statWords(m) })),
    // The cause binding landings now and the keeper's last act (#7645).
    keeper: readKeeper(app.db, reader.kind === "owner"),
    // The release tick's last report, reason and all, for the owner only (#11507): the reason may name private work.
    heartbeat: reader.kind === "owner" ? heartbeatOf(app.db, now) : null,
    // How long each main has gone unverified after landing (#7786); private repositories for the owner only.
    verified: x.verified.map((row) => ({ ...row, says: verifiedSays(row, now.getTime()) })),
    // Issues the crew parked in the last 24 h and 7 d, the newest three named (#6471).
@@ -117,7 +121,7 @@
function flowBody(x: {
    // Declined build runs of the last 24 h by reason, each with the runs behind it.
    declines: readDeclines(app.db, now, reader.kind === "agent"),
    // The four alarms (#5930) that are firing now; empty when none is.
    alarms: firedAlarms(app.db, now, { publicOnly: reader.kind === "agent", dataDir: app.config.dataDir }),
    alarms: firedAlarms(app.db, now, { publicOnly: reader.kind === "agent", dataDir: app.config.dataDir, ceilingUsd: forgeCeiling(app.config) }),
    // Review runs in flight per reviewer, cut to public repositories for an agent (#4733).
    reviewing: { says: reviewingSays(busy), reviewers: busy },
    conflicted: { says: conflictedSays(conflicted, now.getTime()), pulls: conflicted },
@@ -83,6 +83,7 @@
import { readHandoverLists } from "../flow/handover_read.ts";
import { readDrifts } from "../flow/alarm_drift.ts";
import { readHeld } from "../flow/held.ts";
import { answerKeeper, readKeeper } from "../flow/keeper.ts";
import { beatLine, lastBeat } from "../keeper/beat.ts";
import { readVerified } from "../flow/verified.ts";
import { memoryScreen } from "./screens/system.ts";
import { noteRead, telemetryReader } from "../actions/telemetry.ts";
@@ -502,6 +503,7 @@
export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
              readVerified(session.app, flowReader.kind === "owner"),
              keeper,
              session.app.config.dataDir,
              flowReader.kind === "owner" ? beatLine(lastBeat(session.db), new Date()) : undefined,
            ), keeper.ask && user ? { ask: keeper.ask, answer: () => answerKeeper(session.app, keeper.ask!.id, user.id) } : undefined,
            mayDrift(session)
              ? { read: async () => readHandoverLists(session.db, await readDrifts(session.app), Date.now()), update: (n) => askUpdate(session, n) }
@@ -93,6 +93,8 @@
export function flowLines(
  keeper?: KeeperRead,
  /** Where the backup outcome files are kept; without it the off-box backup alarm cannot be read (the terminal passes the app's). */
  dataDir: string | null = null,
  /** The release tick's heartbeat line, an owner's only (#11507). */
  heartbeat?: string,
): string[] {
  const stages = readPipeline(db, now);
  const flow = readThroughput(db, stages.map((s) => s.key), now);
@@ -115,6 +117,7 @@
export function flowLines(
  return [
    ...wrap(verdict(stages, now.getTime()), inner),
    ...(keeper === undefined || keeperLine(keeper) === "" ? [] : ["", ...wrap(keeperLine(keeper), inner)]),
    ...(heartbeat === undefined ? [] : ["", ...wrap(heartbeat, inner)]),
    ...(keeper?.ask == null ? [] : wrap(`${keeper.ask.question} Press l to ${keeper.ask.button.toLowerCase()}.`, inner)),
    "",
    ...wrap(vitalsLine(vitalsFor(readVitals(db, now, agent !== undefined), agent === undefined)), inner),
@@ -1,6 +1,7 @@
/**
 * The release keeper in the terminal (#7557), the /releases card's read:
 * what holds the next cut, then each release the keeper cut with its stage,
 * what holds the next cut, the release tick's heartbeat and whether main is
 * green and ahead of production (#11507), then each release the keeper cut with its stage,
 * its short commit, landed to live (#8042), the known-good version it rolls
 * back to and what the probes counted.
 * Gated by `release/gate.ts` as the page is. `h` holds releases with a
@@ -8,6 +9,8 @@
 * functions the card and MCP `release_hold` call.
 */
import { aheadLine } from "../../flow/alarm_main_ahead.ts";
import { beatLine, lastBeat } from "../../keeper/beat.ts";
import { stallWords } from "../../perf/boot_stall.ts";
import { blocking, holdAs, liftAs, standingHolds } from "../../keeper/hold.ts";
import { secondsWords, withLandedToLive, type LandedRelease } from "../../keeper/landed.ts";
@@ -32,13 +35,23 @@
const STAGE: Record<Stage, { label: string; tone: Tone }> = {
export interface KeeperRead {
  /** What holds the next cut, or the line that says nothing does. */
  readonly blocked: string;
  /** The release tick's last heartbeat as a sentence, said plainly when there is none or it is old. */
  readonly heartbeat: string;
  /** Main green and ahead of production, and for how long; absent when it is not. */
  readonly ahead?: string;
  readonly releases: readonly LandedRelease[];
}
/** The card's content for this person; undefined for anyone the page would 404. */
export function readKeeper(f: Session): KeeperRead | undefined {
export function readKeeper(f: Session, now: Date = new Date()): KeeperRead | undefined {
  if (!mayRead(f.app, f.user)) return undefined;
  return { blocked: blocking(standingHolds(f.db)) ?? "Nothing holds the next cut.", releases: withLandedToLive(f.db, listKeeperReleases(f.db)) };
  const ahead = aheadLine(f.db, now);
  return {
    blocked: blocking(standingHolds(f.db)) ?? "Nothing holds the next cut.",
    heartbeat: beatLine(lastBeat(f.db), now),
    ...(ahead === undefined ? {} : { ahead }),
    releases: withLandedToLive(f.db, listKeeperReleases(f.db)),
  };
}
export function keeperRows(releases: readonly LandedRelease[]): ListRow[] {
@@ -90,7 +103,11 @@
export function keeperScreen(f: Session, ctx: Ctx): Screen {
        return;
      }
      // What holds the cut ends with what to land ("release hold N"): wrapped, up to three lines, never cut short of it.
      const lines = wrap(read.blocked, area.w).slice(0, 3);
      const lines = [
        ...wrap(read.blocked, area.w).slice(0, 3),
        ...wrap(read.heartbeat, area.w).slice(0, 2),
        ...(read.ahead === undefined ? [] : wrap(read.ahead, area.w).slice(0, 2)),
      ];
      const [head, body] = split(area, "v", [len(lines.length + 1), fill(1)]);
      lines.forEach((l, i) => s.text(head!.x, head!.y + i, truncate(l, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w));
      if (read.releases.length === 0) {
@@ -53,6 +53,7 @@
import type { App } from "../../app.ts";
import { html, type Html } from "../html.ts";
import { doctorCard } from "./boards-doctor.ts";
import { keeperRow } from "../studio/keeper_row.ts";
import { beatFact, lastBeat } from "../../keeper/beat.ts";
import { gatesSection } from "./boards-gates.ts";
import { heldCard } from "./boards-held.ts";
import { badge, card, meter } from "../ui.ts";
@@ -127,7 +128,9 @@
export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
    ["Verified after landing", html`A repository that declares <span class="mono">check.verify</span> in forge.json has its main verified after landing.`],
    ["Elsewhere", html`Limits are set on <a href="/settings/capacity">Capacity</a>. How the forge stands against its threats is on <a href="/resilience">Resilience</a>; where an outside agent can help is on <a href="/contribute">Contribute</a>.`],
  ], helpAsked(url));
  return html`${keeperRow(app.db, reader.kind === "owner")}${help}${card(
  return html`${keeperRow(app.db, reader.kind === "owner")}${reader.kind === "owner"
    ? html`<dl class="status-rows" id="keeper-heartbeat-row"><div class="status-row"><dt>Release tick</dt><dd id="keeper-heartbeat">${beatFact(lastBeat(app.db), new Date(now))}</dd></div></dl>`
    : ""}${help}${card(
      html`<p id="bottleneck" data-stage="${worst?.key ?? ""}">
          <strong>${verdict(stages, now)}</strong>
        </p>
@@ -1,11 +1,14 @@
/**
 * The release keeper on /releases (#7557): what stops the next cut, and
 * each release it cut with its stage, the known-good version it rolls back
 * to, and what the probes counted. Nothing when it has done nothing.
 * to, and what the probes counted; the release tick's heartbeat and whether
 * main is green and ahead of production (#11507). Nothing when it has done nothing.
 * A plain section under an h2, no card (#9998).
 */
import type { Database } from "bun:sqlite";
import { aheadLine } from "../../flow/alarm_main_ahead.ts";
import { beatFact, lastBeat } from "../../keeper/beat.ts";
import { blocking, MAX_REASON, standingHolds } from "../../keeper/hold.ts";
import { secondsWords, withLandedToLive } from "../../keeper/landed.ts";
import { listKeeperReleases, type Stage } from "../../keeper/record.ts";
@@ -21,11 +24,13 @@
const STAGE: Record<Stage, { label: string; tone: Tone }> = {
};
/** `mayHold` is `mayRollBack`: the section then offers the owner's hold and its lift. */
export function keeperPanel(db: Database, mayHold = false): Html | undefined {
export function keeperPanel(db: Database, mayHold = false, now: Date = new Date()): Html | undefined {
  const releases = withLandedToLive(db, listKeeperReleases(db));
  const holds = standingHolds(db);
  const blocked = blocking(holds);
  if (releases.length === 0 && blocked === undefined && !mayHold) return undefined;
  const beat = lastBeat(db);
  const ahead = aheadLine(db, now);
  if (releases.length === 0 && blocked === undefined && beat === null && ahead === undefined && !mayHold) return undefined;
  const rows = releases.map((r, i) => {
    const s = STAGE[r.stage];
    return html`<tr>
@@ -43,6 +48,10 @@
export function keeperPanel(db: Database, mayHold = false): Html | undefined {
  return html`<section class="settings-block" id="keeper">
    <h2>Release keeper</h2>
    <p class="small ${blocked === undefined ? "muted" : ""}" id="keeper-blocked">${blocked ?? "Nothing holds the next cut."}</p>
    <dl class="status-rows" id="keeper-facts">
      <div class="status-row"><dt>Release tick</dt><dd id="keeper-heartbeat">${beatFact(beat, now)}</dd></div>
      
        sentinelcrew
        1d ago
        outdated
      
      Principles 1 and 7: a third stacked sentence. Make the heartbeat and ahead facts dl rows, not more paragraphs.
    
      ${when(ahead !== undefined, () => html`<div class="status-row"><dt>Main vs production</dt><dd id="keeper-ahead">${ahead}</dd></div>`)}
    </dl>
    ${when(rows.length > 0, () => html`<div class="table-scroll" role="region" tabindex="0" aria-label="Keeper releases"><table class="table" data-sticky-lead>
      <thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th></tr></thead>
      <tbody>${rows}</tbody>
@@ -0,0 +1,203 @@
/** The lane has stopped (#11507): ten open pulls, 91 minutes without a merge and one repeated failure line raise exactly one alarm; a merge clears it. */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRepo, getRepo } from "../src/core/repos.ts";
import { createUser } from "../src/core/users.ts";
import { getRun } from "../src/core/runs.ts";
import { ownLandingStopped } from "../src/checks/gate_stopped.ts";
import { hunt } from "../src/crew/sentinel.ts";
import { firedAlarms } from "../src/flow/alarms.ts";
import { forgeFailed, laneAlarm, laneMinutes, maskLine } from "../src/flow/alarm_lane.ts";
import { id } from "../src/kernel/id.ts";
const NOW = new Date("2026-11-12T17:11:00.000Z");
const ago = (m: number) => new Date(NOW.getTime() - m * 60_000).toISOString();
const LINE = "stopped: used 16.0 cores for 30s, over its limit of 16";
let root: string;
let app: App;
let repoId: string;
let userId: string;
let n = 0;
const saved = { sentinel: process.env["FORGE_SENTINEL_REPO"], commit: process.env["FORGE_COMMIT"] };
beforeAll(async () => {
  process.env["FORGE_SENTINEL_REPO"] = "trevato/forge";
  process.env["FORGE_COMMIT"] = "abcdef1234567890abcdef1234567890abcdef12";
  root = mkdtempSync(join(tmpdir(), "forge-lane-"));
  app = createApp({ dataDir: root, port: 0 });
  const u = await createUser(app.db, { username: "trevato", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true });
  if (!u.ok) throw new Error(u.error);
  userId = u.value.id;
  const repo = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "forge", visibility: "public", description: "f" });
  if (!repo.ok) throw new Error(repo.error);
  repoId = repo.value.id;
});
afterAll(() => {
  app.close();
  rmSync(root, { recursive: true, force: true });
  for (const [k, v] of [["FORGE_SENTINEL_REPO", saved.sentinel], ["FORGE_COMMIT", saved.commit]] as const)
    if (v === undefined) delete process.env[k];
    else process.env[k] = v;
});
function pull(state: "open" | "closed", mergedMinAgo?: number): string {
  const issueId = id();
  n++;
  app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, ?5, ?6)`).run(issueId, repoId, 2000 + n, userId, state, ago(600));
  app.db
    .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, 'main', ?2, '', '', ?3, ?4, ?5)`)
    .run(issueId, `b/${n}`, repoId, mergedMinAgo === undefined ? null : ago(mergedMinAgo), mergedMinAgo === undefined ? "" : "9".repeat(40));
  return issueId;
}
function failed(issueId: string, minAgo: number, summary = LINE, detail = "{}"): string {
  const runId = `${String(++n).padStart(4, "0")}${id()}`;
  app.db
    .query(`INSERT INTO run (id, repo_id, issue_id, kind, env, state, summary, check_detail, commit_oid, created_at, finished_at) VALUES (?1, ?2, ?3, 'check', 'pull-1', 'failed', ?4, ?5, 'c', ?6, ?6)`)
    .run(runId, repoId, issueId, summary, detail, ago(minAgo));
  return runId;
}
const reset = () => {
  app.db.query(`DELETE FROM run`).run();
  app.db.query(`DELETE FROM pull`).run();
  app.db.query(`DELETE FROM issue`).run();
};
const lane = () => firedAlarms(app.db, NOW, { dataDir: null }).filter((a) => a.key === "lane_stopped");
/** Ten open pulls and a last merge `since` minutes ago; one failed check apiece after it. */
function stalled(since: number): string[] {
  reset();
  pull("closed", since);
  const pulls = Array.from({ length: 10 }, () => pull("open"));
  pulls.forEach((p, i) => {
    failed(p, 75 + i, LINE.replace("16.0", `1${i}.0`));
  });
  return pulls;
}
describe("the lane has stopped", () => {
  test("ten open pulls, 91 minutes without a merge and one repeated line raise exactly one alarm with that line", () => {
    stalled(91);
    const a = lane();
    expect(a).toHaveLength(1);
    expect(a[0]!.says).toContain("Nothing has merged for 1 h 31 min");
    expect(a[0]!.says).toContain("10 pulls open");
    expect(a[0]!.says).toContain("10 checks failed");
    expect(a[0]!.since).toBe(ago(91));
    // The commonest line, verbatim as the forge holds it (the newest instance of the masked group).
    expect(laneAlarm(app.db, NOW).line).toBe(LINE.replace("16.0", "10.0"));
  });
  test("the line is quoted as recorded, with its count in the span", () => {
    reset();
    pull("closed", 120);
    const p = pull("open");
    for (let i = 0; i < 3; i++) failed(p, 100 - i);
    failed(p, 50, "check failed: install broke");
    const a = laneAlarm(app.db, NOW);
    expect(a.fired).toBe(true);
    expect(a.says).toContain(`4 checks failed in that span, 3 of them with: ${LINE}`);
  });
  test("89 minutes is quiet, and so is a setting that moves the limit", () => {
    stalled(89);
    expect(lane()).toHaveLength(0);
    app.db.query(`INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('flow.lane_minutes', '60', 't', 'now')`).run();
    expect(laneMinutes(app.db)).toBe(60);
    expect(lane()).toHaveLength(1);
    app.db.query(`UPDATE forge_setting SET value = 'abc' WHERE key = 'flow.lane_minutes'`).run();
    expect(laneMinutes(app.db)).toBe(90);
    app.db.query(`DELETE FROM forge_setting WHERE key = 'flow.lane_minutes'`).run();
  });
  test("nothing open or ready, or the crew at its ceiling, is quiet", () => {
    reset();
    pull("closed", 300);
    expect(lane()).toHaveLength(0);
    pull("open");
    expect(firedAlarms(app.db, NOW, { dataDir: null, ceilingUsd: null }).filter((a) => a.key === "lane_stopped")).toHaveLength(1);
    app.db
      .query(`INSERT INTO run (id, repo_id, kind, state, cost_usd, created_at, summary, check_detail) VALUES (?1, ?2, 'agent', 'ok', 50, ?3, '', '')`)
      .run(id(), repoId, new Date(NOW.getTime() - 60_000).toISOString());
    expect(firedAlarms(app.db, NOW, { dataDir: null, ceilingUsd: 20 }).filter((a) => a.key === "lane_stopped")).toHaveLength(0);
  });
  test("a merge clears it", () => {
    stalled(91);
    expect(lane()).toHaveLength(1);
    pull("closed", 1);
    expect(lane()).toHaveLength(0);
  });
  test("it reaches the owner's ask and one Sentinel issue, which closes when a pull merges", () => {
    stalled(91);
    const owner = app.db.query<{ id: string }, []>(`SELECT id FROM user WHERE username = 'trevato'`).get()!;
    expect(owner.id).toBe(userId);
    hunt(app, "trevato/forge", NOW.getTime());
    hunt(app, "trevato/forge", NOW.getTime() + 60_000);
    const open = () => app.db.query<{ body: string }, []>(`SELECT body FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: %lane has stopped%'`).all();
    expect(open()).toHaveLength(1);
    expect(open()[0]!.body).toContain("stopped: used");
    expect(open()[0]!.body).toContain("checks failed since the last merge");
    pull("closed", 1);
    hunt(app, "trevato/forge", NOW.getTime() + 120_000);
    expect(open()).toHaveLength(0);
  });
  test("a sweep reads the forge's lines once: the memo answers the second pull", () => {
    reset();
    pull("closed", 10);
    const runs = Array.from({ length: 5 }, (_, i) => failed(pull("open"), 5 + i, LINE.replace("16.0", `${i + 11}.5`)));
    const memo = new Map<string, ReadonlySet<string>>();
    expect(forgeFailed(app.db, getRun(app.db, runs[0]!), NOW.getTime(), memo)).toBe(true);
    expect(memo.get(repoId)?.size).toBe(1);
    app.db.query(`DELETE FROM run WHERE id != ?1`).run(runs[1]!);
    expect(forgeFailed(app.db, getRun(app.db, runs[1]!), NOW.getTime(), memo)).toBe(true);
    expect(forgeFailed(app.db, getRun(app.db, runs[1]!), NOW.getTime())).toBe(false);
  });
  test("land_past_gate stays shut: a merge drought is not a red gate", () => {
    stalled(91);
    expect(laneAlarm(app.db, NOW).fired).toBe(true);
    const repo = getRepo(app.db, "trevato", "forge")!;
    expect(ownLandingStopped(app.db, repo)).toBeUndefined();
  });
});
describe("one cause failing many pulls", () => {
  test("five pulls in an hour on one masked line are the forge's: alarm names it, the check is the forge's", () => {
    reset();
    pull("closed", 10);
    const pulls = Array.from({ length: 5 }, () => pull("open"));
    const runs = pulls.map((p, i) => failed(p, 5 + i, LINE.replace("16.0", `${i + 11}.5`)));
    const a = lane();
    expect(a).toHaveLength(1);
    expect(a[0]!.says).toContain("the forge's failure, not the builders'");
    expect(a[0]!.says).toContain("5 pulls");
    expect(laneAlarm(app.db, NOW).forge).toBe(true);
    expect(forgeFailed(app.db, getRun(app.db, runs[0]!), NOW.getTime())).toBe(true);
  });
  test("four pulls, or five over two hours, or a named test, are the pull's own", () => {
    reset();
    pull("closed", 10);
    const pulls = Array.from({ length: 5 }, () => pull("open"));
    const runs = pulls.slice(0, 4).map((p, i) => failed(p, 5 + i));
    expect(lane()).toHaveLength(0);
    expect(forgeFailed(app.db, getRun(app.db, runs[0]!), NOW.getTime())).toBe(false);
    failed(pulls[4]!, 130);
    expect(forgeFailed(app.db, getRun(app.db, runs[0]!), NOW.getTime())).toBe(false);
    reset();
    pull("closed", 10);
    const named = Array.from({ length: 5 }, () => pull("open")).map((p, i) => failed(p, 5 + i, "1 test failed", JSON.stringify({ names: ["a > b"] })));
    expect(lane()).toHaveLength(0);
    expect(forgeFailed(app.db, getRun(app.db, named[0]!), NOW.getTime())).toBe(false);
  });
  test("numbers are masked", () => {
    expect(maskLine(LINE)).toBe("stopped: used # cores for #s, over its limit of #");
  });
});
@@ -0,0 +1,313 @@
/** Main green and ahead of production (#11507): one alarm past 90 min with the facts, none at 89, closes on catch-up; the heartbeat shows. */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRepo } from "../src/core/repos.ts";
import { createUser, type User } from "../src/core/users.ts";
import { addComment } from "../src/actions/issues.ts";
import { getIssueById } from "../src/core/issues.ts";
import { getRepo } from "../src/core/repos.ts";
import { hunt } from "../src/crew/sentinel.ts";
import { firedAlarms } from "../src/flow/alarms.ts";
import { aheadMinutes, mainAheadAlarm } from "../src/flow/alarm_main_ahead.ts";
import { lastBeat, recordBeat, stepWithBeat } from "../src/keeper/beat.ts";
import { keeperPanel } from "../src/web/pages/keeper-panel.ts";
import { keeperAnswer } from "../src/keeper/landed.ts";
import { tickStep } from "../src/keeper/keeper.ts";
import { keeperForge } from "./helpers/keeper.ts";
import { id } from "../src/kernel/id.ts";
import { forgetLive, keepLive } from "../src/release/landings.ts";
import { alarmLines } from "../src/crew/notify.ts";
import { homeTruth } from "../src/system/home_truth.ts";
import { Session } from "../src/tui/forge.ts";
import { keeperScreen, readKeeper } from "../src/tui/screens/keeper.ts";
import { createForgeApp } from "../src/tui/forge.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { settleApp } from "./helpers/tui.ts";
import { flowFor } from "../src/system/telemetry.ts";
import { flowPanel } from "../src/web/pages/flow-panel.ts";
const NOW = new Date("2026-11-11T12:00:00.000Z");
const ago = (m: number) => new Date(NOW.getTime() - m * 60_000).toISOString();
const PROD = "abcdef1234567890abcdef1234567890abcdef12";
const HEAD = "1111111111111111111111111111111111111111";
let root: string;
let app: App;
let repoId: string;
let branch: string;
let userId: string;
let owner: User;
let n = 0;
const saved = { sentinel: process.env["FORGE_SENTINEL_REPO"], commit: process.env["FORGE_COMMIT"], version: process.env["FORGE_VERSION"] };
beforeAll(async () => {
  process.env["FORGE_SENTINEL_REPO"] = "trevato/forge";
  process.env["FORGE_COMMIT"] = PROD;
  process.env["FORGE_VERSION"] = "0.7.535";
  root = mkdtempSync(join(tmpdir(), "forge-main-ahead-"));
  app = createApp({ dataDir: root, port: 0 });
  const u = await createUser(app.db, { username: "trevato", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true });
  if (!u.ok) throw new Error(u.error);
  userId = u.value.id;
  owner = u.value;
  const repo = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "forge", visibility: "public", description: "f" });
  if (!repo.ok) throw new Error(repo.error);
  repoId = repo.value.id;
  branch = repo.value.defaultBranch;
});
afterAll(() => {
  app.close();
  rmSync(root, { recursive: true, force: true });
  for (const [k, v] of [["FORGE_SENTINEL_REPO", saved.sentinel], ["FORGE_COMMIT", saved.commit], ["FORGE_VERSION", saved.version]] as const)
    if (v === undefined) delete process.env[k];
    else process.env[k] = v;
  forgetLive();
});
function check(state: "ok" | "failed", oid: string): void {
  app.db
    .query(`INSERT INTO run (id, repo_id, kind, env, state, summary, check_detail, commit_oid, created_at, finished_at) VALUES (?1, ?2, 'check', ?3, ?4, '', '{}', ?5, ?6, ?6)`)
    .run(`${String(++n).padStart(4, "0")}${id()}`, repoId, branch, state, oid, ago(1));
}
function merge(minAgo: number, oid: string): void {
  const issueId = id();
  n++;
  app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, 'closed', ?5)`).run(issueId, repoId, 1000 + n, userId, ago(minAgo));
  app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, ?2, ?3, '', '', ?4, ?5, ?6)`).run(issueId, branch, `b/${n}`, repoId, ago(minAgo), oid);
}
const reset = () => {
  app.db.query(`DELETE FROM run`).run();
  app.db.query(`DELETE FROM pull`).run();
  keepLive("trevato/forge", [], NOW.getTime() - 600 * 60_000);
};
describe("main ahead alarm", () => {
  test("none at 89 minutes, one at 91 with the facts", () => {
    reset();
    check("ok", HEAD);
    merge(89, "2".repeat(40));
    expect(mainAheadAlarm(app.db, NOW).fired).toBe(false);
    expect(firedAlarms(app.db, NOW, { dataDir: null }).filter((a) => a.key === "main_ahead")).toHaveLength(0);
    reset();
    check("ok", HEAD);
    merge(91, "2".repeat(40));
    merge(30, "3".repeat(40));
    recordBeat(app.db, "skipped", "no cut: a hold stands", new Date(NOW.getTime() - 61 * 60_000));
    const fired = firedAlarms(app.db, NOW, { dataDir: null }).filter((a) => a.key === "main_ahead");
    expect(fired).toHaveLength(1);
    const says = fired[0]!.says;
    expect(says).toContain("0.7.535");
    expect(says).toContain("abcdef1");
    expect(says).toContain("2 commits and 2 pulls");
    expect(says).toContain("1 h 31 min");
    expect(says).toContain("1 h 1 min ago");
    expect(fired[0]!.since).toBe(ago(91));
  });
  test("a red head, or production on the head, does not fire", () => {
    reset();
    merge(200, "2".repeat(40));
    check("failed", HEAD);
    expect(mainAheadAlarm(app.db, NOW).fired).toBe(false);
    check("ok", PROD);
    expect(mainAheadAlarm(app.db, NOW).fired).toBe(false);
  });
  test("the limit is a setting", () => {
    reset();
    check("ok", HEAD);
    merge(50, "2".repeat(40));
    expect(mainAheadAlarm(app.db, NOW).fired).toBe(false);
    app.db.query(`INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('release.ahead_minutes', '45', 'test', 'now') ON CONFLICT (key) DO UPDATE SET value = '45'`).run();
    expect(mainAheadAlarm(app.db, NOW).fired).toBe(true);
    app.db.query(`DELETE FROM forge_setting WHERE key = 'release.ahead_minutes'`).run();
  });
  test("it reaches the owner's ask and one Sentinel issue, which closes when production catches up", () => {
    reset();
    check("ok", HEAD);
    merge(120, "2".repeat(40));
    expect(owner.isAdmin).toBe(true);
    const lines = alarmLines(app, owner, NOW.getTime());
    expect(lines.some((l) => l.id.startsWith("alarm:main_ahead@"))).toBe(true);
    hunt(app, "trevato/forge", NOW.getTime());
    hunt(app, "trevato/forge", NOW.getTime() + 60_000);
    const open = () => app.db.query<{ title: string }, []>(`SELECT title FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: Main is green and ahead%'`).all();
    expect(open()).toHaveLength(1);
    // Production catches up: it runs the head and the pull is in the running commit.
    keepLive("trevato/forge", ["2".repeat(40)], NOW.getTime());
    hunt(app, "trevato/forge", NOW.getTime() + 120_000);
    expect(open()).toHaveLength(0);
  });
});
describe("heartbeat", () => {
  test("release_keeper shows the last heartbeat and its reason, and says when it is quiet", () => {
    app.db.query(`DELETE FROM keeper_beat`).run();
    expect(keeperAnswer(app.db, NOW).heartbeat).toBeNull();
    recordBeat(app.db, "skipped", "not cut: a hold stands", new Date(NOW.getTime() - 5 * 60_000));
    const h = keeperAnswer(app.db, NOW).heartbeat!;
    expect(h).toMatchObject({ outcome: "skipped", reason: "not cut: a hold stands", silent: false });
    recordBeat(app.db, "failed", "tag refused", new Date(NOW.getTime() - 61 * 60_000));
    const quiet = keeperAnswer(app.db, NOW).heartbeat!;
    expect(quiet.silent).toBe(true);
    expect(quiet.says).toContain("No heartbeat from the release tick for 1 h 1 min");
    expect(lastBeat(app.db)!.reason).toBe("tag refused");
  });
});
describe("home telemetry", () => {
  test("live names the commit production runs, not main's newest deploy", () => {
    reset();
    app.db.query(`INSERT INTO run (id, repo_id, kind, env, state, commit_oid, created_at, serving_at) VALUES (?1, ?2, 'deploy', 'production', 'ok', ?3, ?4, ?4)`).run(id(), repoId, HEAD, ago(5));
    const t = homeTruth(app, owner, NOW);
    expect(t.live).toEqual({ sha: PROD, version: "0.7.535" });
  });
  test("with the running commit unknown, live is null", () => {
    const commit = process.env["FORGE_COMMIT"];
    delete process.env["FORGE_COMMIT"];
    try {
      expect(homeTruth(app, owner, NOW).live).toBeNull();
    } finally {
      process.env["FORGE_COMMIT"] = commit;
    }
  });
});
describe("the main-ahead issue", () => {
  const open = () => app.db.query<{ id: string; body: string }, []>(`SELECT id, body FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: Main is green and ahead%'`).all();
  test("keeps the tick's reason out of the issue, and stays open once a person spoke on it", async () => {
    reset();
    check("ok", HEAD);
    merge(120, "4".repeat(40));
    recordBeat(app.db, "skipped", "private words from the tick", new Date(NOW.getTime() - 5 * 60_000));
    hunt(app, "trevato/forge", NOW.getTime() + 600_000);
    const filed = open();
    expect(filed).toHaveLength(1);
    expect(filed[0]!.body).toContain("skipped");
    expect(filed[0]!.body).not.toContain("private words");
    const repo = getRepo(app.db, "trevato", "forge")!;
    const said = addComment(app, owner, repo, getIssueById(app.db, filed[0]!.id)!, "Looking at the tick now.");
    expect(said.ok).toBe(true);
    keepLive("trevato/forge", ["4".repeat(40)], NOW.getTime());
    hunt(app, "trevato/forge", NOW.getTime() + 720_000);
    expect(open()).toHaveLength(1);
  });
});
describe("the limit setting", () => {
  const set = (v: string) =>
    app.db.query(`INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('release.ahead_minutes', ?1, 'test', 'now') ON CONFLICT (key) DO UPDATE SET value = ?1`).run(v);
  test("a value out of range or not a whole number is 90", () => {
    for (const bad of ["0", "20000", "abc", "1.5", ""]) {
      set(bad);
      expect(aheadMinutes(app.db)).toBe(90);
    }
    set("10080");
    expect(aheadMinutes(app.db)).toBe(10080);
    app.db.query(`DELETE FROM forge_setting WHERE key = 'release.ahead_minutes'`).run();
    expect(aheadMinutes(app.db)).toBe(90);
  });
});
describe("the tick's beat", () => {
  test("a step's outcome is recorded as the keeper said it, whatever its words", async () => {
    const at = new Date(NOW.getTime() - 60_000);
    const line = await stepWithBeat(app.db, async () => ({ line: "not cut: a failed hold stands", outcome: "skipped" }), () => at);
    expect(line).toBe("not cut: a failed hold stands");
    expect(lastBeat(app.db)).toEqual({ at: at.toISOString(), outcome: "skipped", reason: "not cut: a failed hold stands" });
  });
  test("a step that throws is recorded as failed, and the error goes on", async () => {
    const step = stepWithBeat(app.db, async () => { throw new Error("git is gone"); }, () => NOW);
    await expect(step).rejects.toThrow("git is gone");
    expect(lastBeat(app.db)).toEqual({ at: NOW.toISOString(), outcome: "failed", reason: "step failed: git is gone" });
  });
});
describe("a beat that cannot be written", () => {
  test("a good tick is still reported good", async () => {
    const broken = { query: () => { throw new Error("disk full"); } } as never;
    const line = await stepWithBeat(broken, async () => ({ line: "cut 0.7.9", outcome: "cut" }), () => NOW);
    expect(line).toBe("cut 0.7.9");
  });
});
describe("both doors", () => {
  test("/releases and the terminal's keeper screen show the heartbeat and that main is ahead", () => {
    reset();
    check("ok", HEAD);
    merge(120, "5".repeat(40));
    recordBeat(app.db, "skipped", "<b>hold</b> stands", new Date(NOW.getTime() - 3 * 60_000));
    const page = String(keeperPanel(app.db, false, NOW));
    expect(page).toContain('id="keeper-heartbeat"');
    expect(page).toContain("skipped, 3 min ago: ");
    expect(page).toContain('<dt>Release tick</dt>');
    expect(page).toContain("&lt;b&gt;hold&lt;/b&gt;");
    expect(page).not.toContain("<b>hold");
    expect(page).toMatch(/<dt>Main vs production<\/dt><dd id="keeper-ahead">Main is green and 1 commit and 1 pull ahead/);
    const read = readKeeper(new Session(app, { sshUsername: owner.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: owner }), NOW)!;
    expect(read.heartbeat).toContain("3 min ago (skipped)");
    expect(read.ahead).toContain("for 2 h 0 min");
  });
  test("with main on production, no ahead line, and the heartbeat says when it is quiet", () => {
    reset();
    check("ok", PROD);
    recordBeat(app.db, "cut", "cut 0.7.536", new Date(NOW.getTime() - 90 * 60_000));
    const read = readKeeper(new Session(app, { sshUsername: owner.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: owner }), NOW)!;
    expect(read.ahead).toBeUndefined();
    expect(read.heartbeat).toContain("No heartbeat from the release tick for 1 h 30 min");
    expect(String(keeperPanel(app.db, false, NOW))).toContain("cut, 1 h 30 min ago (silent): cut 0.7.536");
    expect(String(keeperPanel(app.db, false, NOW))).not.toContain("keeper-ahead");
  });
});
describe("the terminal screen and /flow", () => {
  test("the keeper screen draws the heartbeat and the ahead line", async () => {
    reset();
    check("ok", HEAD);
    merge(120, "5".repeat(40));
    recordBeat(app.db, "failed", "tag refused", new Date());
    const id = { sshUsername: owner.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: owner };
    const { app: tui, session } = createForgeApp(app, id, { terminal: headlessTerminal(160, 20), exitHooks: false });
    tui.start();
    tui.ctx.push(keeperScreen(session, tui.ctx));
    await settleApp(tui);
    const text = tui.screen.toText().replace(/\s+/g, " ");
    tui.stop?.();
    expect(text).toContain("The release tick last reported 0 min ago (failed): tag refused");
    expect(text).toContain("Main is green and 1 commit and 1 pull ahead");
  });
  test("/flow shows the heartbeat with its reason to the owner and never to an agent", () => {
    reset();
    recordBeat(app.db, "skipped", "a private reason", new Date(NOW.getTime() - 5 * 60_000));
    expect(flowFor(app, { kind: "owner" }, NOW).heartbeat?.reason).toBe("a private reason");
    expect(flowFor(app, { kind: "agent" }, NOW).heartbeat).toBeNull();
    expect(JSON.stringify(flowFor(app, { kind: "agent" }, NOW))).not.toContain("a private reason");
    const url = new URL("http://forge.test/flow");
    expect(String(flowPanel(app, { kind: "owner" }, url, "/flow"))).toContain("skipped, ");
    expect(String(flowPanel(app, { kind: "agent" }, url, "/flow"))).not.toContain("a private reason");
  });
});
describe("the keeper's own outcome", () => {
  test("a cut, a failed verify, then a skip because the failed hold stands", async () => {
    const f = await keeperForge();
    expect((await tickStep(f.ports)).outcome).toBe("cut");
    f.world.live = "0.7.284";
    f.world.healthz = (n) => (n >= 5 ? { status: 503, ms: 2 } : { status: 200, ms: 2 });
    const failed = await tickStep(f.ports);
    expect(failed.outcome).toBe("failed");
    expect(failed.line).toContain("failed its verify");
    const held = await tickStep(f.ports);
    expect(held.line).toContain("failed");
    expect(held.outcome).toBe("skipped");
  });
});
@@ -227,3 +227,26 @@
test("the flow report counts stale reds and the keeper names them as a cause", (
  const found = pickCause({ repos: [{ repoId: "r", slug: "a/b", startable: 1, running: 1, slots: 4, ceilingUsd: 10, atCeiling: false, noHeadroom: false, resetsAt: null }], red: 0, staleReds: STALE_REDS, stale: 0 });
  expect(found?.cause).toBe("stale_reds");
});
test("one failure line on five pulls within the hour is the forge's (#11507): past their tries, none is parked", async () => {
  app.db.query(`UPDATE issue SET state = 'closed' WHERE state = 'open' AND kind = 'pull'`).run();
  const made: Awaited<ReturnType<typeof pullBy>>[] = [];
  for (let i = 0; i < 5; i++) {
    const p = await pullBy(mason);
    const red = finished(p, "check", "failed", 5000 + i);
    app.db.query("UPDATE run SET summary = ?2 WHERE id = ?1").run(red.id, `check failed: stopped: used 1${i}.0 cores for 30s, over its limit of 16`);
    finished(p, "agent", "ok", 5010 + i, "check_failed");
    finished(p, "agent", "ok", 5020 + i, "check_failed");
    made.push(p);
  }
  const moves = await sweepStaleReds(app, repo, at(5060), 0);
  expect(moves.filter((m) => made.some((p) => p.pull.number === m.pull))).toEqual([]);
  expect(made.every((p) => open(p.pull.id))).toBe(true);
  // A line on one pull only is the pull's: past its tries, it is parked.
  const lone = await pullBy(mason);
  finished(lone, "check", "failed", 5025, "", "engine > lone");
  finished(lone, "agent", "ok", 5026, "check_failed");
  finished(lone, "agent", "ok", 5027, "check_failed");
  const later = await sweepStaleReds(app, repo, at(5060), 0);
  expect(later).toEqual([{ pull: lone.pull.number, did: "parked" }]);
});
@@ -56,6 +56,6 @@
describe("the release keeper screen", () => {
  test("with nothing cut, it says nothing holds the next cut and that no releases are kept yet", async () => {
    const f = await keeperForge();
    const read = readKeeper(new Session(f.app, identity(f.user)));
    expect(read).toEqual({ blocked: "Nothing holds the next cut.", releases: [] });
    expect(read).toEqual({ blocked: "Nothing holds the next cut.", heartbeat: "The release tick has never reported a heartbeat.", releases: [] });
  });
});
		@@ -0,0 +1,10 @@
	1	# The lane has stopped (#11507)
	2
	3	On 2026-11-12 nothing merged for 12 h 51 min and nothing said so: every check was being stopped for CPU, and main was not ahead of production, so the main-ahead alarm stayed quiet. The flow alarm `lane_stopped` (`src/flow/alarm_lane.ts`) fires for the forge's own repository in two states, as one alarm:
	4
	5	- The lane has stopped: pulls are open (or work is ready), the crew is not at its ceiling, and no pull has merged for 90 minutes. Change the limit with the `flow.lane_minutes` row of `forge_setting` (a whole number, 1 to 10080; anything else is 90). It names how long, how many checks failed since the last merge and the commonest failure line verbatim.
	6	- One cause failing many pulls: the same failure line (numbers masked) failed checks on five different pulls within an hour and since the last merge. The line is the first line of a check's summary, for a check that named no failing test; a named test is `src/checks/shared_red.ts`'s. /flow says it is the forge's failure; `check_failed` is not sent to builders for it (`waitOnMain` in `src/crew/dispatch.ts`) and the stale-red sweep neither parks nor reworks its pulls, only checks them again.
	7
	8	It is an ask for the owner and one Sentinel issue (`src/crew/sentinel_lane.ts`), closed at the next hunt after a pull merges and the cause is gone, unless a person commented on it. `land_past_gate` does not open for it: the break-glass rests on red gates only (`landingStopped` in `src/checks/gate_stopped.ts`), since a merge drought with pulls waiting on review is not a red gate.
	9
	10	Left out: the Sentinel issue's evidence carries the failure count, so it is commented again as that count moves.
		@@ -0,0 +1,13 @@
	1	# Release heartbeat and the main-ahead alarm (#11507)
	2
	3	The release tick records a heartbeat each time it runs (`src/keeper/beat.ts`, table `keeper_beat`): when, whether it cut, skipped or failed, and why. The outcome is the keeper's own (`tickStep` in `src/keeper/keeper.ts` says it where it decides), never read back from the line's words; a tick that throws is recorded as failed (`stepWithBeat`). None for 60 minutes is said plainly ("No heartbeat from the release tick for …").
	4
	5	Where to see it:
	6
	7	- `/releases`, in the release keeper section: the last heartbeat (`#keeper-heartbeat`) and, while main is green and ahead of production, how far and for how long (`#keeper-ahead`).
	8	- The terminal: the palette's Release keeper screen, gated as /releases is, shows the same two lines under what holds the next cut.
	9	- `release_keeper` (MCP) and `GET /api/v1/keeper`: `heartbeat` (at, outcome, reason, silent, says).
	10
	11	The flow alarm `main_ahead` (`src/flow/alarm_main_ahead.ts`) fires when main's last finished check passed, its head is not the commit production runs, and main has been ahead for more than 90 minutes. Change the limit with the `release.ahead_minutes` row of `forge_setting` (a whole number, 1 to 10080; anything else is 90). It names production's version and commit, the commits and pulls ahead, how long, and when the tick was last heard from. It is an ask for the owner (home) and one Sentinel issue (`src/crew/sentinel_main_ahead.ts`), which closes itself at the next hunt after production catches up, unless a person commented on it. The issue names the tick's last outcome and time, never its reason: the issue may be public, and the reason is shown only to readers of /releases.
	12
	13	`/api/v1/telemetry/home`: `live` is the version and commit this process runs (null when unstamped); `main` is the head of the forge's own default branch and `behind` what main holds that live does not (#11676).
		@@ -66,7 +66,10 @@ function oneCause(gates: readonly { reason: string \| null; tests: string }[]): b
66	66	export function ownLandingStopped(db: Database, repo: { readonly id: string }): Stopped \| undefined {
67	67	const [owner, name] = ownRepoPath().split("/");
68	68	const own = owner && name ? getRepo(db, owner, name) : undefined;
69		return own !== undefined && own.id === repo.id ? landingStopped(db, own.id) : undefined;
	69	if (own === undefined \|\| own.id !== repo.id) return undefined;
	70	// Gate evidence only: the lane alarm (#11507) is a heuristic over merges and
	71	// run rows, and a merge drought with pulls waiting on review is not a red gate.
	72	return landingStopped(db, own.id);
70	73	}
71	74
72	75	/** The one sentence home and the pipeline carry. */
		sentinelcrew 1d ago outdated Blocking: do not let the lane alarm make landing past the gate available. A merge drought with open pulls is not a red gate. Return only `landingStopped` here, or add an explicit, tested condition based on failing gates or checks.
		@@ -0,0 +1,1 @@
	1	a57e110db188bf5db091597675159f303ad60ac496f0c841a66d62e09a6df833
		@@ -0,0 +1,10 @@
	1	// The release tick's last heartbeat (#11507): one row, replaced each time the tick
	2	// reports in — when, what it did (cut, skipped, failed) and why. Additive.
	3	export default `
	4	CREATE TABLE keeper_beat (
	5	id INTEGER PRIMARY KEY CHECK (id = 1),
	6	at TEXT NOT NULL,
	7	outcome TEXT NOT NULL CHECK (outcome IN ('cut', 'skipped', 'failed')),
	8	reason TEXT NOT NULL
	9	);
	10	`;
		@@ -48,6 +48,7 @@ import { appendRunLog } from "../runs/log.ts";
48	48	import { noteHeld, type HoldCause } from "../runs/held.ts";
49	49	import { decideAndRecord } from "./ledger.ts";
50	50	import { bugsFirst } from "../flow/bugs.ts";
	51	import { forgeFailed } from "../flow/alarm_lane.ts";
51	52	import { byDials } from "./team_share.ts";
52	53	import { peopleFirst } from "./people_first.ts";
53	54	import { inMaintenance } from "../maintenance/state.ts";
		@@ -1099,6 +1100,11 @@ function waitOnMain(
1099	1100	});
1100	1101	return true;
1101	1102	}
	1103	// One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent, no pull parked.
	1104	if (forgeFailed(app.db, run, Date.now())) {
	1105	log.info("check failed on a line five pulls share; the forge's failure, not the pull's", { issue: issue.number });
	1106	return true;
	1107	}
1102	1108	const main = sharedWithMain(redMain(app.db, repo.id), run);
1103	1109	if (!main && !sharedWithPulls(app.db, run)) return false;
1104	1110	if (typeof commitOid === "string" && typeof env === "string") {
		@@ -0,0 +1,13 @@
	1	/** Whether a person, not an agent, has commented on an issue: then they decide, not the sentinel. */
	2
	3	import type { Database } from "bun:sqlite";
	4
	5	export function personSpoke(db: Database, issueId: string): boolean {
	6	const spoke = db
	7	.query<{ n: number }, [string]>(
	8	`SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id
	9	WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`,
	10	)
	11	.get(issueId);
	12	return spoke !== null && spoke.n > 0;
	13	}
		@@ -47,6 +47,10 @@ import {
47	47	} from "../checks/shared_red.ts";
48	48	import { readRunLogTail } from "../runs/log.ts";
49	49	import { clip } from "../text/clip.ts";
	50	import { forgeCeiling } from "../flow/ceiling.ts";
	51	import { personSpoke } from "./person_spoke.ts";
	52	import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts";
	53	import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts";
50	54	import type { FindingKind } from "../hunt/finding.ts";
51	55	import { addLabels, kindLabels } from "../hunt/label.ts";
52	56
		@@ -400,7 +404,7 @@ export function repeatedErrors(now: number): Finding[] {
400	404	* Each cause whose spans total more than ten minutes in the day ending at
401	405	* `now`, quoting its sentence. Shorter totals stay in the history.
402	406	*/
403		export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string): Finding[] {
	407	export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number \| null = null): Finding[] {
404	408	const from = now - DAY_MS;
405	409	const byCause = new Map<
406	410	string,
		@@ -479,7 +483,7 @@ export function ownCause(cause: string, forgeRepoId: string \| undefined): boolea
479	483	* at once, not when the day has added up. The finding shares its key with
480	484	* `degradedCauses`, so one cause is one issue.
481	485	*/
482		export function budgetBreaches(db: Database, now: number, forgeRepoId?: string, dataDir?: string): Finding[] {
	486	export function budgetBreaches(db: Database, now: number, forgeRepoId?: string, dataDir?: string, ceilingUsd: number \| null = null): Finding[] {
483	487	const out: Finding[] = [];
484	488	for (const s of spans(db, now - DAY_MS, now)) {
485	489	if (s.closedAt !== null \|\| !BREACH_CAUSE.test(s.cause) \|\| !ownCause(s.cause, forgeRepoId)) continue;
		@@ -569,8 +573,10 @@ function sharedLines(
569	573	}
570	574
571	575	/** Everything the sentinel reads, in one pass. */
572		export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string): Finding[] {
	576	export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number \| null = null): Finding[] {
573	577	const found = [
	578	...mainAhead(db, now),
	579	...laneStopped(db, now, ceilingUsd),
574	580	...budgetBreaches(db, now, forgeRepoId, dataDir),
575	581	...heldSlots(db, now),
576	582	...strayTriggers(db),
		@@ -770,17 +776,6 @@ function sinceFix(app: App, repo: Repo, actor: User, f: Finding, evidence: strin
770	776	return 1;
771	777	}
772	778
773		/** Whether a person, not an agent, has commented on the issue: then they decide, not the sentinel. */
774		function personSpoke(db: Database, issueId: string): boolean {
775		const spoke = db
776		.query<{ n: number }, [string]>(
777		`SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id
778		WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`,
779		)
780		.get(issueId);
781		return spoke !== null && spoke.n > 0;
782		}
783
784	779	/** The test a shared finding names, read back from its evidence. */
785	780	const testOf = (evidence: readonly string[]): string \| undefined =>
786	781	evidence.find((e) => e.startsWith("test: "))?.slice(6);
		@@ -1031,7 +1026,7 @@ export function hunt(
1031	1026	let commented = 0;
1032	1027	const [owner = "", name = ""] = repoPath.split("/");
1033	1028	const repo = getRepo(app.db, owner, name);
1034		const findings = find(app.db, now, repo?.id, app.config.dataDir);
	1029	const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config));
1035	1030	const actor = sentinelActor(app.db);
1036	1031	if (repo !== undefined && actor !== undefined) {
1037	1032	settleShared(app, repo, actor, now);
		@@ -1106,8 +1101,12 @@ export function hunt(
1106	1101	}
1107	1102	}
1108	1103	}
1109		if (repo !== undefined && actor !== undefined)
1110		closeHeld(app, repo, actor, new Set(findings.map((f) => f.key)), now);
	1104	if (repo !== undefined && actor !== undefined) {
	1105	const active = new Set(findings.map((f) => f.key));
	1106	closeHeld(app, repo, actor, active, now);
	1107	closeCaughtUp(app, repo, actor, active, now);
	1108	closeLaneMoving(app, repo, actor, active, now);
	1109	}
1111	1110	// A hunt with nowhere to file is recorded as such, never as a quiet one.
1112	1111	const unfiled = repo === undefined \|\| actor === undefined;
1113	1112	app.db
		@@ -0,0 +1,34 @@
	1	/**
	2	* The sentinel's half of the lane-stopped alarm (#11507): nothing merging for
	3	* the limit, or one cause failing many pulls, is one finding; its issue closes
	4	* at the next hunt after the lane moves, unless a person spoke on it.
	5	*/
	6
	7	import type { Database } from "bun:sqlite";
	8	import type { App } from "../app.ts";
	9	import type { Repo } from "../core/repos.ts";
	10	import type { User } from "../core/users.ts";
	11	import { laneAlarm } from "../flow/alarm_lane.ts";
	12	import { closeCaughtUp } from "./sentinel_main_ahead.ts";
	13	import type { Finding } from "./sentinel.ts";
	14
	15	export const LANE_KEY = "lane-stopped";
	16
	17	export function laneStopped(db: Database, now: number, ceilingUsd: number \| null): Finding[] {
	18	const a = laneAlarm(db, new Date(now), { ceilingUsd });
	19	if (!a.fired) return [];
	20	return [{
	21	key: LANE_KEY,
	22	title: a.forge === true ? "One failure is stopping many pulls: the forge's, not theirs" : "The lane has stopped: nothing has merged and nothing said so",
	23	evidence: a.facts ?? [a.says],
	24	pillar: "P0 Trust: the forge says when it is not well, and is well again",
	25	kind: "defect",
	26	lookFor: "A pull merged since, and no one failure line failing five pulls within an hour.",
	27	urgent: true,
	28	}];
	29	}
	30
	31	/** Close the open lane issue once the lane moves; `active` is this hunt's finding keys. */
	32	export function closeLaneMoving(app: App, repo: Repo, actor: User, active: ReadonlySet<string>, now: number): void {
	33	closeCaughtUp(app, repo, actor, active, now, LANE_KEY, "The lane moved: a pull merged and no one failure is stopping many");
	34	}
		@@ -0,0 +1,65 @@
	1	/**
	2	* The sentinel's half of the main-ahead alarm (#11507): main green and ahead
	3	* of production past the setting is one finding, and its issue closes itself
	4	* at the next hunt after production catches up — unless a person spoke on it,
	5	* then it is theirs to close.
	6	*/
	7
	8	import type { Database } from "bun:sqlite";
	9	import type { App } from "../app.ts";
	10	import { addComment, setIssueState } from "../actions/issues.ts";
	11	import { getIssueById } from "../core/issues.ts";
	12	import type { Repo } from "../core/repos.ts";
	13	import type { User } from "../core/users.ts";
	14	import { mainAheadAlarm } from "../flow/alarm_main_ahead.ts";
	15	import { createLog } from "../kernel/log.ts";
	16	import { personSpoke } from "./person_spoke.ts";
	17	import type { Finding } from "./sentinel.ts";
	18
	19	const log = createLog("sentinel");
	20
	21	export const MAIN_AHEAD_KEY = "main-ahead";
	22
	23	export function mainAhead(db: Database, now: number): Finding[] {
	24	const a = mainAheadAlarm(db, new Date(now));
	25	if (!a.fired) return [];
	26	return [{
	27	key: MAIN_AHEAD_KEY,
	28	title: "Main is green and ahead of production: the release tick has not cut",
	29	evidence: a.facts ?? [a.says],
	30	pillar: "P0 Trust: the forge says when it is not well, and is well again",
	31	kind: "defect",
	32	lookFor: "Production runs main's head, or main has been ahead of it for under the limit.",
	33	urgent: true,
	34	}];
	35	}
	36
	37	/** Close the open main-ahead issue once its finding is gone; `active` is this hunt's finding keys. */
	38	export function closeCaughtUp(
	39	app: App,
	40	repo: Repo,
	41	actor: User,
	42	active: ReadonlySet<string>,
	43	now: number,
	44	key: string = MAIN_AHEAD_KEY,
	45	cleared: string = "Production caught up with main",
	46	): void {
	47	if (active.has(key)) return;
	48	const open = app.db
	49	.query<{ issue_id: string }, [string]>(
	50	`SELECT t.issue_id FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id WHERE i.state = 'open' AND t.key = ?1`,
	51	)
	52	.all(key);
	53	for (const t of open) {
	54	try {
	55	if (personSpoke(app.db, t.issue_id)) continue;
	56	const issue = getIssueById(app.db, t.issue_id);
	57	if (issue === undefined \|\| issue.repoId !== repo.id) continue;
	58	const said = addComment(app, actor, repo, issue, `${cleared} at ${new Date(now).toISOString()}. Closing as done; if it falls behind again it is filed again.`);
	59	if (!said.ok) continue;
	60	setIssueState(app, actor, repo, issue, "closed", { reason: "done", system: true });
	61	} catch (error) {
	62	log.error("sentinel close failed", { key, error: String(error) });
	63	}
	64	}
	65	}