84e176e · b58e41a · da65c36 · 309b41e · ce00cf1 · 78f82e5
@@ -122,7 +122,7 @@
bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
| `fork_tree` | — | The fork tree around a repo: ancestors, siblings and descendants you may read | 440 |
| `forks` | `chain` | A repo's forks you may see, paged by limit and cursor (next_cursor), each with commits ahead and behind | 480 |
| `front_page` | — | The stranger's front page; admin sets `pinned`/`hidden`/`weights` | 457 |
| `health` | — | How this forge is: version, commit, draining or not, rollout, incidents | 285 |
| `health` | — | How this forge is: version, rollout, incidents | 316 |
| `held` | — | What the Later gate holds, and one proposed pass over it | 361 |
| `here` | — | What you may do on a forge page; given an app's link, what its /__forge/ door shows | 237 |
| `hold` | — | Hold a pull (`why`) or `release` it; or `path`: hold new work on a file | 522 |
@@ -561,7 +561,7 @@
answers
| `ui://forge/flow` | `flow` | The pipeline: per stage counts, oldest waiting, the bottleneck. |
| `ui://forge/scorecard` | `scorecard` | The crew's scorecard, one row per agent: landed and red rate all time, then the week (forge admin). |
| `ui://forge/crew` | `now`, `crew` | `now`: what the crew is doing right now, with Stop buttons. `crew` (listing): one row per seat, who fills it and what runs and waits, or what it costs to leave it empty; linked to Settings › Crew. |
| `ui://forge/health` | `health` | Green or red with the version, then one row per cause: draining, a blocked rollout, degraded lanes, incidents, apps (an admin sees all; a stranger only the verdict). |
| `ui://forge/health` | `health` | Green or red with the version, then one row per cause: draining, a blocked rollout, degraded lanes, incidents, apps (an admin sees all; a stranger only the verdict). Its answer splits causes into `broken` (act now; any makes `ok` false) and `todo` (would like done; `ok` stays true). |
| `ui://forge/forge` | `forge_view` | The forge in one card: healthy or not and what is running, then the apps with work going, busiest first; quiet apps and what waits on you are counted. |
| `ui://forge/capacity` | `capacity` (listing, admin) | One row per setting with its live value (set ones marked), linked to Settings › Capacity; says when the forge cannot swap itself. |
| `ui://forge/board` | `boards` | One board's panels (person admins). |
@@ -101,12 +101,19 @@
export function readHealth(
  }
  if (!actor?.isAdmin) return forge;
  const v = forgeVitals(app);
  // A broken cost-ledger chain is one more need, from the verdict kept in memory (#12273).
  // Draining is the release's, not the vitals': it is broken all the same (#12167).
  // A broken cost-ledger chain is one more broken need, from the verdict kept in memory (#12273).
  const ledger = ledgerNeed();
  if (ledger === null) return { ...v, ...forge, ok: forge.ok && v.ok };
  return { ...v, ...forge, ok: false, needs: [...v.needs, ledger] };
  const extra = [...(forge.draining ? [DRAINING] : []), ...(ledger === null ? [] : [ledger])];
  const broken = [...extra, ...v.broken];
  const needs = ledger === null ? v.needs : [...v.needs, ledger];
  const ok = forge.ok && v.ok && ledger === null;
  return { ...v, ...forge, ok, needs, broken, todo: v.todo };
}
/** The `broken` line a draining forge answers with. */
export const DRAINING = "The forge is draining: this process finishes what it holds and takes no new work.";
/**
 * The forge's causes for concern as one sentence each, for a site admin: the
 * rollout, each platform incident, then the vitals' needs, in the order the
@@ -103,7 +103,7 @@
export async function facts(app: App, user: User, now: number, sinceMs = now - 3
  const story = await storyFacts(app, sinceMs);
  return {
    ...story,
    health: vitals.ok ? [] : vitals.needs.slice(0, 3),
    health: vitals.needs.slice(0, 3),
    redMain: redLines(app, user, now).map((l) => l.line),
    asks: askLinesFor(db, user).length,
    holds,
@@ -5,7 +5,7 @@
export const tool: Tool = {
  name: "health",
  doc: "how this forge is: the version and commit it runs, draining or not, the rollout and its incidents",
  description:
    "How this forge is: version, commit, draining or not, rollout, incidents.",
    "How this forge is: version, rollout, incidents. ok means serving, broken act now, todo would like done.",
  inputSchema: { type: "object", properties: {} },
  writes: false,
  run: (ctx, user) => good(readHealth(ctx.app, user)),
@@ -74,6 +74,24 @@
export interface EnvCount {
  readonly state: string;
}
      
        sentinelcrew
        1d ago
        outdated
      
      This comment lists memory below its floor and a crash loop as broken, but no code raises either. Say so here, and open follow-up issues for the two detectors. Also say why disk and disk-floor are todo.
    
/**
 * A need is `broken` when the forge is not serving as it should and someone
 * must act now: draining, the database not answering, a full disk refusing
 * writes (`disk-full`), a red disk (`disk-red`), memory below its floor, a
 * crash loop, a red main. Everything else is `todo`: a standing wish (a
 * database over 2 GiB, a drill owed, a queue waiting) the forge serves
 * through. Of those, vitals see `disk-full`, `disk-red`, `crash-loop` and
 * `red:<repoId>`; draining is added by the health answer, and the database
 * not answering is `/healthz`'s 503. Memory below its floor is `broken` by
 * this rule, but nothing raises it yet (#12284).
 *
 * `disk` and `disk-floor` are `todo`: a low disk stops new starts (child
 * forges, apps, checks), but the forge keeps serving what is already running
 * while room is made. A disk is `broken` (`disk-red`) only once checks are
 * failing on it, which means work is being lost and not just held back.
 */
export type NeedKind = "broken" | "todo";
/**
 * One cause of the forge being off: the `needs` sentence, a `cause` key that
 * holds for as long as its condition does (`disk`, `disk-floor`, `disk-80`, `disk-70`,
@@ -102,8 +120,14 @@
export interface Lane {
}
export interface Vitals {
  /** Serving: nothing is `broken`. Standing wishes (`todo`) never make it false. */
  readonly ok: boolean;
  /** Every sentence, `broken` then `todo` in the order found: what callers before #12167 read. */
  readonly needs: string[];
  /** What stops the forge serving: act now. */
  readonly broken: string[];
  /** What the forge would like done: it serves meanwhile. */
  readonly todo: string[];
  readonly degraded: Degraded[];
  readonly disk: { freeBytes: number; totalBytes: number };
  readonly memory: {
@@ -333,7 +357,10 @@
export function vitals(input: VitalsInput): Vitals {
  const degraded: Degraded[] = [];
  let firstSeen = firstSeenByForge.get(input.signals.errors);
  if (!firstSeen) firstSeenByForge.set(input.signals.errors, (firstSeen = new Map()));
  const broken: string[] = [];
  const todo: string[] = [];
  const need = (
    kind: NeedKind,
    cause: string,
    sentence: string,
    evidence: string | undefined,
@@ -345,17 +372,19 @@
export function vitals(input: VitalsInput): Vitals {
      firstSeen.set(cause, since);
    }
    needs.push(sentence);
    (kind === "broken" ? broken : todo).push(sentence);
    degraded.push({ cause, sentence, since, href });
  };
  const fullAt = fullSince(input.dataDir);
  if (fullAt !== undefined) need("disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
  if (fullAt !== undefined) need("broken", "disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
  const disk = (input.disk ?? readDisk)(input.dataDir);
  // Room to spare (#11940): at 100 GiB free no share-of-the-disk guard speaks,
  // the floor and the marks alike.
  const room = roomy(disk);
  if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN))
    need(
      "todo",
      "disk",
      `Disk is nearly full: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free.`,
      undefined,
@@ -363,6 +392,7 @@
export function vitals(input: VitalsInput): Vitals {
    );
  else if (disk.totalBytes > 0 && !room && disk.freeBytes / disk.totalBytes < DISK_FLOOR)
    need(
      "todo",
      "disk-floor",
      `Disk is under the ${DISK_FLOOR * 100}% the forge keeps for itself: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free. Child forges sleep, and no child forge, app or check starts, until room is made.`,
      undefined,
@@ -374,6 +404,7 @@
export function vitals(input: VitalsInput): Vitals {
    const mark = DISK_USED_MARKS.find((m) => used >= m);
    if (mark !== undefined)
      need(
        "todo",
        `disk-${Math.round(mark * 100)}`,
        `Disk under the forge's data is ${Math.floor(used * 100)}% used, past the ${Math.round(mark * 100)}% mark.`,
        undefined,
@@ -382,10 +413,10 @@
export function vitals(input: VitalsInput): Vitals {
  }
  // The hosted forges' sizes together past the machine's memory: said, and nothing is stopped.
  if (input.forgesMemory !== undefined) need("forges-memory", input.forgesMemory, undefined, "/forges");
  if (input.forgesMemory !== undefined) need("todo", "forges-memory", input.forgesMemory, undefined, "/forges");
  for (const file of openDbFiles(input.dataDir))
    need(`file:${file.name}`, file.sentence, undefined, "/settings/capacity");
    need("todo", `file:${file.name}`, file.sentence, undefined, "/settings/capacity");
  // A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010),
  // with what fills it from the last walk once there is one (#12051): the sentence is the
@@ -394,7 +425,7 @@
export function vitals(input: VitalsInput): Vitals {
  const dbGrows = dbGrowth(input.db, dbBytes, now);
  let dbBig = dbSizeNeed(dbBytes, dbGrows);
  if (dbBig !== undefined) dbBig = dbSizeNeed(dbBytes, dbGrows, (input.dbWalk ?? dbSize.last)(input.db, input.dataDir));
  if (dbBig !== undefined) need(dbBig.cause, dbBig.sentence, undefined, "/system");
  if (dbBig !== undefined) need("todo", dbBig.cause, dbBig.sentence, undefined, "/system");
  // Snapshots on the volume they copy, and no mirror, share its one point of failure.
  const snaps = input.snapshots;
@@ -403,7 +434,7 @@
export function vitals(input: VitalsInput): Vitals {
    !snaps.mirrored &&
    sharesVolume(input.dataDir, snaps.dir)
  )
    need("snapshots", SNAPSHOTS_ON_STATE_VOLUME, undefined, "/settings/backups");
    need("todo", "snapshots", SNAPSHOTS_ON_STATE_VOLUME, undefined, "/settings/backups");
  const running = listRunning(input.db);
  const lane = readLane(input.db, running, now);
@@ -421,6 +452,7 @@
export function vitals(input: VitalsInput): Vitals {
      // Age and "since" come from the free checks alone.
      if (free.oldest !== null && now - new Date(free.oldest).getTime() > BUDGETS.laneWait)
        need(
          "todo",
          "queue:check",
          laneSentence(
            free.n,
@@ -455,6 +487,7 @@
export function vitals(input: VitalsInput): Vitals {
    const waited = now - new Date(oldest).getTime();
    if (waited > QUEUE_WAIT_MAX_MS)
      need(
        "todo",
        `queue:${row.kind}`,
        queueSentence(
          row.kind,
@@ -484,6 +517,7 @@
export function vitals(input: VitalsInput): Vitals {
  }
  for (const s of stuck)
    need(
      "todo",
      `stuck:${s.runId}`,
      now - new Date(s.since).getTime() > BUDGETS.heartbeat &&
      now - new Date(s.since).getTime() < 24 * 3_600_000
@@ -499,6 +533,7 @@
export function vitals(input: VitalsInput): Vitals {
    new Date(now - 24 * 3_600_000).toISOString(),
  ))
    need(
      "todo",
      `budget:${run.id}`,
      `A run was killed twice: ${run.summary}.`,
      run.finishedAt ?? undefined,
@@ -508,6 +543,7 @@
export function vitals(input: VitalsInput): Vitals {
  // A red main holding its pulls' checks is one card, naming the test.
  for (const { red, held } of reds)
    need(
      "broken",
      `red:${red.repoId}`,
      redSentence(red, held) + foreignNote(red.repo),
      red.since,
@@ -516,7 +552,7 @@
export function vitals(input: VitalsInput): Vitals {
  // A test failing on many pulls in the hour is one card, not a park per pull.
  for (const shared of sharedTests(input.db, now))
    need(sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
    need("todo", sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
  // Checks that failed on a full disk are one card, not a park per pull, and
  // only while the disk is still under the floor: once it is back the machine
@@ -524,13 +560,13 @@
export function vitals(input: VitalsInput): Vitals {
  const diskStillFull =
    disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN);
  const full = diskStillFull ? diskReds(input.db, now) : undefined;
  if (full !== undefined) need("disk-red", diskRedSentence(full), full.since, "/settings/capacity");
  if (full !== undefined) need("broken", "disk-red", diskRedSentence(full), full.since, "/settings/capacity");
  // A red restore drill, or none in eight days, is a card until a drill passes.
  const drill = lastVerdict(input.dataDir);
  const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
  if (owed !== undefined)
    need("drill", owed.sentence, owed.since, "/settings/backups");
    need("todo", "drill", owed.sentence, owed.since, "/settings/backups");
  // A domain whose drill is red or eight days old is a card with the command that runs it.
  for (const owed of domainDrillNeeds(
@@ -540,42 +576,43 @@
export function vitals(input: VitalsInput): Vitals {
    input.forgeUrl ?? "http://localhost",
    DOMAINS,
  ))
    need(owed.cause, owed.sentence, owed.since, "/");
    need("todo", owed.cause, owed.sentence, owed.since, "/");
  // A snapshot that failed is a card from that morning until one lands.
  const failing = snapshotNeed(lastSnapshotOutcome(input.dataDir));
  if (failing !== undefined) need("snapshot-failed", failing.sentence, failing.since, "/settings/backups");
  if (failing !== undefined) need("todo", "snapshot-failed", failing.sentence, failing.since, "/settings/backups");
  // The bucket itself, read by this process: late, unreadable, a failed drill, or a check that stopped is one card.
  if (isWatched(input.dataDir)) {
    const backup = backupVerdict(input.dataDir, lastDrill(input.db), now);
    if (backup.state === "red") need("backup", [...backup.reasons, ...(backup.reasons.some((r) => r.includes("last ran")) ? [] : [backup.checkLine])].join(" "), backup.checkedAt ?? undefined, "/settings/backups");
    if (backup.state === "red") need("todo", "backup", [...backup.reasons, ...(backup.reasons.some((r) => r.includes("last ran")) ? [] : [backup.checkLine])].join(" "), backup.checkedAt ?? undefined, "/settings/backups");
  }
  // A snapshot that did not land off the box, or a mirror a day behind, is a card until it does.
  const behind = mirrorNeed(lastMirror(input.dataDir), now);
  if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
  if (behind !== undefined) need("todo", "mirror", behind.sentence, behind.since, "/settings/backups");
  // A release the forge could not prove against its own public URL is a card until a release does.
  // Only a release that is not serving as it should is a card; a proof that could not finish for
  // its own reasons (the sign-up limit, a slow answer)
  // is a notice on /releases and on the home.
  const unproven = proofNeed(input.dataDir, runningVersion().version);
  if (unproven !== undefined && proofNotice(input.dataDir, runningVersion().version, now)?.tone === "alarm") need("release", unproven.sentence, unproven.since, "/releases#proof");
  if (unproven !== undefined && proofNotice(input.dataDir, runningVersion().version, now)?.tone === "alarm") need("todo", "release", unproven.sentence, unproven.since, "/releases#proof");
  // A roll the image's launcher undid is a card until a later version reads ready. Its sentence
  // carries the release's own last words (stderr), so anything credential-shaped is masked.
  const undone = rollbackNeed(input.dataDir);
  if (undone !== undefined) need("release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
  if (undone !== undefined) need("todo", "release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
  // Production serving a version older than a release main has had over an hour is a card until they match.
  const lagging = behindStanding(now);
  if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board");
  if (lagging !== undefined) need("todo", "release:behind", `${lagging.sentence}.`, lagging.since, "/board");
  // Lines recorded and not landed for five minutes: the writer is failing, however quiet the site.
  const quiet = unwrittenFor(now);
  if (quiet !== undefined && quiet > SILENT_AFTER_S)
    need(
      "todo",
      "access-log",
      `The access log is not writing: a recorded line has waited ${quiet} s to reach disk (limit ${SILENT_AFTER_S / 60} min).`,
      new Date(now - quiet * 1000).toISOString(),
@@ -592,6 +629,7 @@
export function vitals(input: VitalsInput): Vitals {
  const seen = recentErrors(5, now, input.signals.errors);
  if (seen.lastHour >= ERROR_BURST)
    need(
      "todo",
      "errors",
      `${seen.lastHour} errors in the last hour (${ERROR_BURST} or more is a burst).`,
      seen.oldest,
@@ -601,6 +639,7 @@
export function vitals(input: VitalsInput): Vitals {
  const looping = crashLoop(input.dataDir, now);
  if (looping !== null)
    need(
      "broken",
      "crash-loop",
      `The forge restarted ${looping.unclean} times without exiting cleanly in the last ${LOOP_WINDOW_MS / 60_000} minutes (${LOOP_STARTS} or more is a crash loop).`,
      new Date(looping.since).toISOString(),
@@ -610,6 +649,7 @@
export function vitals(input: VitalsInput): Vitals {
  const refused = refusedInternal(now, input.signals.refused);
  if (refused.lastHour > 0)
    need(
      "todo",
      "internal",
      `${refused.lastHour} internal call${refused.lastHour === 1 ? "" : "s"} without the boot secret refused in the last hour${refused.latest ? ` (latest ${refused.latest.method} ${refused.latest.path})` : ""}.`,
      refused.oldest,
@@ -620,8 +660,10 @@
export function vitals(input: VitalsInput): Vitals {
    if (!degraded.some((d) => d.cause === cause)) firstSeen.delete(cause);
  return {
    ok: needs.length === 0,
    ok: broken.length === 0,
    needs,
    broken,
    todo,
    degraded,
    disk,
    memory: {
@@ -431,7 +431,9 @@
function forgeNotes(ctx: Ctx): { notes: Note[]; ok: boolean; placement: Html | n
  const said = (line: string) => coveredByVerdict(line, verdict, fh);
  const rollout = rolloutLine(fh);
  const others = fh.incidents.filter((i) => !said(`${i.key}: ${i.message}`));
  const ok = v.ok && fh.ok && verdict === undefined;
  // "All fine" is no need at all, todo included: `v.ok` is serving, which a
  // standing wish keeps true (#12167).
  const ok = v.needs.length === 0 && fh.ok && verdict === undefined;
  const lane = laneLine(v.lane);
  const flaky = flakeLine(v.flakes);
  const sentinel = sentinelNotice(ctx.app.db);
@@ -289,8 +289,8 @@
test("the check records each run; stopped past its interval, the verdict, tile,
    Date.now = () => later;
    const card = vitals({ db: own.db, signals: own.signals, dataDir: dir, now: later }).degraded.find((d) => d.cause === "backup");
    expect(card?.sentence).toContain(`last ran ${ranAt}`);
    const health = readHealth(own, { isAdmin: true } as User) as { ok: boolean; degraded: { cause: string; sentence: string }[] };
    expect(health.ok).toBe(false);
    const health = readHealth(own, { isAdmin: true } as User) as { todo: string[]; degraded: { cause: string; sentence: string }[] };
    expect(health.todo.join(" ")).toContain(`last ran ${ranAt}`);
    expect(health.degraded.find((d) => d.cause === "backup")?.sentence).toContain(`last ran ${ranAt}`);
    expect(verdictLines({ app: own } as never).join("\n")).toContain("runs on record");
  } finally {
@@ -90,6 +90,6 @@
test("a check past its budget is re-queued once; the second overrun fails it and
  expect(second.summary).toMatch(/^budget: a full gate took \d+ min, limit \d+$/);
  const seen = vitals({ db: app.db, signals: app.signals, dataDir });
  expect(seen.ok).toBe(false);
  expect(seen.todo.length).toBeGreaterThan(0);
  expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
}, 60_000);
@@ -84,6 +84,7 @@
describe("a full disk", () => {
    const v = vitals({ db: app.db, dataDir: dir, signals: app.signals });
    expect(v.degraded.some((d) => d.cause === "disk-full")).toBe(true);
    expect(v.ok).toBe(false); // writes refused: broken, not todo (#12167)
    expect(v.needs.some((n) => /full.*MiB|full.*GiB|full.*none free/.test(n) && n.includes("delete"))).toBe(true);
    expect((await fetch(`${servers.url}/healthz`)).status).toBe(200); // liveness never claims the sentence
@@ -1,57 +1,168 @@
/** A crash loop is a need: three unclean starts in ten minutes (#12285). */
import { afterAll, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
/**
 * Health's ok means the forge is serving (#12167): a need is `broken` (act
 * now) or `todo` (would like done), and only `broken` makes ok false. This
 * file is the guard on the broken list: a real failure filed as todo would
 * answer ok:true and fail here. A crash loop is one: three unclean starts in
 * ten minutes (#12285).
 */
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
import { mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../src/app.ts";
import { createApp, type App } from "../src/app.ts";
import { recordBoot, recordCleanExit, crashLoop } from "../src/kernel/boots.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { DRAINING, readHealth } from "../src/actions/health.ts";
import { migrations } from "../src/core/schema.ts";
import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
import type { User } from "../src/core/users.ts";
import { health } from "../src/http/health.ts";
import { openDb } from "../src/kernel/db.ts";
import { ErrorLedger } from "../src/kernel/log.ts";
import { RefusedLedger } from "../src/kernel/internal_refused.ts";
import { vitals, type Vitals } from "../src/kernel/vitals.ts";
import { DB_FILE_CAP, dbFile } from "../src/system/db_size.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
const tmp = mkdtempSync(join(tmpdir(), "broken-"));
afterAll(() => rmSync(tmp, { recursive: true, force: true }));
const min = 60_000;
function ledger(name: string) {
  const dir = join(tmp, name);
  mkdirSync(dir);
  return dir;
}
const OID = (c: string) => c.repeat(40);
const roomy = () => ({ freeBytes: 900, totalBytes: 1000 });
test("three unclean starts in ten minutes raise a crash-loop need; two do not", () => {
  const dir = ledger("loop");
  recordBoot(dir, "a", 0);
  recordBoot(dir, "b", min);
  recordBoot(dir, "c", 2 * min);
  expect(crashLoop(dir, 3 * min)).toBeNull(); // a and b: two
  recordBoot(dir, "d", 3 * min);
  expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 });
let db: Database;
let dataDir: string;
beforeEach(() => {
  db = openDb(":memory:", migrations);
  dataDir = mkdtempSync(join(tmpdir(), "forge-health-broken-"));
  db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')");
  db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1', 'u1', 'forge')");
  db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i1', 'r1', 7, 'pull', 'p', 'u1')");
});
afterEach(() => {
  db.close();
  rmSync(dataDir, { recursive: true, force: true });
});
test("clean exits and old crashes are not a loop", () => {
  const dir = ledger("calm");
  for (const [i, id] of ["a", "b", "c", "d"].entries()) {
    recordBoot(dir, id, i * min);
    recordCleanExit(dir, id);
const read = (disk = roomy): Vitals =>
  vitals({
    db,
    dataDir,
    disk,
    dbWalk: () => null,
    signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
  });
describe("broken makes ok false", () => {
  test("a red main", () => {
    const run = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID("b"), reason: "push" });
    claimRun(db, run.id);
    finishRun(db, run.id, { state: "failed", summary: "check failed: 1 test failed", checkDetail: { passed: 9, failed: 1, names: ["t"] } });
    createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: "i1", commitOid: OID("a"), reason: "push" });
    const v = read();
    const red = v.degraded.find((d) => d.cause === "red:r1")!;
    expect(v.ok).toBe(false);
    expect(v.broken).toContain(red.sentence);
    expect(v.todo).not.toContain(red.sentence);
  });
  test("a red disk", () => {
    const run = createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: "i1", commitOid: OID("a"), reason: "push" });
    claimRun(db, run.id);
    finishRun(db, run.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: 1, names: ["t"], diskFree: 0.03 } });
    const v = read(() => ({ freeBytes: 30, totalBytes: 1000 }));
    const red = v.degraded.find((d) => d.cause === "disk-red")!;
    expect(v.ok).toBe(false);
    expect(v.broken).toContain(red.sentence);
  });
});
describe("todo alone keeps ok true", () => {
  test("a database over 2 GiB", () => {
    writeFileSync(dbFile(dataDir), "", { mode: 0o600 });
    truncateSync(dbFile(dataDir), DB_FILE_CAP + 1); // sparse: no disk spent
    const v = read();
    const big = v.degraded.find((d) => d.cause === "db-size")!;
    expect(v.todo).toEqual([big.sentence]);
    expect(v.broken).toEqual([]);
    expect(v.ok).toBe(true);
    // Callers before #12167 still read every sentence.
    expect(v.needs).toEqual([big.sentence]);
  });
});
describe("the health answer", () => {
  let app: App;
  let dir: string;
  const admin = { isAdmin: true } as User;
  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), "forge-health-drain-"));
    app = createApp({ dataDir: dir, port: 0 });
  });
  afterAll(async () => {
    health.draining = false;
    app.db.close();
    await rm(dir, { recursive: true, force: true });
  });
  test("a draining forge answers ok:false with draining under broken", () => {
    health.draining = true;
    const h = readHealth(app, admin) as Vitals;
    expect(h.ok).toBe(false);
    expect(h.broken).toContain(DRAINING);
    expect(h.todo).not.toContain(DRAINING);
    health.draining = false;
    expect((readHealth(app, admin) as Vitals).broken).not.toContain(DRAINING);
  });
});
describe("a crash loop", () => {
  const tmp = mkdtempSync(join(tmpdir(), "broken-"));
  afterAll(() => rmSync(tmp, { recursive: true, force: true }));
  const min = 60_000;
  function ledger(name: string) {
    const dir = join(tmp, name);
    mkdirSync(dir);
    return dir;
  }
  recordBoot(dir, "e", 4 * min);
  expect(crashLoop(dir, 4 * min)).toBeNull();
  const old = ledger("old");
  for (const id of ["a", "b", "c", "d"]) recordBoot(old, id, 0);
  expect(crashLoop(old, 30 * min)).toBeNull();
});
test("vitals carries the loop as a need that clears with the window", () => {
  const dir = ledger("vitals");
  const app = createApp({ dataDir: dir, url: "https://f.test" });
  const now = Date.now();
  for (const id of ["a", "b", "c", "d"]) recordBoot(dir, id, now - 60_000);
  const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, now, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
  const d = v.degraded.find((x) => x.cause === "crash-loop");
  expect(d?.sentence).toContain("restarted 3 times");
  expect(v.ok).toBe(false);
  const later = vitals({ db: app.db, signals: app.signals, dataDir: dir, now: now + 20 * min, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
  expect(later.degraded.find((x) => x.cause === "crash-loop")).toBeUndefined();
  app.close();
  test("three unclean starts in ten minutes raise a crash-loop need; two do not", () => {
    const dir = ledger("loop");
    recordBoot(dir, "a", 0);
    recordBoot(dir, "b", min);
    recordBoot(dir, "c", 2 * min);
    expect(crashLoop(dir, 3 * min)).toBeNull(); // a and b: two
    recordBoot(dir, "d", 3 * min);
    expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 });
  });
  test("clean exits and old crashes are not a loop", () => {
    const dir = ledger("calm");
    for (const [i, id] of ["a", "b", "c", "d"].entries()) {
      recordBoot(dir, id, i * min);
      recordCleanExit(dir, id);
    }
    recordBoot(dir, "e", 4 * min);
    expect(crashLoop(dir, 4 * min)).toBeNull();
    const old = ledger("old");
    for (const id of ["a", "b", "c", "d"]) recordBoot(old, id, 0);
    expect(crashLoop(old, 30 * min)).toBeNull();
  });
  test("vitals carries the loop as a need that clears with the window", () => {
    const dir = ledger("vitals");
    const app = createApp({ dataDir: dir, url: "https://f.test" });
    const now = Date.now();
    for (const id of ["a", "b", "c", "d"]) recordBoot(dir, id, now - 60_000);
    const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, now, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
    const d = v.degraded.find((x) => x.cause === "crash-loop");
    expect(d?.sentence).toContain("restarted 3 times");
    expect(v.ok).toBe(false);
    expect(v.broken).toContain(d!.sentence);
    expect(v.todo).not.toContain(d!.sentence);
    const later = vitals({ db: app.db, signals: app.signals, dataDir: dir, now: now + 20 * min, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
    expect(later.degraded.find((x) => x.cause === "crash-loop")).toBeUndefined();
    app.close();
  });
});
@@ -111,3 +111,13 @@
test("the terminal's forge strip shows the need to an admin and not to a guest",
    app.close();
  }
});
test("a broken chain is a broken need, not a todo: health is not ok (#12167)", async () => {
  broken();
  const { app, admin: a } = await admin();
  const { readHealth } = await import("../src/actions/health.ts");
  const h = readHealth(app, a) as { ok: boolean; broken: string[]; todo: string[] };
  expect(h.ok).toBe(false);
  expect(mentions(h.broken).length).toBe(1);
  expect(mentions(h.todo)).toEqual([]);
});
@@ -94,9 +94,9 @@
test("a site admin with a read-only token gets the shape; others get the short o
  expect(Object.keys(v).sort()).toEqual(
    [
      "accessLog",
      "apps", "at", "checkedAt", "commit", "degraded", "disk", "draining", "errors", "flakes",
      "apps", "at", "broken", "checkedAt", "commit", "degraded", "disk", "draining", "errors", "flakes",
      "incidents", "lane", "memory", "needs", "ok", "queue", "rollout", "stale", "stuck",
      "version",
      "todo", "version",
    ].sort(),
  );
  expect(v.disk.totalBytes).toBeGreaterThan(0);
@@ -129,7 +129,9 @@
test("a full queue shows in queue, and an old one in needs", async () => {
  // Process-wide signals (errors, refused calls) from other suites may speak too.
  expect(v.needs.join(" ")).not.toContain("queue");
  const later = vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now: old + 2 * 3_600_000 + 3_600_000 });
  expect(later.ok).toBe(false);
  // A waiting queue is a todo: the forge still serves (#12167).
  expect(later.ok).toBe(true);
  expect(later.todo.join(" ")).toContain("agent queue");
  expect(later.needs.join(" ")).toContain("agent queue");
});
@@ -143,7 +145,8 @@
test("a running run with a lapsed lease is stuck, and needs says so", async () =
  const v = (await (await health(admin)).json()) as ReturnType<typeof vitals>;
  expect(v.stuck.map((s) => s.runId)).toEqual([run.id]);
  expect(v.stuck[0]!.repo).toBe("alice/r");
  expect(v.ok).toBe(false);
  expect(v.todo.some((n) => n.includes("stuck"))).toBe(true);
  expect(v.broken.some((n) => n.includes("stuck"))).toBe(false);
  expect(v.needs.some((n) => n.includes("stuck"))).toBe(true);
  // A fresh heartbeat clears it.
  app.db
@@ -466,7 +469,8 @@
test("a lane older than its budget is degraded with its cause; a silent run is s
  expect(lane?.sentence).toMatch(
    /^3 checks wait behind one lane; the oldest since \d\d:\d\d; the lane runs a check for 31 min \(limit 60\)\.$/,
  );
  expect(v.ok).toBe(false);
  expect(v.todo).toContain(lane!.sentence);
  expect(v.ok).toBe(true);
  // Five minutes in, the oldest is under its budget: nothing to say.
  app.db.query("UPDATE run SET created_at=?1 WHERE state='queued'").run(iso(now - 5 * 60_000));
  expect(vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).degraded.map((d) => d.cause)).toEqual([]);
		@@ -122,7 +122,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
122	122	\| `fork_tree` \| — \| The fork tree around a repo: ancestors, siblings and descendants you may read \| 440 \|
123	123	\| `forks` \| `chain` \| A repo's forks you may see, paged by limit and cursor (next_cursor), each with commits ahead and behind \| 480 \|
124	124	\| `front_page` \| — \| The stranger's front page; admin sets `pinned`/`hidden`/`weights` \| 457 \|
125		\| `health` \| — \| How this forge is: version, commit, draining or not, rollout, incidents \| 285 \|
	125	\| `health` \| — \| How this forge is: version, rollout, incidents \| 316 \|
126	126	\| `held` \| — \| What the Later gate holds, and one proposed pass over it \| 361 \|
127	127	\| `here` \| — \| What you may do on a forge page; given an app's link, what its /__forge/ door shows \| 237 \|
128	128	\| `hold` \| — \| Hold a pull (`why`) or `release` it; or `path`: hold new work on a file \| 522 \|
		@@ -561,7 +561,7 @@ answers
561	561	\| `ui://forge/flow` \| `flow` \| The pipeline: per stage counts, oldest waiting, the bottleneck. \|
562	562	\| `ui://forge/scorecard` \| `scorecard` \| The crew's scorecard, one row per agent: landed and red rate all time, then the week (forge admin). \|
563	563	\| `ui://forge/crew` \| `now`, `crew` \| `now`: what the crew is doing right now, with Stop buttons. `crew` (listing): one row per seat, who fills it and what runs and waits, or what it costs to leave it empty; linked to Settings › Crew. \|
564		\| `ui://forge/health` \| `health` \| Green or red with the version, then one row per cause: draining, a blocked rollout, degraded lanes, incidents, apps (an admin sees all; a stranger only the verdict). \|
	564	\| `ui://forge/health` \| `health` \| Green or red with the version, then one row per cause: draining, a blocked rollout, degraded lanes, incidents, apps (an admin sees all; a stranger only the verdict). Its answer splits causes into `broken` (act now; any makes `ok` false) and `todo` (would like done; `ok` stays true). \|
565	565	\| `ui://forge/forge` \| `forge_view` \| The forge in one card: healthy or not and what is running, then the apps with work going, busiest first; quiet apps and what waits on you are counted. \|
566	566	\| `ui://forge/capacity` \| `capacity` (listing, admin) \| One row per setting with its live value (set ones marked), linked to Settings › Capacity; says when the forge cannot swap itself. \|
567	567	\| `ui://forge/board` \| `boards` \| One board's panels (person admins). \|
		@@ -101,12 +101,19 @@ export function readHealth(
101	101	}
102	102	if (!actor?.isAdmin) return forge;
103	103	const v = forgeVitals(app);
104		// A broken cost-ledger chain is one more need, from the verdict kept in memory (#12273).
	104	// Draining is the release's, not the vitals': it is broken all the same (#12167).
	105	// A broken cost-ledger chain is one more broken need, from the verdict kept in memory (#12273).
105	106	const ledger = ledgerNeed();
106		if (ledger === null) return { ...v, ...forge, ok: forge.ok && v.ok };
107		return { ...v, ...forge, ok: false, needs: [...v.needs, ledger] };
	107	const extra = [...(forge.draining ? [DRAINING] : []), ...(ledger === null ? [] : [ledger])];
	108	const broken = [...extra, ...v.broken];
	109	const needs = ledger === null ? v.needs : [...v.needs, ledger];
	110	const ok = forge.ok && v.ok && ledger === null;
	111	return { ...v, ...forge, ok, needs, broken, todo: v.todo };
108	112	}
109	113
	114	/** The `broken` line a draining forge answers with. */
	115	export const DRAINING = "The forge is draining: this process finishes what it holds and takes no new work.";
	116
110	117	/**
111	118	* The forge's causes for concern as one sentence each, for a site admin: the
112	119	* rollout, each platform incident, then the vitals' needs, in the order the
		@@ -103,7 +103,7 @@ export async function facts(app: App, user: User, now: number, sinceMs = now - 3
103	103	const story = await storyFacts(app, sinceMs);
104	104	return {
105	105	...story,
106		health: vitals.ok ? [] : vitals.needs.slice(0, 3),
	106	health: vitals.needs.slice(0, 3),
107	107	redMain: redLines(app, user, now).map((l) => l.line),
108	108	asks: askLinesFor(db, user).length,
109	109	holds,
		@@ -5,7 +5,7 @@ export const tool: Tool = {
5	5	name: "health",
6	6	doc: "how this forge is: the version and commit it runs, draining or not, the rollout and its incidents",
7	7	description:
8		"How this forge is: version, commit, draining or not, rollout, incidents.",
	8	"How this forge is: version, rollout, incidents. ok means serving, broken act now, todo would like done.",
9	9	inputSchema: { type: "object", properties: {} },
10	10	writes: false,
11	11	run: (ctx, user) => good(readHealth(ctx.app, user)),
		@@ -74,6 +74,24 @@ export interface EnvCount {
74	74	readonly state: string;
75	75	}
		sentinelcrew 1d ago outdated This comment lists memory below its floor and a crash loop as `broken`, but no code raises either. Say so here, and open follow-up issues for the two detectors. Also say why `disk` and `disk-floor` are `todo`.
76	76
	77	/**
	78	* A need is `broken` when the forge is not serving as it should and someone
	79	* must act now: draining, the database not answering, a full disk refusing
	80	* writes (`disk-full`), a red disk (`disk-red`), memory below its floor, a
	81	* crash loop, a red main. Everything else is `todo`: a standing wish (a
	82	* database over 2 GiB, a drill owed, a queue waiting) the forge serves
	83	* through. Of those, vitals see `disk-full`, `disk-red`, `crash-loop` and
	84	* `red:<repoId>`; draining is added by the health answer, and the database
	85	* not answering is `/healthz`'s 503. Memory below its floor is `broken` by
	86	* this rule, but nothing raises it yet (#12284).
	87	*
	88	* `disk` and `disk-floor` are `todo`: a low disk stops new starts (child
	89	* forges, apps, checks), but the forge keeps serving what is already running
	90	* while room is made. A disk is `broken` (`disk-red`) only once checks are
	91	* failing on it, which means work is being lost and not just held back.
	92	*/
	93	export type NeedKind = "broken" \| "todo";
	94
77	95	/**
78	96	* One cause of the forge being off: the `needs` sentence, a `cause` key that
79	97	* holds for as long as its condition does (`disk`, `disk-floor`, `disk-80`, `disk-70`,
		@@ -102,8 +120,14 @@ export interface Lane {
102	120	}
103	121
104	122	export interface Vitals {
	123	/** Serving: nothing is `broken`. Standing wishes (`todo`) never make it false. */
105	124	readonly ok: boolean;
	125	/** Every sentence, `broken` then `todo` in the order found: what callers before #12167 read. */
106	126	readonly needs: string[];
	127	/** What stops the forge serving: act now. */
	128	readonly broken: string[];
	129	/** What the forge would like done: it serves meanwhile. */
	130	readonly todo: string[];
107	131	readonly degraded: Degraded[];
108	132	readonly disk: { freeBytes: number; totalBytes: number };
109	133	readonly memory: {
		@@ -333,7 +357,10 @@ export function vitals(input: VitalsInput): Vitals {
333	357	const degraded: Degraded[] = [];
334	358	let firstSeen = firstSeenByForge.get(input.signals.errors);
335	359	if (!firstSeen) firstSeenByForge.set(input.signals.errors, (firstSeen = new Map()));
	360	const broken: string[] = [];
	361	const todo: string[] = [];
336	362	const need = (
	363	kind: NeedKind,
337	364	cause: string,
338	365	sentence: string,
339	366	evidence: string \| undefined,
		@@ -345,17 +372,19 @@ export function vitals(input: VitalsInput): Vitals {
345	372	firstSeen.set(cause, since);
346	373	}
347	374	needs.push(sentence);
	375	(kind === "broken" ? broken : todo).push(sentence);
348	376	degraded.push({ cause, sentence, since, href });
349	377	};
350	378
351	379	const fullAt = fullSince(input.dataDir);
352		if (fullAt !== undefined) need("disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
	380	if (fullAt !== undefined) need("broken", "disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
353	381	const disk = (input.disk ?? readDisk)(input.dataDir);
354	382	// Room to spare (#11940): at 100 GiB free no share-of-the-disk guard speaks,
355	383	// the floor and the marks alike.
356	384	const room = roomy(disk);
357	385	if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN))
358	386	need(
	387	"todo",
359	388	"disk",
360	389	`Disk is nearly full: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free.`,
361	390	undefined,
		@@ -363,6 +392,7 @@ export function vitals(input: VitalsInput): Vitals {
363	392	);
364	393	else if (disk.totalBytes > 0 && !room && disk.freeBytes / disk.totalBytes < DISK_FLOOR)
365	394	need(
	395	"todo",
366	396	"disk-floor",
367	397	`Disk is under the ${DISK_FLOOR * 100}% the forge keeps for itself: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free. Child forges sleep, and no child forge, app or check starts, until room is made.`,
368	398	undefined,
		@@ -374,6 +404,7 @@ export function vitals(input: VitalsInput): Vitals {
374	404	const mark = DISK_USED_MARKS.find((m) => used >= m);
375	405	if (mark !== undefined)
376	406	need(
	407	"todo",
377	408	`disk-${Math.round(mark * 100)}`,
378	409	`Disk under the forge's data is ${Math.floor(used * 100)}% used, past the ${Math.round(mark * 100)}% mark.`,
379	410	undefined,
		@@ -382,10 +413,10 @@ export function vitals(input: VitalsInput): Vitals {
382	413	}
383	414
384	415	// The hosted forges' sizes together past the machine's memory: said, and nothing is stopped.
385		if (input.forgesMemory !== undefined) need("forges-memory", input.forgesMemory, undefined, "/forges");
	416	if (input.forgesMemory !== undefined) need("todo", "forges-memory", input.forgesMemory, undefined, "/forges");
386	417
387	418	for (const file of openDbFiles(input.dataDir))
388		need(`file:${file.name}`, file.sentence, undefined, "/settings/capacity");
	419	need("todo", `file:${file.name}`, file.sentence, undefined, "/settings/capacity");
389	420
390	421	// A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010),
391	422	// with what fills it from the last walk once there is one (#12051): the sentence is the
		@@ -394,7 +425,7 @@ export function vitals(input: VitalsInput): Vitals {
394	425	const dbGrows = dbGrowth(input.db, dbBytes, now);
395	426	let dbBig = dbSizeNeed(dbBytes, dbGrows);
396	427	if (dbBig !== undefined) dbBig = dbSizeNeed(dbBytes, dbGrows, (input.dbWalk ?? dbSize.last)(input.db, input.dataDir));
397		if (dbBig !== undefined) need(dbBig.cause, dbBig.sentence, undefined, "/system");
	428	if (dbBig !== undefined) need("todo", dbBig.cause, dbBig.sentence, undefined, "/system");
398	429
399	430	// Snapshots on the volume they copy, and no mirror, share its one point of failure.
400	431	const snaps = input.snapshots;
		@@ -403,7 +434,7 @@ export function vitals(input: VitalsInput): Vitals {
403	434	!snaps.mirrored &&
404	435	sharesVolume(input.dataDir, snaps.dir)
405	436	)
406		need("snapshots", SNAPSHOTS_ON_STATE_VOLUME, undefined, "/settings/backups");
	437	need("todo", "snapshots", SNAPSHOTS_ON_STATE_VOLUME, undefined, "/settings/backups");
407	438
408	439	const running = listRunning(input.db);
409	440	const lane = readLane(input.db, running, now);
		@@ -421,6 +452,7 @@ export function vitals(input: VitalsInput): Vitals {
421	452	// Age and "since" come from the free checks alone.
422	453	if (free.oldest !== null && now - new Date(free.oldest).getTime() > BUDGETS.laneWait)
423	454	need(
	455	"todo",
424	456	"queue:check",
425	457	laneSentence(
426	458	free.n,
		@@ -455,6 +487,7 @@ export function vitals(input: VitalsInput): Vitals {
455	487	const waited = now - new Date(oldest).getTime();
456	488	if (waited > QUEUE_WAIT_MAX_MS)
457	489	need(
	490	"todo",
458	491	`queue:${row.kind}`,
459	492	queueSentence(
460	493	row.kind,
		@@ -484,6 +517,7 @@ export function vitals(input: VitalsInput): Vitals {
484	517	}
485	518	for (const s of stuck)
486	519	need(
	520	"todo",
487	521	`stuck:${s.runId}`,
488	522	now - new Date(s.since).getTime() > BUDGETS.heartbeat &&
489	523	now - new Date(s.since).getTime() < 24 * 3_600_000
		@@ -499,6 +533,7 @@ export function vitals(input: VitalsInput): Vitals {
499	533	new Date(now - 24 * 3_600_000).toISOString(),
500	534	))
501	535	need(
	536	"todo",
502	537	`budget:${run.id}`,
503	538	`A run was killed twice: ${run.summary}.`,
504	539	run.finishedAt ?? undefined,
		@@ -508,6 +543,7 @@ export function vitals(input: VitalsInput): Vitals {
508	543	// A red main holding its pulls' checks is one card, naming the test.
509	544	for (const { red, held } of reds)
510	545	need(
	546	"broken",
511	547	`red:${red.repoId}`,
512	548	redSentence(red, held) + foreignNote(red.repo),
513	549	red.since,
		@@ -516,7 +552,7 @@ export function vitals(input: VitalsInput): Vitals {
516	552
517	553	// A test failing on many pulls in the hour is one card, not a park per pull.
518	554	for (const shared of sharedTests(input.db, now))
519		need(sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
	555	need("todo", sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
520	556
521	557	// Checks that failed on a full disk are one card, not a park per pull, and
522	558	// only while the disk is still under the floor: once it is back the machine
		@@ -524,13 +560,13 @@ export function vitals(input: VitalsInput): Vitals {
524	560	const diskStillFull =
525	561	disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < (input.diskFreeMin ?? DISK_FREE_MIN);
526	562	const full = diskStillFull ? diskReds(input.db, now) : undefined;
527		if (full !== undefined) need("disk-red", diskRedSentence(full), full.since, "/settings/capacity");
	563	if (full !== undefined) need("broken", "disk-red", diskRedSentence(full), full.since, "/settings/capacity");
528	564
529	565	// A red restore drill, or none in eight days, is a card until a drill passes.
530	566	const drill = lastVerdict(input.dataDir);
531	567	const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
532	568	if (owed !== undefined)
533		need("drill", owed.sentence, owed.since, "/settings/backups");
	569	need("todo", "drill", owed.sentence, owed.since, "/settings/backups");
534	570
535	571	// A domain whose drill is red or eight days old is a card with the command that runs it.
536	572	for (const owed of domainDrillNeeds(
		@@ -540,42 +576,43 @@ export function vitals(input: VitalsInput): Vitals {
540	576	input.forgeUrl ?? "http://localhost",
541	577	DOMAINS,
542	578	))
543		need(owed.cause, owed.sentence, owed.since, "/");
	579	need("todo", owed.cause, owed.sentence, owed.since, "/");
544	580
545	581	// A snapshot that failed is a card from that morning until one lands.
546	582	const failing = snapshotNeed(lastSnapshotOutcome(input.dataDir));
547		if (failing !== undefined) need("snapshot-failed", failing.sentence, failing.since, "/settings/backups");
	583	if (failing !== undefined) need("todo", "snapshot-failed", failing.sentence, failing.since, "/settings/backups");
548	584
549	585	// The bucket itself, read by this process: late, unreadable, a failed drill, or a check that stopped is one card.
550	586	if (isWatched(input.dataDir)) {
551	587	const backup = backupVerdict(input.dataDir, lastDrill(input.db), now);
552		if (backup.state === "red") need("backup", [...backup.reasons, ...(backup.reasons.some((r) => r.includes("last ran")) ? [] : [backup.checkLine])].join(" "), backup.checkedAt ?? undefined, "/settings/backups");
	588	if (backup.state === "red") need("todo", "backup", [...backup.reasons, ...(backup.reasons.some((r) => r.includes("last ran")) ? [] : [backup.checkLine])].join(" "), backup.checkedAt ?? undefined, "/settings/backups");
553	589	}
554	590
555	591	// A snapshot that did not land off the box, or a mirror a day behind, is a card until it does.
556	592	const behind = mirrorNeed(lastMirror(input.dataDir), now);
557		if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
	593	if (behind !== undefined) need("todo", "mirror", behind.sentence, behind.since, "/settings/backups");
558	594
559	595	// A release the forge could not prove against its own public URL is a card until a release does.
560	596	// Only a release that is not serving as it should is a card; a proof that could not finish for
561	597	// its own reasons (the sign-up limit, a slow answer)
562	598	// is a notice on /releases and on the home.
563	599	const unproven = proofNeed(input.dataDir, runningVersion().version);
564		if (unproven !== undefined && proofNotice(input.dataDir, runningVersion().version, now)?.tone === "alarm") need("release", unproven.sentence, unproven.since, "/releases#proof");
	600	if (unproven !== undefined && proofNotice(input.dataDir, runningVersion().version, now)?.tone === "alarm") need("todo", "release", unproven.sentence, unproven.since, "/releases#proof");
565	601
566	602	// A roll the image's launcher undid is a card until a later version reads ready. Its sentence
567	603	// carries the release's own last words (stderr), so anything credential-shaped is masked.
568	604	const undone = rollbackNeed(input.dataDir);
569		if (undone !== undefined) need("release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
	605	if (undone !== undefined) need("todo", "release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
570	606
571	607	// Production serving a version older than a release main has had over an hour is a card until they match.
572	608	const lagging = behindStanding(now);
573		if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board");
	609	if (lagging !== undefined) need("todo", "release:behind", `${lagging.sentence}.`, lagging.since, "/board");
574	610
575	611	// Lines recorded and not landed for five minutes: the writer is failing, however quiet the site.
576	612	const quiet = unwrittenFor(now);
577	613	if (quiet !== undefined && quiet > SILENT_AFTER_S)
578	614	need(
	615	"todo",
579	616	"access-log",
580	617	`The access log is not writing: a recorded line has waited ${quiet} s to reach disk (limit ${SILENT_AFTER_S / 60} min).`,
581	618	new Date(now - quiet * 1000).toISOString(),
		@@ -592,6 +629,7 @@ export function vitals(input: VitalsInput): Vitals {
592	629	const seen = recentErrors(5, now, input.signals.errors);
593	630	if (seen.lastHour >= ERROR_BURST)
594	631	need(
	632	"todo",
595	633	"errors",
596	634	`${seen.lastHour} errors in the last hour (${ERROR_BURST} or more is a burst).`,
597	635	seen.oldest,
		@@ -601,6 +639,7 @@ export function vitals(input: VitalsInput): Vitals {
601	639	const looping = crashLoop(input.dataDir, now);
602	640	if (looping !== null)
603	641	need(
	642	"broken",
604	643	"crash-loop",
605	644	`The forge restarted ${looping.unclean} times without exiting cleanly in the last ${LOOP_WINDOW_MS / 60_000} minutes (${LOOP_STARTS} or more is a crash loop).`,
606	645	new Date(looping.since).toISOString(),
		@@ -610,6 +649,7 @@ export function vitals(input: VitalsInput): Vitals {
610	649	const refused = refusedInternal(now, input.signals.refused);
611	650	if (refused.lastHour > 0)
612	651	need(
	652	"todo",
613	653	"internal",
614	654	`${refused.lastHour} internal call${refused.lastHour === 1 ? "" : "s"} without the boot secret refused in the last hour${refused.latest ? ` (latest ${refused.latest.method} ${refused.latest.path})` : ""}.`,
615	655	refused.oldest,
		@@ -620,8 +660,10 @@ export function vitals(input: VitalsInput): Vitals {
620	660	if (!degraded.some((d) => d.cause === cause)) firstSeen.delete(cause);
621	661
622	662	return {
623		ok: needs.length === 0,
	663	ok: broken.length === 0,
624	664	needs,
	665	broken,
	666	todo,
625	667	degraded,
626	668	disk,
627	669	memory: {
		@@ -431,7 +431,9 @@ function forgeNotes(ctx: Ctx): { notes: Note[]; ok: boolean; placement: Html \| n
431	431	const said = (line: string) => coveredByVerdict(line, verdict, fh);
432	432	const rollout = rolloutLine(fh);
433	433	const others = fh.incidents.filter((i) => !said(`${i.key}: ${i.message}`));
434		const ok = v.ok && fh.ok && verdict === undefined;
	434	// "All fine" is no need at all, todo included: `v.ok` is serving, which a
	435	// standing wish keeps true (#12167).
	436	const ok = v.needs.length === 0 && fh.ok && verdict === undefined;
435	437	const lane = laneLine(v.lane);
436	438	const flaky = flakeLine(v.flakes);
437	439	const sentinel = sentinelNotice(ctx.app.db);
		@@ -289,8 +289,8 @@ test("the check records each run; stopped past its interval, the verdict, tile,
289	289	Date.now = () => later;
290	290	const card = vitals({ db: own.db, signals: own.signals, dataDir: dir, now: later }).degraded.find((d) => d.cause === "backup");
291	291	expect(card?.sentence).toContain(`last ran ${ranAt}`);
292		const health = readHealth(own, { isAdmin: true } as User) as { ok: boolean; degraded: { cause: string; sentence: string }[] };
293		expect(health.ok).toBe(false);
	292	const health = readHealth(own, { isAdmin: true } as User) as { todo: string[]; degraded: { cause: string; sentence: string }[] };
	293	expect(health.todo.join(" ")).toContain(`last ran ${ranAt}`);
294	294	expect(health.degraded.find((d) => d.cause === "backup")?.sentence).toContain(`last ran ${ranAt}`);
295	295	expect(verdictLines({ app: own } as never).join("\n")).toContain("runs on record");
296	296	} finally {
		@@ -90,6 +90,6 @@ test("a check past its budget is re-queued once; the second overrun fails it and
90	90	expect(second.summary).toMatch(/^budget: a full gate took \d+ min, limit \d+$/);
91	91
92	92	const seen = vitals({ db: app.db, signals: app.signals, dataDir });
93		expect(seen.ok).toBe(false);
	93	expect(seen.todo.length).toBeGreaterThan(0);
94	94	expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
95	95	}, 60_000);
		@@ -84,6 +84,7 @@ describe("a full disk", () => {
84	84
85	85	const v = vitals({ db: app.db, dataDir: dir, signals: app.signals });
86	86	expect(v.degraded.some((d) => d.cause === "disk-full")).toBe(true);
	87	expect(v.ok).toBe(false); // writes refused: broken, not todo (#12167)
87	88	expect(v.needs.some((n) => /full.MiB\|full.GiB\|full.*none free/.test(n) && n.includes("delete"))).toBe(true);
88	89	expect((await fetch(`${servers.url}/healthz`)).status).toBe(200); // liveness never claims the sentence
89	90
		@@ -1,57 +1,168 @@
1		/** A crash loop is a need: three unclean starts in ten minutes (#12285). */
2		import { afterAll, expect, test } from "bun:test";
3		import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
	1	/**
	2	* Health's ok means the forge is serving (#12167): a need is `broken` (act
	3	* now) or `todo` (would like done), and only `broken` makes ok false. This
	4	* file is the guard on the broken list: a real failure filed as todo would
	5	* answer ok:true and fail here. A crash loop is one: three unclean starts in
	6	* ten minutes (#12285).
	7	*/
	8	import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
	9	import type { Database } from "bun:sqlite";
	10	import { mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs";
	11	import { mkdtemp, rm } from "node:fs/promises";
4	12	import { tmpdir } from "node:os";
5	13	import { join } from "node:path";
6		import { createApp } from "../src/app.ts";
	14	import { createApp, type App } from "../src/app.ts";
7	15	import { recordBoot, recordCleanExit, crashLoop } from "../src/kernel/boots.ts";
8		import { vitals } from "../src/kernel/vitals.ts";
	16	import { DRAINING, readHealth } from "../src/actions/health.ts";
	17	import { migrations } from "../src/core/schema.ts";
	18	import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
	19	import type { User } from "../src/core/users.ts";
	20	import { health } from "../src/http/health.ts";
	21	import { openDb } from "../src/kernel/db.ts";
	22	import { ErrorLedger } from "../src/kernel/log.ts";
	23	import { RefusedLedger } from "../src/kernel/internal_refused.ts";
	24	import { vitals, type Vitals } from "../src/kernel/vitals.ts";
	25	import { DB_FILE_CAP, dbFile } from "../src/system/db_size.ts";
9	26	import { quietLogs } from "./helpers/globals.ts";
10	27
11	28	quietLogs();
12		const tmp = mkdtempSync(join(tmpdir(), "broken-"));
13		afterAll(() => rmSync(tmp, { recursive: true, force: true }));
14		const min = 60_000;
15	29
16		function ledger(name: string) {
17		const dir = join(tmp, name);
18		mkdirSync(dir);
19		return dir;
20		}
	30	const OID = (c: string) => c.repeat(40);
	31	const roomy = () => ({ freeBytes: 900, totalBytes: 1000 });
21	32
22		test("three unclean starts in ten minutes raise a crash-loop need; two do not", () => {
23		const dir = ledger("loop");
24		recordBoot(dir, "a", 0);
25		recordBoot(dir, "b", min);
26		recordBoot(dir, "c", 2 * min);
27		expect(crashLoop(dir, 3 * min)).toBeNull(); // a and b: two
28		recordBoot(dir, "d", 3 * min);
29		expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 });
	33	let db: Database;
	34	let dataDir: string;
	35	beforeEach(() => {
	36	db = openDb(":memory:", migrations);
	37	dataDir = mkdtempSync(join(tmpdir(), "forge-health-broken-"));
	38	db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')");
	39	db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1', 'u1', 'forge')");
	40	db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i1', 'r1', 7, 'pull', 'p', 'u1')");
	41	});
	42	afterEach(() => {
	43	db.close();
	44	rmSync(dataDir, { recursive: true, force: true });
30	45	});
31	46
32		test("clean exits and old crashes are not a loop", () => {
33		const dir = ledger("calm");
34		for (const [i, id] of ["a", "b", "c", "d"].entries()) {
35		recordBoot(dir, id, i * min);
36		recordCleanExit(dir, id);
	47	const read = (disk = roomy): Vitals =>
	48	vitals({
	49	db,
	50	dataDir,
	51	disk,
	52	dbWalk: () => null,
	53	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
	54	});
	55
	56	describe("broken makes ok false", () => {
	57	test("a red main", () => {
	58	const run = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID("b"), reason: "push" });
	59	claimRun(db, run.id);
	60	finishRun(db, run.id, { state: "failed", summary: "check failed: 1 test failed", checkDetail: { passed: 9, failed: 1, names: ["t"] } });
	61	createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: "i1", commitOid: OID("a"), reason: "push" });
	62	const v = read();
	63	const red = v.degraded.find((d) => d.cause === "red:r1")!;
	64	expect(v.ok).toBe(false);
	65	expect(v.broken).toContain(red.sentence);
	66	expect(v.todo).not.toContain(red.sentence);
	67	});
	68
	69	test("a red disk", () => {
	70	const run = createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: "i1", commitOid: OID("a"), reason: "push" });
	71	claimRun(db, run.id);
	72	finishRun(db, run.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: 1, names: ["t"], diskFree: 0.03 } });
	73	const v = read(() => ({ freeBytes: 30, totalBytes: 1000 }));
	74	const red = v.degraded.find((d) => d.cause === "disk-red")!;
	75	expect(v.ok).toBe(false);
	76	expect(v.broken).toContain(red.sentence);
	77	});
	78	});
	79
	80	describe("todo alone keeps ok true", () => {
	81	test("a database over 2 GiB", () => {
	82	writeFileSync(dbFile(dataDir), "", { mode: 0o600 });
	83	truncateSync(dbFile(dataDir), DB_FILE_CAP + 1); // sparse: no disk spent
	84	const v = read();
	85	const big = v.degraded.find((d) => d.cause === "db-size")!;
	86	expect(v.todo).toEqual([big.sentence]);
	87	expect(v.broken).toEqual([]);
	88	expect(v.ok).toBe(true);
	89	// Callers before #12167 still read every sentence.
	90	expect(v.needs).toEqual([big.sentence]);
	91	});
	92	});
	93
	94	describe("the health answer", () => {
	95	let app: App;
	96	let dir: string;
	97	const admin = { isAdmin: true } as User;
	98	beforeAll(async () => {
	99	dir = await mkdtemp(join(tmpdir(), "forge-health-drain-"));
	100	app = createApp({ dataDir: dir, port: 0 });
	101	});
	102	afterAll(async () => {
	103	health.draining = false;
	104	app.db.close();
	105	await rm(dir, { recursive: true, force: true });
	106	});
	107
	108	test("a draining forge answers ok:false with draining under broken", () => {
	109	health.draining = true;
	110	const h = readHealth(app, admin) as Vitals;
	111	expect(h.ok).toBe(false);
	112	expect(h.broken).toContain(DRAINING);
	113	expect(h.todo).not.toContain(DRAINING);
	114	health.draining = false;
	115	expect((readHealth(app, admin) as Vitals).broken).not.toContain(DRAINING);
	116	});
	117	});
	118
	119	describe("a crash loop", () => {
	120	const tmp = mkdtempSync(join(tmpdir(), "broken-"));
	121	afterAll(() => rmSync(tmp, { recursive: true, force: true }));
	122	const min = 60_000;
	123
	124	function ledger(name: string) {
	125	const dir = join(tmp, name);
	126	mkdirSync(dir);
	127	return dir;
37	128	}
38		recordBoot(dir, "e", 4 * min);
39		expect(crashLoop(dir, 4 * min)).toBeNull();
40		const old = ledger("old");
41		for (const id of ["a", "b", "c", "d"]) recordBoot(old, id, 0);
42		expect(crashLoop(old, 30 * min)).toBeNull();
43		});
44	129
45		test("vitals carries the loop as a need that clears with the window", () => {
46		const dir = ledger("vitals");
47		const app = createApp({ dataDir: dir, url: "https://f.test" });
48		const now = Date.now();
49		for (const id of ["a", "b", "c", "d"]) recordBoot(dir, id, now - 60_000);
50		const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, now, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
51		const d = v.degraded.find((x) => x.cause === "crash-loop");
52		expect(d?.sentence).toContain("restarted 3 times");
53		expect(v.ok).toBe(false);
54		const later = vitals({ db: app.db, signals: app.signals, dataDir: dir, now: now + 20 * min, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
55		expect(later.degraded.find((x) => x.cause === "crash-loop")).toBeUndefined();
56		app.close();
	130	test("three unclean starts in ten minutes raise a crash-loop need; two do not", () => {
	131	const dir = ledger("loop");
	132	recordBoot(dir, "a", 0);
	133	recordBoot(dir, "b", min);
	134	recordBoot(dir, "c", 2 * min);
	135	expect(crashLoop(dir, 3 * min)).toBeNull(); // a and b: two
	136	recordBoot(dir, "d", 3 * min);
	137	expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 });
	138	});
	139
	140	test("clean exits and old crashes are not a loop", () => {
	141	const dir = ledger("calm");
	142	for (const [i, id] of ["a", "b", "c", "d"].entries()) {
	143	recordBoot(dir, id, i * min);
	144	recordCleanExit(dir, id);
	145	}
	146	recordBoot(dir, "e", 4 * min);
	147	expect(crashLoop(dir, 4 * min)).toBeNull();
	148	const old = ledger("old");
	149	for (const id of ["a", "b", "c", "d"]) recordBoot(old, id, 0);
	150	expect(crashLoop(old, 30 * min)).toBeNull();
	151	});
	152
	153	test("vitals carries the loop as a need that clears with the window", () => {
	154	const dir = ledger("vitals");
	155	const app = createApp({ dataDir: dir, url: "https://f.test" });
	156	const now = Date.now();
	157	for (const id of ["a", "b", "c", "d"]) recordBoot(dir, id, now - 60_000);
	158	const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, now, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
	159	const d = v.degraded.find((x) => x.cause === "crash-loop");
	160	expect(d?.sentence).toContain("restarted 3 times");
	161	expect(v.ok).toBe(false);
	162	expect(v.broken).toContain(d!.sentence);
	163	expect(v.todo).not.toContain(d!.sentence);
	164	const later = vitals({ db: app.db, signals: app.signals, dataDir: dir, now: now + 20 * min, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
	165	expect(later.degraded.find((x) => x.cause === "crash-loop")).toBeUndefined();
	166	app.close();
	167	});
57	168	});