6008501 · 7c722f3
@@ -0,0 +1,88 @@
/**
 * A red recorded while the check machine's disk was under its free floor is
 * the machine's, not the pull's (#5453): a full disk sank unrelated pulls
 * whose builders found nothing to fix, and one such red is enough, unlike
 * `shared_red.ts`, which needs three pulls to agree. So the check service
 * records the disk's free fraction on every red run (`CheckDetail.diskFree`),
 * the crew neither reworks nor parks one that finished under
 * `DISK_FREE_MIN` and checks it again once the disk is back above the floor,
 * and the vitals carry one card for all of them.
 *
 * Plain data: a reading in, a verdict out; the one place the disk is read is
 * {@link diskReading}, which a test replaces.
 */
import type { Database } from "bun:sqlite";
import type { CheckDetail, Run } from "../core/runs.ts";
import { idFloor } from "../kernel/id.ts";
import { DISK_FREE_MIN, statDisk, type DiskReading } from "../kernel/vitals.ts";
/** How far back a disk-full red still counts for the card. */
export const DISK_RED_WINDOW_MS = 3_600_000;
/** Checks are created before they finish; see `shared_red.ts`. */
const QUEUED_SLACK_MS = 6 * 3_600_000;
/** Where the disk is read from; a test swaps `read` for plain data. */
export const diskReading: { read: (dir: string) => DiskReading } = { read: statDisk };
/** The free share of a reading, 0..1; undefined for a volume that reports no size. */
export const freeFraction = (d: DiskReading): number | undefined =>
  d.totalBytes > 0 ? d.freeBytes / d.totalBytes : undefined;
/** Whether a free share is under the floor. */
export const underFloor = (free: number | undefined): boolean =>
  free !== undefined && free < DISK_FREE_MIN;
/** A failed check's detail with the disk's free share recorded on it. */
export function withDisk(detail: CheckDetail | undefined, dataDir: string): CheckDetail {
  const free = freeFraction(diskReading.read(dataDir));
  const base = detail ?? { passed: 0, failed: 0, names: [] };
  return free === undefined ? base : { ...base, diskFree: free };
}
/** Whether a run finished red while the disk was under its floor. */
export const diskRed = (run: Pick<Run, "checkDetail"> | undefined): boolean =>
  underFloor(run?.checkDetail?.diskFree);
/** Whether the disk under `dataDir` is back above the floor. */
export const diskRecovered = (dataDir: string): boolean => {
  const free = freeFraction(diskReading.read(dataDir));
  return free === undefined || free >= DISK_FREE_MIN;
};
export interface DiskRed {
  /** Pull checks that failed with the disk under the floor in the window. */
  readonly checks: number;
  /** The lowest free share any recorded. */
  readonly free: number;
  /** When the earliest finished. */
  readonly since: string;
}
/** The disk-full reds of the last hour, or undefined when there are none. */
export function diskReds(db: Database, now: number = Date.now()): DiskRed | undefined {
  const row = db
    .query<{ n: number; free: number | null; since: string | null }, [string, string, number]>(
      `SELECT COUNT(*) AS n,
              MIN(json_extract(check_detail, '$.diskFree')) AS free,
              MIN(finished_at) AS since
         FROM run
        WHERE id >= ?1 AND finished_at >= ?2
          AND kind = 'check' AND issue_id IS NOT NULL AND state = 'failed'
          AND json_valid(check_detail)
          AND json_type(check_detail, '$.diskFree') IN ('real', 'integer')
          AND json_extract(check_detail, '$.diskFree') < ?3`,
    )
    .get(
      idFloor(now - DISK_RED_WINDOW_MS - QUEUED_SLACK_MS),
      new Date(now - DISK_RED_WINDOW_MS).toISOString(),
      DISK_FREE_MIN,
    );
  if (row === null || row.n === 0 || row.free === null || row.since === null) return undefined;
  return { checks: row.n, free: row.free, since: row.since };
}
/** The card's sentence: how many checks, and how full the disk was. */
export const diskRedSentence = (r: DiskRed): string =>
  `The check machine's disk was full when ${r.checks} check${r.checks === 1 ? "" : "s"} failed: ${Math.round(r.free * 100)}% free, under the ${Math.round(DISK_FREE_MIN * 100)}% floor — the machine's red, not the pulls'; they wait for the disk, unparked.`;
@@ -53,6 +53,7 @@
import { giveHistory } from "./history.ts";
import { readLearned } from "./learned.ts";
import { keepCache, restoreCache } from "./cache.ts";
import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
import { withDisk } from "./disk_red.ts";
import { ASKED, redMain, unheld } from "./red_main.ts";
import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
import { getIssueById } from "../core/issues.ts";
@@ -617,9 +618,9 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
            ? `check passed in ${seconds}s`
            : `already proven green by ${outcome.proven}`
          : `check failed: ${outcome.error}`,
        ...(outcome.ok || outcome.detail === undefined
        ...(outcome.ok
          ? {}
          : { checkDetail: outcome.detail }),
          : { checkDetail: withDisk(outcome.detail, app.config.dataDir) }),
      });
      write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`);
      if (done) announceRun(app, done);
@@ -41,6 +41,8 @@
export interface CheckDetail {
  readonly passed: number;
  readonly failed: number;
  readonly names: readonly string[];
  /** The check machine's free disk share (0..1) when a red finished (#5453). */
  readonly diskFree?: number;
}
export interface Run {
@@ -88,6 +88,7 @@
import {
} from "./autonomy.ts";
import { checksFor } from "../checks/service.ts";
import { redMain, sharedWithMain } from "../checks/red_main.ts";
import { diskRecovered, diskRed } from "../checks/disk_red.ts";
import { sharedWithPulls } from "../checks/shared_red.ts";
import { certain } from "./decide.ts";
import { readRunLog } from "../runs/log.ts";
@@ -207,6 +208,7 @@
export async function startCrew(
  const timer = setInterval(() => {
    if (stopped) return;
    requeueInterrupted(app);
    recheckForDisk(app);
    if (Date.now() - lastAutonomy >= AUTONOMY_TICK_MS) {
      lastAutonomy = Date.now();
      settleAll(app).catch((error) =>
@@ -731,6 +733,18 @@
function waitOnMain(
  const { runId, commitOid, env } = payload;
  if (payload["kind"] !== "check" || typeof runId !== "string") return false;
  const run = getRun(app.db, runId);
  // The check machine's disk was full when it failed (#5453): the machine's red.
  if (diskRed(run)) {
    if (typeof commitOid === "string" && typeof env === "string") {
      const waiting = waitingOnDisk.get(repo.id) ?? new Map();
      waiting.set(issue.id, { env, commitOid, repo });
      waitingOnDisk.set(repo.id, waiting);
    }
    log.info("check failed while the check machine's disk was full; waiting for the disk", {
      issue: issue.number,
    });
    return true;
  }
  const main = sharedWithMain(redMain(app.db, repo.id), run);
  if (!main && !sharedWithPulls(app.db, run)) return false;
  if (typeof commitOid === "string" && typeof env === "string") {
@@ -747,6 +761,29 @@
function waitOnMain(
  return true;
}
/** Pulls whose check failed on a full disk, per repository; memory only, like {@link waitingOnMain}. */
      
        junipercrew
        20d ago
        
      
      Memory-only wait: after a restart the held pulls are never re-run and the card ages out after an hour. Next step: rebuild this map from recent disk-red runs at boot.
    
const waitingOnDisk = new Map<string, Map<string, { env: string; commitOid: string; repo: Repo }>>();
/** Once the disk is back above its floor, the pulls that waited for it are checked again. */
export function recheckForDisk(app: App): void {
  if (waitingOnDisk.size === 0 || !diskRecovered(app.config.dataDir)) return;
  const checks = checksFor(app);
  if (checks === undefined) return;
  const all = [...waitingOnDisk.values()];
  waitingOnDisk.clear();
  for (const waiting of all)
    for (const [issueId, head] of waiting) {
      const issue = getIssueById(app.db, issueId);
      const pull = issue ? getPull(app.db, head.repo.id, issue.number) : undefined;
      if (!pull || pull.state !== "open" || pull.headOid !== head.commitOid) continue;
      void checks
        .request(head.repo, head.env, head.commitOid, "retry", issueId)
        .catch((error) =>
          log.warn("a pull waiting for the disk could not be checked again", { issue: issue?.number, error: String(error) }),
        );
    }
}
/** Main's check finished: when it passed, the pulls that waited are checked again. */
function recheckWaiting(app: App, repo: Repo, state: unknown): void {
  const waiting = waitingOnMain.get(repo.id);
@@ -35,6 +35,7 @@
import { capacityValue } from "../core/capacity.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
import { freeChecks, heldChecks, redHref, redSentence } from "../checks/red_main.ts";
import { diskRedSentence, diskReds } from "../checks/disk_red.ts";
import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
/** A red in a repository other than the forge's own is not fixable from here: say so, so it reads as known. */
@@ -422,6 +423,10 @@
export function vitals(input: VitalsInput): Vitals {
  for (const shared of sharedTests(input.db, now))
    need(sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
  // Checks that failed on a full disk are one card, not a park per pull.
  const full = diskReds(input.db, now);
  if (full !== undefined) need("disk-red", diskRedSentence(full), full.since, "/settings/capacity");
  // A red restore drill, or none in eight days, is a card until a drill passes.
  const drill = lastVerdict(input.dataDir);
  const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
@@ -186,7 +186,7 @@
function checkRunWord(check: CheckSummary): { word: Word; note: string } {
    const detail = check.run?.checkDetail;
    return {
      word: "stopped",
      note: detail ? `${base} — ${detailNote(detail)}` : base,
      note: detail && detail.names.length > 0 ? `${base} — ${detailNote(detail)}` : base,
    };
  }
  if (check.state === "running")
@@ -0,0 +1,128 @@
/**
 * A red recorded while the check machine's disk was under its free floor is
 * the machine's (#5453): not reworked, not parked, checked again when the
 * disk is back above the floor, and one vitals card for all of them. The
 * disk reading is plain data, injected.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { App } from "../src/app.ts";
import { announceRun } from "../src/actions/runs.ts";
import { commitToBranch } from "../src/actions/repos.ts";
import { openPull } from "../src/actions/pulls.ts";
import { diskReading, withDisk } from "../src/checks/disk_red.ts";
import { CHECK_FAILED_LIMIT, recheckForDisk } from "../src/crew/dispatch.ts";
import { getIssueById } from "../src/core/issues.ts";
import { getPull, type Pull } from "../src/core/pulls.ts";
import type { Repo } from "../src/core/repos.ts";
import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
import type { User } from "../src/core/users.ts";
import { statDisk, vitals } from "../src/kernel/vitals.ts";
import { ErrorLedger } from "../src/kernel/log.ts";
import { RefusedLedger } from "../src/kernel/internal_refused.ts";
import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
describe("a red on a full check machine", () => {
  let standIn: StandInCrew;
  let app: App;
  let alice: User;
  let bot: User;
  let repo: Repo;
  let requested: unknown[][] = [];
  let n = 0;
  const reading = (permille: number) => () => ({ freeBytes: permille, totalBytes: 1000 });
  beforeAll(async () => {
    standIn = await startStandInCrew();
    ({ app, alice, bot, repo } = standIn);
    standIn.checks.request = async (...args: unknown[]) => {
      requested.push(args);
      return undefined;
    };
    app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
  }, 30_000);
  afterAll(() => {
    diskReading.read = statDisk;
    return stopStandInCrew(standIn);
  });
  const spentPull = async (): Promise<Pull> => {
    const branch = `disk-${++n}`;
    const c = await commitToBranch(app, alice, repo, {
      message: branch,
      branch,
      files: [{ path: `${branch}.md`, content: "x\n" }],
    });
    if (!c.ok) throw new Error(JSON.stringify(c.error));
    const o = await openPull(app, alice, repo, { title: branch, baseBranch: "main", headBranch: branch });
    if (!o.ok) throw new Error(String(o.error));
    app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
    for (let i = 0; i < CHECK_FAILED_LIMIT; i++) {
      const tried = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: o.value.id, reason: "check_failed" });
      claimRun(app.db, tried.id);
      finishRun(app.db, tried.id, { state: "ok", summary: "reworked" });
    }
    return getPull(app.db, repo.id, o.value.number)!;
  };
  /** A red check, its detail recorded the way the service does, under a given disk. */
  const failed = (pull: Pull, free: number): Run => {
    diskReading.read = reading(free);
    const run = createRun(app.db, {
      repoId: repo.id,
      kind: "check",
      env: `pull-${pull.number}`,
      commitOid: pull.headOid,
      reason: "push",
      issueId: pull.id,
    });
    claimRun(app.db, run.id);
    return finishRun(app.db, run.id, {
      state: "failed",
      summary: "check failed",
      checkDetail: withDisk({ passed: 1, failed: 1, names: [`T${n}`] }, app.config.dataDir),
    })!;
  };
  const open = (pull: Pull) => getIssueById(app.db, pull.id)?.state === "open";
  const cards = () =>
    vitals({
      db: app.db,
      dataDir: app.config.dataDir,
      disk: reading(500),
      signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
    }).degraded.filter((d) => d.cause === "disk-red");
  test("the service records the free share on a red, even one that named no test", () => {
    diskReading.read = reading(30);
    expect(withDisk(undefined, "/").diskFree).toBeCloseTo(0.03);
    expect(withDisk({ passed: 1, failed: 1, names: ["X"] }, "/")).toEqual({ passed: 1, failed: 1, names: ["X"], diskFree: 0.03 });
  });
  test("under the floor: not parked, not reworked, re-run once the disk recovers; one card", async () => {
    const pulls = [await spentPull(), await spentPull(), await spentPull()];
    for (const p of pulls) announceRun(app, failed(p, 40));
    for (const p of pulls) expect(open(p)).toBe(true);
    const found = cards();
    expect(found).toHaveLength(1);
    expect(found[0]!.sentence).toContain("when 3 checks failed");
    expect(found[0]!.sentence).toContain("4% free");
    requested = [];
    diskReading.read = reading(50);
    recheckForDisk(app);
    expect(requested).toHaveLength(0);
    diskReading.read = reading(300);
    recheckForDisk(app);
    expect(requested.map((r) => r[4]).sort()).toEqual(pulls.map((p) => p.id).sort());
    recheckForDisk(app);
    expect(requested).toHaveLength(3);
  });
  test("above the floor: a red is the pull's own, parked as ever, and no card", async () => {
    const before = cards()[0]!.sentence;
    const own = await spentPull();
    announceRun(app, failed(own, 300));
    expect(open(own)).toBe(false);
    expect(cards()[0]!.sentence).toBe(before);
  });
});
@@ -458,7 +458,9 @@
describe("checks", () => {
    expect(failed.summary).toBe("check failed: check exited with code 1");
    // `test -f` is not `bun test`: its output is not this shape, so nothing
    // is guessed at it.
    expect(failed.checkDetail).toBeNull();
    // ... but the disk's free share is recorded on every red (#5453).
    expect(failed.checkDetail?.names).toEqual([]);
    expect(typeof failed.checkDetail?.diskFree).toBe("number");
    // The check names the base tip it merged with; a default-branch push
    // merged nothing and records none (#1810).
    expect(failed.baseOid).toBe(gate.value.commitOid);
@@ -541,7 +543,7 @@
describe("checks", () => {
      (r) => r.issueId === pull.id && r.commitOid === branched.value.commitOid,
    );
    expect(failed.state).toBe("failed");
    expect(failed.checkDetail).toEqual({
    expect(failed.checkDetail).toMatchObject({
      passed: 1,
      failed: 2,
      names: ["first failing test", "second failing test"],
@@ -168,8 +168,8 @@
test("the number of queries does not grow with runs", () => {
    if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
  }
  expect(count()).toBe(before);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain, and the checks no red main holds.
  expect(before).toBe(9);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, the checks red on a full disk, and the newest drill of every domain, and the checks no red main holds.
  expect(before).toBe(10);
});
test("errors are counted, bounded, redacted, and a burst is said", async () => {
		@@ -0,0 +1,88 @@
	1	/**
	2	* A red recorded while the check machine's disk was under its free floor is
	3	* the machine's, not the pull's (#5453): a full disk sank unrelated pulls
	4	* whose builders found nothing to fix, and one such red is enough, unlike
	5	* `shared_red.ts`, which needs three pulls to agree. So the check service
	6	* records the disk's free fraction on every red run (`CheckDetail.diskFree`),
	7	* the crew neither reworks nor parks one that finished under
	8	* `DISK_FREE_MIN` and checks it again once the disk is back above the floor,
	9	* and the vitals carry one card for all of them.
	10	*
	11	* Plain data: a reading in, a verdict out; the one place the disk is read is
	12	* {@link diskReading}, which a test replaces.
	13	*/
	14
	15	import type { Database } from "bun:sqlite";
	16	import type { CheckDetail, Run } from "../core/runs.ts";
	17	import { idFloor } from "../kernel/id.ts";
	18	import { DISK_FREE_MIN, statDisk, type DiskReading } from "../kernel/vitals.ts";
	19
	20	/** How far back a disk-full red still counts for the card. */
	21	export const DISK_RED_WINDOW_MS = 3_600_000;
	22
	23	/** Checks are created before they finish; see `shared_red.ts`. */
	24	const QUEUED_SLACK_MS = 6 * 3_600_000;
	25
	26	/** Where the disk is read from; a test swaps `read` for plain data. */
	27	export const diskReading: { read: (dir: string) => DiskReading } = { read: statDisk };
	28
	29	/** The free share of a reading, 0..1; undefined for a volume that reports no size. */
	30	export const freeFraction = (d: DiskReading): number \| undefined =>
	31	d.totalBytes > 0 ? d.freeBytes / d.totalBytes : undefined;
	32
	33	/** Whether a free share is under the floor. */
	34	export const underFloor = (free: number \| undefined): boolean =>
	35	free !== undefined && free < DISK_FREE_MIN;
	36
	37	/** A failed check's detail with the disk's free share recorded on it. */
	38	export function withDisk(detail: CheckDetail \| undefined, dataDir: string): CheckDetail {
	39	const free = freeFraction(diskReading.read(dataDir));
	40	const base = detail ?? { passed: 0, failed: 0, names: [] };
	41	return free === undefined ? base : { ...base, diskFree: free };
	42	}
	43
	44	/** Whether a run finished red while the disk was under its floor. */
	45	export const diskRed = (run: Pick<Run, "checkDetail"> \| undefined): boolean =>
	46	underFloor(run?.checkDetail?.diskFree);
	47
	48	/** Whether the disk under `dataDir` is back above the floor. */
	49	export const diskRecovered = (dataDir: string): boolean => {
	50	const free = freeFraction(diskReading.read(dataDir));
	51	return free === undefined \|\| free >= DISK_FREE_MIN;
	52	};
	53
	54	export interface DiskRed {
	55	/** Pull checks that failed with the disk under the floor in the window. */
	56	readonly checks: number;
	57	/** The lowest free share any recorded. */
	58	readonly free: number;
	59	/** When the earliest finished. */
	60	readonly since: string;
	61	}
	62
	63	/** The disk-full reds of the last hour, or undefined when there are none. */
	64	export function diskReds(db: Database, now: number = Date.now()): DiskRed \| undefined {
	65	const row = db
	66	.query<{ n: number; free: number \| null; since: string \| null }, [string, string, number]>(
	67	`SELECT COUNT(*) AS n,
	68	MIN(json_extract(check_detail, '$.diskFree')) AS free,
	69	MIN(finished_at) AS since
	70	FROM run
	71	WHERE id >= ?1 AND finished_at >= ?2
	72	AND kind = 'check' AND issue_id IS NOT NULL AND state = 'failed'
	73	AND json_valid(check_detail)
	74	AND json_type(check_detail, '$.diskFree') IN ('real', 'integer')
	75	AND json_extract(check_detail, '$.diskFree') < ?3`,
	76	)
	77	.get(
	78	idFloor(now - DISK_RED_WINDOW_MS - QUEUED_SLACK_MS),
	79	new Date(now - DISK_RED_WINDOW_MS).toISOString(),
	80	DISK_FREE_MIN,
	81	);
	82	if (row === null \|\| row.n === 0 \|\| row.free === null \|\| row.since === null) return undefined;
	83	return { checks: row.n, free: row.free, since: row.since };
	84	}
	85
	86	/** The card's sentence: how many checks, and how full the disk was. */
	87	export const diskRedSentence = (r: DiskRed): string =>
	88	`The check machine's disk was full when ${r.checks} check${r.checks === 1 ? "" : "s"} failed: ${Math.round(r.free * 100)}% free, under the ${Math.round(DISK_FREE_MIN * 100)}% floor — the machine's red, not the pulls'; they wait for the disk, unparked.`;
		@@ -53,6 +53,7 @@ import { giveHistory } from "./history.ts";
53	53	import { readLearned } from "./learned.ts";
54	54	import { keepCache, restoreCache } from "./cache.ts";
55	55	import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
	56	import { withDisk } from "./disk_red.ts";
56	57	import { ASKED, redMain, unheld } from "./red_main.ts";
57	58	import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
58	59	import { getIssueById } from "../core/issues.ts";
		@@ -617,9 +618,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
617	618	? `check passed in ${seconds}s`
618	619	: `already proven green by ${outcome.proven}`
619	620	: `check failed: ${outcome.error}`,
620		...(outcome.ok \|\| outcome.detail === undefined
	621	...(outcome.ok
621	622	? {}
622		: { checkDetail: outcome.detail }),
	623	: { checkDetail: withDisk(outcome.detail, app.config.dataDir) }),
623	624	});
624	625	write(outcome.ok ? `passed in ${seconds}s` : `failed: ${outcome.error}`);
625	626	if (done) announceRun(app, done);
		@@ -41,6 +41,8 @@ export interface CheckDetail {
41	41	readonly passed: number;
42	42	readonly failed: number;
43	43	readonly names: readonly string[];
	44	/** The check machine's free disk share (0..1) when a red finished (#5453). */
	45	readonly diskFree?: number;
44	46	}
45	47
46	48	export interface Run {
		@@ -88,6 +88,7 @@ import {
88	88	} from "./autonomy.ts";
89	89	import { checksFor } from "../checks/service.ts";
90	90	import { redMain, sharedWithMain } from "../checks/red_main.ts";
	91	import { diskRecovered, diskRed } from "../checks/disk_red.ts";
91	92	import { sharedWithPulls } from "../checks/shared_red.ts";
92	93	import { certain } from "./decide.ts";
93	94	import { readRunLog } from "../runs/log.ts";
		@@ -207,6 +208,7 @@ export async function startCrew(
207	208	const timer = setInterval(() => {
208	209	if (stopped) return;
209	210	requeueInterrupted(app);
	211	recheckForDisk(app);
210	212	if (Date.now() - lastAutonomy >= AUTONOMY_TICK_MS) {
211	213	lastAutonomy = Date.now();
212	214	settleAll(app).catch((error) =>
		@@ -731,6 +733,18 @@ function waitOnMain(
731	733	const { runId, commitOid, env } = payload;
732	734	if (payload["kind"] !== "check" \|\| typeof runId !== "string") return false;
733	735	const run = getRun(app.db, runId);
	736	// The check machine's disk was full when it failed (#5453): the machine's red.
	737	if (diskRed(run)) {
	738	if (typeof commitOid === "string" && typeof env === "string") {
	739	const waiting = waitingOnDisk.get(repo.id) ?? new Map();
	740	waiting.set(issue.id, { env, commitOid, repo });
	741	waitingOnDisk.set(repo.id, waiting);
	742	}
	743	log.info("check failed while the check machine's disk was full; waiting for the disk", {
	744	issue: issue.number,
	745	});
	746	return true;
	747	}
734	748	const main = sharedWithMain(redMain(app.db, repo.id), run);
735	749	if (!main && !sharedWithPulls(app.db, run)) return false;
736	750	if (typeof commitOid === "string" && typeof env === "string") {
		@@ -747,6 +761,29 @@ function waitOnMain(
747	761	return true;
748	762	}
749	763
	764	/** Pulls whose check failed on a full disk, per repository; memory only, like {@link waitingOnMain}. */
		junipercrew 20d ago Memory-only wait: after a restart the held pulls are never re-run and the card ages out after an hour. Next step: rebuild this map from recent disk-red runs at boot.
	765	const waitingOnDisk = new Map<string, Map<string, { env: string; commitOid: string; repo: Repo }>>();
	766
	767	/** Once the disk is back above its floor, the pulls that waited for it are checked again. */
	768	export function recheckForDisk(app: App): void {
	769	if (waitingOnDisk.size === 0 \|\| !diskRecovered(app.config.dataDir)) return;
	770	const checks = checksFor(app);
	771	if (checks === undefined) return;
	772	const all = [...waitingOnDisk.values()];
	773	waitingOnDisk.clear();
	774	for (const waiting of all)
	775	for (const [issueId, head] of waiting) {
	776	const issue = getIssueById(app.db, issueId);
	777	const pull = issue ? getPull(app.db, head.repo.id, issue.number) : undefined;
	778	if (!pull \|\| pull.state !== "open" \|\| pull.headOid !== head.commitOid) continue;
	779	void checks
	780	.request(head.repo, head.env, head.commitOid, "retry", issueId)
	781	.catch((error) =>
	782	log.warn("a pull waiting for the disk could not be checked again", { issue: issue?.number, error: String(error) }),
	783	);
	784	}
	785	}
	786
750	787	/** Main's check finished: when it passed, the pulls that waited are checked again. */
751	788	function recheckWaiting(app: App, repo: Repo, state: unknown): void {
752	789	const waiting = waitingOnMain.get(repo.id);
		@@ -35,6 +35,7 @@ import { capacityValue } from "../core/capacity.ts";
35	35	import { DOMAINS } from "./domains.ts";
36	36	import { domainDrillNeeds } from "./domain_drill.ts";
37	37	import { freeChecks, heldChecks, redHref, redSentence } from "../checks/red_main.ts";
	38	import { diskRedSentence, diskReds } from "../checks/disk_red.ts";
38	39	import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
39	40
40	41	/** A red in a repository other than the forge's own is not fixable from here: say so, so it reads as known. */
		@@ -422,6 +423,10 @@ export function vitals(input: VitalsInput): Vitals {
422	423	for (const shared of sharedTests(input.db, now))
423	424	need(sharedCause(shared), sharedSentence(shared) + foreignNote(shared.repo), shared.since, flakeHref(input.db, shared));
424	425
	426	// Checks that failed on a full disk are one card, not a park per pull.
	427	const full = diskReds(input.db, now);
	428	if (full !== undefined) need("disk-red", diskRedSentence(full), full.since, "/settings/capacity");
	429
425	430	// A red restore drill, or none in eight days, is a card until a drill passes.
426	431	const drill = lastVerdict(input.dataDir);
427	432	const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
		@@ -186,7 +186,7 @@ function checkRunWord(check: CheckSummary): { word: Word; note: string } {
186	186	const detail = check.run?.checkDetail;
187	187	return {
188	188	word: "stopped",
189		note: detail ? `${base} — ${detailNote(detail)}` : base,
	189	note: detail && detail.names.length > 0 ? `${base} — ${detailNote(detail)}` : base,
190	190	};
191	191	}
192	192	if (check.state === "running")
		@@ -0,0 +1,128 @@
	1	/**
	2	* A red recorded while the check machine's disk was under its free floor is
	3	* the machine's (#5453): not reworked, not parked, checked again when the
	4	* disk is back above the floor, and one vitals card for all of them. The
	5	* disk reading is plain data, injected.
	6	*/
	7
	8	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	9	import type { App } from "../src/app.ts";
	10	import { announceRun } from "../src/actions/runs.ts";
	11	import { commitToBranch } from "../src/actions/repos.ts";
	12	import { openPull } from "../src/actions/pulls.ts";
	13	import { diskReading, withDisk } from "../src/checks/disk_red.ts";
	14	import { CHECK_FAILED_LIMIT, recheckForDisk } from "../src/crew/dispatch.ts";
	15	import { getIssueById } from "../src/core/issues.ts";
	16	import { getPull, type Pull } from "../src/core/pulls.ts";
	17	import type { Repo } from "../src/core/repos.ts";
	18	import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
	19	import type { User } from "../src/core/users.ts";
	20	import { statDisk, vitals } from "../src/kernel/vitals.ts";
	21	import { ErrorLedger } from "../src/kernel/log.ts";
	22	import { RefusedLedger } from "../src/kernel/internal_refused.ts";
	23	import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
	24
	25	describe("a red on a full check machine", () => {
	26	let standIn: StandInCrew;
	27	let app: App;
	28	let alice: User;
	29	let bot: User;
	30	let repo: Repo;
	31	let requested: unknown[][] = [];
	32	let n = 0;
	33	const reading = (permille: number) => () => ({ freeBytes: permille, totalBytes: 1000 });
	34
	35	beforeAll(async () => {
	36	standIn = await startStandInCrew();
	37	({ app, alice, bot, repo } = standIn);
	38	standIn.checks.request = async (...args: unknown[]) => {
	39	requested.push(args);
	40	return undefined;
	41	};
	42	app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
	43	}, 30_000);
	44	afterAll(() => {
	45	diskReading.read = statDisk;
	46	return stopStandInCrew(standIn);
	47	});
	48
	49	const spentPull = async (): Promise<Pull> => {
	50	const branch = `disk-${++n}`;
	51	const c = await commitToBranch(app, alice, repo, {
	52	message: branch,
	53	branch,
	54	files: [{ path: `${branch}.md`, content: "x\n" }],
	55	});
	56	if (!c.ok) throw new Error(JSON.stringify(c.error));
	57	const o = await openPull(app, alice, repo, { title: branch, baseBranch: "main", headBranch: branch });
	58	if (!o.ok) throw new Error(String(o.error));
	59	app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
	60	for (let i = 0; i < CHECK_FAILED_LIMIT; i++) {
	61	const tried = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: o.value.id, reason: "check_failed" });
	62	claimRun(app.db, tried.id);
	63	finishRun(app.db, tried.id, { state: "ok", summary: "reworked" });
	64	}
	65	return getPull(app.db, repo.id, o.value.number)!;
	66	};
	67	/** A red check, its detail recorded the way the service does, under a given disk. */
	68	const failed = (pull: Pull, free: number): Run => {
	69	diskReading.read = reading(free);
	70	const run = createRun(app.db, {
	71	repoId: repo.id,
	72	kind: "check",
	73	env: `pull-${pull.number}`,
	74	commitOid: pull.headOid,
	75	reason: "push",
	76	issueId: pull.id,
	77	});
	78	claimRun(app.db, run.id);
	79	return finishRun(app.db, run.id, {
	80	state: "failed",
	81	summary: "check failed",
	82	checkDetail: withDisk({ passed: 1, failed: 1, names: [`T${n}`] }, app.config.dataDir),
	83	})!;
	84	};
	85	const open = (pull: Pull) => getIssueById(app.db, pull.id)?.state === "open";
	86	const cards = () =>
	87	vitals({
	88	db: app.db,
	89	dataDir: app.config.dataDir,
	90	disk: reading(500),
	91	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
	92	}).degraded.filter((d) => d.cause === "disk-red");
	93
	94	test("the service records the free share on a red, even one that named no test", () => {
	95	diskReading.read = reading(30);
	96	expect(withDisk(undefined, "/").diskFree).toBeCloseTo(0.03);
	97	expect(withDisk({ passed: 1, failed: 1, names: ["X"] }, "/")).toEqual({ passed: 1, failed: 1, names: ["X"], diskFree: 0.03 });
	98	});
	99
	100	test("under the floor: not parked, not reworked, re-run once the disk recovers; one card", async () => {
	101	const pulls = [await spentPull(), await spentPull(), await spentPull()];
	102	for (const p of pulls) announceRun(app, failed(p, 40));
	103	for (const p of pulls) expect(open(p)).toBe(true);
	104
	105	const found = cards();
	106	expect(found).toHaveLength(1);
	107	expect(found[0]!.sentence).toContain("when 3 checks failed");
	108	expect(found[0]!.sentence).toContain("4% free");
	109
	110	requested = [];
	111	diskReading.read = reading(50);
	112	recheckForDisk(app);
	113	expect(requested).toHaveLength(0);
	114	diskReading.read = reading(300);
	115	recheckForDisk(app);
	116	expect(requested.map((r) => r[4]).sort()).toEqual(pulls.map((p) => p.id).sort());
	117	recheckForDisk(app);
	118	expect(requested).toHaveLength(3);
	119	});
	120
	121	test("above the floor: a red is the pull's own, parked as ever, and no card", async () => {
	122	const before = cards()[0]!.sentence;
	123	const own = await spentPull();
	124	announceRun(app, failed(own, 300));
	125	expect(open(own)).toBe(false);
	126	expect(cards()[0]!.sentence).toBe(before);
	127	});
	128	});
		@@ -458,7 +458,9 @@ describe("checks", () => {
458	458	expect(failed.summary).toBe("check failed: check exited with code 1");
459	459	// `test -f` is not `bun test`: its output is not this shape, so nothing
460	460	// is guessed at it.
461		expect(failed.checkDetail).toBeNull();
	461	// ... but the disk's free share is recorded on every red (#5453).
	462	expect(failed.checkDetail?.names).toEqual([]);
	463	expect(typeof failed.checkDetail?.diskFree).toBe("number");
462	464	// The check names the base tip it merged with; a default-branch push
463	465	// merged nothing and records none (#1810).
464	466	expect(failed.baseOid).toBe(gate.value.commitOid);
		@@ -541,7 +543,7 @@ describe("checks", () => {
541	543	(r) => r.issueId === pull.id && r.commitOid === branched.value.commitOid,
542	544	);
543	545	expect(failed.state).toBe("failed");
544		expect(failed.checkDetail).toEqual({
	546	expect(failed.checkDetail).toMatchObject({
545	547	passed: 1,
546	548	failed: 2,
547	549	names: ["first failing test", "second failing test"],
		@@ -168,8 +168,8 @@ test("the number of queries does not grow with runs", () => {
168	168	if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
169	169	}
170	170	expect(count()).toBe(before);
171		// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain, and the checks no red main holds.
172		expect(before).toBe(9);
	171	// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, the checks red on a full disk, and the newest drill of every domain, and the checks no red main holds.
	172	expect(before).toBe(10);
173	173	});
174	174
175	175	test("errors are counted, bounded, redacted, and a burst is said", async () => {