5e7d948 · c1715fa
@@ -0,0 +1,145 @@
/**
 * A red many pulls share is the lane's, not theirs (#2266). One test whose
 * verdict hung on how full the check machine's disk was failed ten pulls in
 * an hour while main's own check stayed green, so the red-main rule
 * (`checks/red_main.ts`) never applied: each was parked, its issue struck,
 * and every one reopened by hand. So when a pull's check fails only on tests
 * that {@link SHARED_PULLS} or more pulls of the repository failed within
 * {@link SHARED_WINDOW_MS}, the crew neither reworks nor parks it; it waits
 * for main's next green check, as a red main's pulls do, and the vitals
 * carry one card per shared test in place of a park per pull.
 *
 * Plain data: failed pull checks in the window, read by the `run_repo`
 * index (an id is time-sortable, so the window is a range), their recorded
 * test names unpacked by SQLite's own `json_each`.
 */
import type { Database } from "bun:sqlite";
import type { Run } from "../core/runs.ts";
import { idFloor } from "../kernel/id.ts";
/** How far back another pull's red on the same test counts. */
export const SHARED_WINDOW_MS = 3_600_000;
/** How many pulls, this one among them, failing one test make it shared. */
export const SHARED_PULLS = 3;
/**
 * Checks are created before they finish; one created this long before the
 * window may still have failed inside it. Bounds the scan, never the answer
 * for any check that ran under its lane's own limits.
 */
const QUEUED_SLACK_MS = 6 * 3_600_000;
export interface SharedTest {
  readonly repoId: string;
  /** `owner/name`, for the card. */
  readonly repo: string;
  readonly test: string;
  /** Distinct pulls whose check failed on it in the window. */
  readonly pulls: number;
  /** When the earliest of those checks finished. */
  readonly since: string;
}
interface Row {
  repo_id: string;
  repo: string;
  test: string;
  pulls: number;
  since: string;
}
/**
 * Each test failed by pull checks in the window, with how many distinct
 * pulls failed it. `repo` narrows to one repository; `test` to one name.
 * A constant of this module, never input: every value is a parameter.
 */
const PULLS_PER_TEST = `
  SELECT run.repo_id, u.username || '/' || repo.name AS repo,
         n.value AS test, COUNT(DISTINCT run.issue_id) AS pulls,
         MIN(run.finished_at) AS since
    FROM run
    JOIN repo ON repo.id = run.repo_id
    JOIN user u ON u.id = repo.owner_id,
         json_each(CASE WHEN json_valid(run.check_detail) THEN run.check_detail ELSE '{}' END, '$.names') n
   WHERE run.id >= ?1 AND run.finished_at >= ?2
     AND run.kind = 'check' AND run.issue_id IS NOT NULL AND run.state = 'failed'
     AND (?3 IS NULL OR run.repo_id = ?3)
     AND n.type = 'text'
   GROUP BY run.repo_id, n.value`;
function pullsPerTest(db: Database, now: number, repoId: string | null): Row[] {
  return db
    .query<Row, [string, string, string | null]>(PULLS_PER_TEST)
    .all(
      idFloor(now - SHARED_WINDOW_MS - QUEUED_SLACK_MS),
      new Date(now - SHARED_WINDOW_MS).toISOString(),
      repoId,
    );
}
/**
 * Whether a failed check fails only what the lane fails: it named at least
 * one test, and every one was failed by {@link SHARED_PULLS} or more pulls
 * (this one counted, once it is recorded) in the window. A check that named
 * nothing, or one test few others fail, is the pull's own.
 */
export function sharedWithPulls(
  db: Database,
  run: Pick<Run, "repoId" | "checkDetail"> | undefined,
  now: number = Date.now(),
): boolean {
  const names = run?.checkDetail?.names ?? [];
  if (run === undefined || names.length === 0) return false;
  const pulls = new Map(pullsPerTest(db, now, run.repoId).map((r) => [r.test, r.pulls]));
  return names.every((n) => (pulls.get(n) ?? 0) >= SHARED_PULLS);
}
/** Every shared test across the forge's repositories, oldest first: one query. */
export function sharedTests(db: Database, now: number = Date.now()): SharedTest[] {
  return pullsPerTest(db, now, null)
    .filter((r) => r.pulls >= SHARED_PULLS)
    .sort((a, b) => a.since.localeCompare(b.since))
    .map((r) => ({ repoId: r.repo_id, repo: r.repo, test: r.test, pulls: r.pulls, since: r.since }));
}
/**
 * The vitals cause for one shared test: `shared:<repoId>:<hash>`, stable
 * for the test, short enough that the sentinel's finding key
 * ({@link flakeKey}) keeps all of it.
 */
export function sharedCause(t: Pick<SharedTest, "repoId" | "test">): string {
  return `shared:${t.repoId}:${Bun.hash(t.test).toString(36).slice(0, 10)}`;
}
/**
 * The sentinel's finding key for a cause, `degraded:${keyPart(cause)}`
 * (`crew/sentinel.ts`, which reads the vitals and so is not imported here):
 * a shared cause is already lowercase base32 and base36, so only `:` moves.
 */
export const flakeKey = (cause: string): string => `degraded:${cause.replaceAll(":", "-")}`;
/**
 * Where the card points: the open flake issue the sentinel filed for this
 * test, to follow it, else the repository's new-issue page, to file one.
 */
export function flakeHref(db: Database, t: SharedTest): string {
  const filed = db
    .query<{ repo: string; number: number }, [string]>(
      `SELECT u.username || '/' || repo.name AS repo, i.number
         FROM sentinel_finding f
         JOIN issue i ON i.id = f.issue_id
         JOIN repo ON repo.id = i.repo_id
         JOIN user u ON u.id = repo.owner_id
        WHERE f.key = ?1 AND i.state = 'open'
        LIMIT 1`,
    )
    .get(flakeKey(sharedCause(t)));
  return filed === null ? `/${t.repo}/issues/new` : `/${filed.repo}/issues/${filed.number}`;
}
/** The card's sentence: which test, on how many pulls, and that they wait. */
export function sharedSentence(t: SharedTest): string {
  return `${t.repo}: "${t.test}" failed on ${t.pulls} pulls in the last hour — the lane's red, not theirs; they wait for main's next green check, unparked.`;
}
@@ -84,6 +84,7 @@
import {
} from "./autonomy.ts";
import { checksFor } from "../checks/service.ts";
import { redMain, sharedWithMain } from "../checks/red_main.ts";
import { sharedWithPulls } from "../checks/shared_red.ts";
import { certain } from "./decide.ts";
import { readRunLog } from "../runs/log.ts";
import { whenText } from "./limits.ts";
@@ -588,8 +589,9 @@
function route(app: App, event: ForgeEvent): void {
      // is the agent's to look at.
      const agent = agentFor(app, issue);
      if (!agent) return;
      // A red that main shares is not the pull's: no rework, no park, no
      // strike; it waits and is checked again when main is green (#1971).
      // A red that main shares (#1971), or that three pulls share within
      // the hour (#2266), is not the pull's: no rework, no park, no
      // strike; it waits and is checked again when main is green.
      if (waitOnMain(app, repo, issue, event.payload)) return;
      // Bounded: after a few failed checks in a row the agent stops and a
      // person must speak (docs/APPS.md, rule 7).
@@ -674,13 +676,14 @@
function route(app: App, event: ForgeEvent): void {
}
/**
 * Pulls whose check failed only on what main fails, per repository, with
 * Pulls whose check failed only on what main fails, or on what the lane
 * fails across pulls (`checks/shared_red.ts`), per repository, with
 * the head that was checked. Memory only: a restart forgets them, and the
 * pull's next push or a person's ask checks it again.
 */
const waitingOnMain = new Map<string, Map<string, { env: string; commitOid: string }>>();
/** Whether a failed check is main's red, shared: if so, the pull is noted as waiting. */
/** Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting. */
function waitOnMain(
  app: App,
  repo: Repo,
@@ -689,15 +692,20 @@
function waitOnMain(
): boolean {
  const { runId, commitOid, env } = payload;
  if (payload["kind"] !== "check" || typeof runId !== "string") return false;
  if (!sharedWithMain(redMain(app.db, repo.id), getRun(app.db, runId))) return false;
  const run = getRun(app.db, runId);
  const main = sharedWithMain(redMain(app.db, repo.id), run);
  if (!main && !sharedWithPulls(app.db, run)) return false;
  if (typeof commitOid === "string" && typeof env === "string") {
    const waiting = waitingOnMain.get(repo.id) ?? new Map();
    waiting.set(issue.id, { env, commitOid });
    waitingOnMain.set(repo.id, waiting);
  }
  log.info("check failed only what main fails; waiting for main to be green", {
    issue: issue.number,
  });
  log.info(
    main
      ? "check failed only what main fails; waiting for main to be green"
      : "check failed only what other pulls fail this hour; waiting for main to be green",
    { issue: issue.number },
  );
  return true;
}
@@ -407,10 +407,12 @@
export function degradedCauses(db: Database, now: number): Finding[] {
/**
 * The causes that are a budget breached: the lane, a silent run, a run
 * killed twice, a red main holding its pulls' checks. The last gets one fix
 * issue (#1971), and `crew/red_fix.ts` keeps other builders off its test.
 * killed twice, a red main holding its pulls' checks, a test red on many
 * pulls at once. A red main gets one fix issue (#1971), and
 * `crew/red_fix.ts` keeps other builders off its test; a shared test gets
 * one flake issue (#2266), which its vitals card links to.
 */
const BREACH_CAUSE = /^(queue:check|stuck:|budget:|red:)/;
const BREACH_CAUSE = /^(queue:check|stuck:|budget:|red:|shared:)/;
/**
 * Each budget breach that holds right now, however briefly: the crew hears
@@ -435,7 +437,9 @@
export function budgetBreaches(db: Database, now: number): Finding[] {
      lookFor: `\`${s.cause}\` is gone from the degraded list and the cause is fixed, not waited out.`,
      fix: s.cause.startsWith("red:")
        ? "Fix the failing test on main in one pull. This is the only issue for it: pulls red on the same test wait for main, and no other builder is handed it while this is open."
        : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
        : s.cause.startsWith("shared:")
          ? "Make the named test's verdict depend only on the code under test, not on the check machine's load, disk or clock, in one pull. Pulls red on it alone wait, unparked, for main's next green check."
          : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
      urgent: true,
    });
  }
@@ -51,6 +51,16 @@
export function id(now: number = Date.now()): string {
  return out;
}
/**
 * The least id `id()` hands out at `ms`: every id made at or after it sorts
 * at or above this, so a time window over an id-indexed table is a range.
 */
export function idFloor(ms: number): string {
  let out = "";
  for (let shift = 45; shift >= 0; shift -= 5) out += alphabet[Math.floor(Math.max(0, ms) / 2 ** shift) % 32];
  return out + "0".repeat(16);
}
/** 256 bits of entropy, base64url, ~43 chars. For tokens and session ids. */
export function secret(): string {
  const bytes = new Uint8Array(32);
@@ -28,6 +28,7 @@
import { newestDrills } from "../core/domain_drills.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
import { heldChecks, redSentence } from "../checks/red_main.ts";
import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
/** Free disk under this share of the volume is something to say. */
export const DISK_FREE_MIN = 0.1;
@@ -49,7 +50,7 @@
export interface EnvCount {
/**
 * One cause of the forge being off: the `needs` sentence, a `cause` key that
 * holds for as long as its condition does (`disk`, `errors`, `internal`,
 * `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`), when it began by the forge's own evidence
 * `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * (the oldest evidence the forge holds, else the first time this process saw
 * it, held until the cause clears; `at` is never the answer to a repeat
 * call), and where to look.
@@ -363,6 +364,10 @@
export function vitals(input: VitalsInput): Vitals {
      `/${red.repo}/runs/${red.runId}`,
    );
  // A test failing on many pulls in the hour is one card, not a park per pull.
  for (const shared of sharedTests(input.db, now))
    need(sharedCause(shared), sharedSentence(shared), shared.since, flakeHref(input.db, shared));
  // A red restore drill, or none in eight days, is a card until a drill passes.
  const drill = lastVerdict(input.dataDir);
  const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
@@ -0,0 +1,122 @@
/**
 * A red many pulls share never parks them (#2266): through the crew's event
 * path, with main green, three pulls whose checks fail one test in the hour
 * are neither parked nor reworked and are checked again on main's next green
 * check; a pull red on a test only it fails is parked as ever. Each pull has
 * already spent its automatic tries, so any red that is its own parks it.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { App } from "../src/app.ts";
import { announceRun } from "../src/actions/runs.ts";
import { commitToBranch } from "../src/actions/repos.ts";
import { openPull } from "../src/actions/pulls.ts";
import { CHECK_FAILED_LIMIT } from "../src/crew/dispatch.ts";
import { getIssueById } from "../src/core/issues.ts";
import { getPull, type Pull } from "../src/core/pulls.ts";
import type { Repo } from "../src/core/repos.ts";
import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
import type { User } from "../src/core/users.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { ErrorLedger } from "../src/kernel/log.ts";
import { RefusedLedger } from "../src/kernel/internal_refused.ts";
import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
describe("the crew, with a red many pulls share", () => {
  let standIn: StandInCrew;
  let app: App;
  let alice: User;
  let bot: User;
  let repo: Repo;
  let requested: unknown[][] = [];
  let n = 0;
  beforeAll(async () => {
    standIn = await startStandInCrew();
    ({ app, alice, bot, repo } = standIn);
    standIn.checks.request = async (...args: unknown[]) => {
      requested.push(args);
      return undefined;
    };
    // Parking is autonomy's: the repository runs on its own, on alice's word.
    app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
  }, 30_000);
  afterAll(() => stopStandInCrew(standIn));
  /** A pull of the crew's whose automatic tries are spent: its own next red parks it. */
  const spentPull = async (): Promise<Pull> => {
    const branch = `feat-${++n}`;
    const c = await commitToBranch(app, alice, repo, {
      message: branch,
      branch,
      files: [{ path: `${branch}.md`, content: "x\n" }],
    });
    if (!c.ok) throw new Error(JSON.stringify(c.error));
    const o = await openPull(app, alice, repo, { title: branch, baseBranch: "main", headBranch: branch });
    if (!o.ok) throw new Error(String(o.error));
    app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
    for (let i = 0; i < CHECK_FAILED_LIMIT; i++) {
      const tried = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: o.value.id, reason: "check_failed" });
      claimRun(app.db, tried.id);
      finishRun(app.db, tried.id, { state: "ok", summary: "reworked" });
    }
    return getPull(app.db, repo.id, o.value.number)!;
  };
  const failed = (pull: Pull, names: string[]): Run => {
    const run = createRun(app.db, {
      repoId: repo.id,
      kind: "check",
      env: `pull-${pull.number}`,
      commitOid: pull.headOid,
      reason: "push",
      issueId: pull.id,
    });
    claimRun(app.db, run.id);
    return finishRun(app.db, run.id, {
      state: "failed",
      summary: "check failed",
      checkDetail: { passed: 1, failed: names.length, names },
    })!;
  };
  const mainGreen = () => {
    const run = createRun(app.db, { repoId: repo.id, kind: "check", env: "main", commitOid: "a".repeat(40), reason: "push" });
    claimRun(app.db, run.id);
    announceRun(app, finishRun(app.db, run.id, { state: "ok", summary: "check passed" })!);
  };
  const open = (pull: Pull) => getIssueById(app.db, pull.id)?.state === "open";
  const sharedCards = () =>
    vitals({
      db: app.db,
      dataDir: app.config.dataDir,
      signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
    }).degraded.filter((d) => d.cause.startsWith("shared:"));
  test("three pulls red on X in the hour: none parked, one card; a fourth red on Y only is parked", async () => {
    mainGreen();
    const pulls = [await spentPull(), await spentPull(), await spentPull()];
    const runs = pulls.map((p) => failed(p, ["X"]));
    for (const run of runs) announceRun(app, run);
    for (const p of pulls) expect(open(p)).toBe(true);
    const cards = sharedCards();
    expect(cards).toHaveLength(1);
    expect(cards[0]!.sentence).toContain('"X" failed on 3 pulls in the last hour');
    expect(cards[0]!.href).toBe(`/${alice.username}/${repo.name}/issues/new`);
    const own = await spentPull();
    announceRun(app, failed(own, ["Y"]));
    expect(open(own)).toBe(false);
    // X and Y together is not only the lane's red: Y is the pull's own.
    const mixed = await spentPull();
    announceRun(app, failed(mixed, ["X", "Y"]));
    expect(open(mixed)).toBe(false);
    expect(sharedCards()).toHaveLength(1);
    // Main's next green check sends the three back through the lane, once.
    requested = [];
    mainGreen();
    expect(requested.map((r) => r[4]).sort()).toEqual(pulls.map((p) => p.id).sort());
    mainGreen();
    expect(requested).toHaveLength(3);
  });
});
@@ -0,0 +1,136 @@
/**
 * A red many pulls share (#2266), on in-memory SQLite: a test failed by
 * three pulls in the hour is the lane's, one failed by fewer, or failed
 * longer ago, is the pull's own; the vitals carry one card per shared test,
 * linked to its flake issue once the sentinel has filed one.
 */
import { beforeEach, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
import { tmpdir } from "node:os";
import { migrations } from "../src/core/schema.ts";
import { openDb } from "../src/kernel/db.ts";
import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
import {
  SHARED_PULLS,
  SHARED_WINDOW_MS,
  flakeHref,
  flakeKey,
  sharedCause,
  sharedTests,
  sharedWithPulls,
} from "../src/checks/shared_red.ts";
import { budgetBreaches, keyPart } from "../src/crew/sentinel.ts";
import { id, idFloor } from "../src/kernel/id.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { ErrorLedger } from "../src/kernel/log.ts";
import { RefusedLedger } from "../src/kernel/internal_refused.ts";
let db: Database;
beforeEach(() => {
  db = openDb(":memory:", migrations);
  db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')");
  db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1', 'u1', 'forge')");
  db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r2', 'u1', 'other')");
  for (let i = 1; i <= 6; i++)
    db.run(
      `INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i${i}', 'r1', ${i}, 'pull', 'p', 'u1')`,
    );
  db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('j1', 'r2', 1, 'pull', 'p', 'u1')");
});
const red = (issueId: string, names: string[], repoId = "r1"): Run => {
  const run = createRun(db, { repoId, kind: "check", env: `pull-${issueId}`, issueId, commitOid: "a".repeat(40), reason: "push" });
  claimRun(db, run.id);
  return finishRun(db, run.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: names.length, names } })!;
};
const shared = (run: Run | undefined, now?: number) => sharedWithPulls(db, run, now);
describe("a red many pulls share", () => {
  test("three pulls on one test share it; two do not", () => {
    red("i1", ["X"]);
    const second = red("i2", ["X"]);
    expect(shared(second)).toBe(false);
    const third = red("i3", ["X"]);
    expect(SHARED_PULLS).toBe(3);
    expect(shared(third)).toBe(true);
    expect(shared(second)).toBe(true);
  });
  test("one pull red three times is its own red, not the lane's", () => {
    red("i1", ["X"]);
    red("i1", ["X"]);
    expect(shared(red("i1", ["X"]))).toBe(false);
  });
  test("every test the check names must be shared; naming none is the pull's own", () => {
    for (const i of ["i1", "i2", "i3"]) red(i, ["X"]);
    expect(shared(red("i4", ["X", "Y"]))).toBe(false);
    expect(shared(red("i5", ["Y"]))).toBe(false);
    expect(shared(red("i6", []))).toBe(false);
    expect(shared(undefined)).toBe(false);
  });
  test("another repository's reds, main's own, and reds older than the hour count for nothing", () => {
    red("j1", ["X"], "r2");
    const main = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: "b".repeat(40), reason: "push" });
    claimRun(db, main.id);
    finishRun(db, main.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: 1, names: ["X"] } });
    red("i1", ["X"]);
    expect(shared(red("i2", ["X"]))).toBe(false);
    const third = red("i3", ["X"]);
    expect(shared(third)).toBe(true);
    expect(shared(third, Date.now() + SHARED_WINDOW_MS + 1_000)).toBe(false);
  });
  test("a check detail that is not JSON is skipped, not thrown on", () => {
    for (const i of ["i1", "i2"]) red(i, ["X"]);
    const broken = red("i3", ["X"]);
    db.run("UPDATE run SET check_detail = '{not json' WHERE id = ?1", [broken.id]);
    expect(shared(red("i4", ["X"]))).toBe(true);
    expect(sharedTests(db)[0]!.pulls).toBe(3);
  });
  test("the vitals carry one card per shared test, naming it, to file its flake issue", () => {
    for (const i of ["i1", "i2", "i3", "i4"]) red(i, ["X", "Y"]);
    red("i5", ["Z"]);
    const v = vitals({ db, dataDir: tmpdir(), signals: { errors: new ErrorLedger(), refused: new RefusedLedger() } });
    const cards = v.degraded.filter((d) => d.cause.startsWith("shared:"));
    expect(cards.map((c) => c.cause).sort()).toEqual(
      [sharedCause({ repoId: "r1", test: "X" }), sharedCause({ repoId: "r1", test: "Y" })].sort(),
    );
    expect(cards.find((c) => c.cause === sharedCause({ repoId: "r1", test: "X" }))!.sentence).toBe(
      `trevato/forge: "X" failed on 4 pulls in the last hour — the lane's red, not theirs; they wait for main's next green check, unparked.`,
    );
    expect(cards[0]!.href).toBe("/trevato/forge/issues/new");
  });
  test("once the sentinel files the flake issue, the card follows it; closed, it files again", () => {
    for (const i of ["i1", "i2", "i3"]) red(i, ["X"]);
    const [t] = sharedTests(db);
    db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('f1', 'r2', 9, 'issue', 'flake', 'u1')");
    db.run(
      "INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen) VALUES (?1, 'f1', '', 'x', 'x')",
      [flakeKey(sharedCause(t!))],
    );
    expect(flakeHref(db, t!)).toBe("/trevato/other/issues/9");
    db.run("UPDATE issue SET state = 'closed' WHERE id = 'f1'");
    expect(flakeHref(db, t!)).toBe("/trevato/forge/issues/new");
  });
  test("the cause is the sentinel's key, whole, and a breach it files at once", () => {
    const cause = sharedCause({ repoId: "01m23vygyffx5hg2f5vm8r74sr", test: "two forges in one process keep their own error and refusal signals" });
    expect(flakeKey(cause)).toBe(`degraded:${keyPart(cause)}`);
    expect(keyPart(cause)).toBe(cause.replaceAll(":", "-"));
    db.run("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 's', '/x', 1000)", [cause]);
    const [found] = budgetBreaches(db, 2000);
    expect(found!.key).toBe(flakeKey(cause));
    expect(found!.urgent).toBe(true);
  });
  test("an id made at or after a moment sorts at or above its floor", () => {
    const at = Date.now();
    expect(id(at) >= idFloor(at)).toBe(true);
    expect(idFloor(at - 1) < idFloor(at)).toBe(true);
    expect(idFloor(at)).toHaveLength(26);
  });
});
@@ -156,8 +156,8 @@
test("the number of queries does not grow with runs", () => {
    if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
  }
  expect(count()).toBe(before);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, and the red mains holding pulls' checks. and the newest drill of every domain.
  expect(before).toBe(7);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain.
  expect(before).toBe(8);
});
test("errors are counted, bounded, redacted, and a burst is said", async () => {
		@@ -0,0 +1,145 @@
	1	/**
	2	* A red many pulls share is the lane's, not theirs (#2266). One test whose
	3	* verdict hung on how full the check machine's disk was failed ten pulls in
	4	* an hour while main's own check stayed green, so the red-main rule
	5	* (`checks/red_main.ts`) never applied: each was parked, its issue struck,
	6	* and every one reopened by hand. So when a pull's check fails only on tests
	7	* that {@link SHARED_PULLS} or more pulls of the repository failed within
	8	* {@link SHARED_WINDOW_MS}, the crew neither reworks nor parks it; it waits
	9	* for main's next green check, as a red main's pulls do, and the vitals
	10	* carry one card per shared test in place of a park per pull.
	11	*
	12	* Plain data: failed pull checks in the window, read by the `run_repo`
	13	* index (an id is time-sortable, so the window is a range), their recorded
	14	* test names unpacked by SQLite's own `json_each`.
	15	*/
	16
	17	import type { Database } from "bun:sqlite";
	18	import type { Run } from "../core/runs.ts";
	19	import { idFloor } from "../kernel/id.ts";
	20
	21	/** How far back another pull's red on the same test counts. */
	22	export const SHARED_WINDOW_MS = 3_600_000;
	23
	24	/** How many pulls, this one among them, failing one test make it shared. */
	25	export const SHARED_PULLS = 3;
	26
	27	/**
	28	* Checks are created before they finish; one created this long before the
	29	* window may still have failed inside it. Bounds the scan, never the answer
	30	* for any check that ran under its lane's own limits.
	31	*/
	32	const QUEUED_SLACK_MS = 6 * 3_600_000;
	33
	34	export interface SharedTest {
	35	readonly repoId: string;
	36	/** `owner/name`, for the card. */
	37	readonly repo: string;
	38	readonly test: string;
	39	/** Distinct pulls whose check failed on it in the window. */
	40	readonly pulls: number;
	41	/** When the earliest of those checks finished. */
	42	readonly since: string;
	43	}
	44
	45	interface Row {
	46	repo_id: string;
	47	repo: string;
	48	test: string;
	49	pulls: number;
	50	since: string;
	51	}
	52
	53	/**
	54	* Each test failed by pull checks in the window, with how many distinct
	55	* pulls failed it. `repo` narrows to one repository; `test` to one name.
	56	* A constant of this module, never input: every value is a parameter.
	57	*/
	58	const PULLS_PER_TEST = `
	59	SELECT run.repo_id, u.username \|\| '/' \|\| repo.name AS repo,
	60	n.value AS test, COUNT(DISTINCT run.issue_id) AS pulls,
	61	MIN(run.finished_at) AS since
	62	FROM run
	63	JOIN repo ON repo.id = run.repo_id
	64	JOIN user u ON u.id = repo.owner_id,
	65	json_each(CASE WHEN json_valid(run.check_detail) THEN run.check_detail ELSE '{}' END, '$.names') n
	66	WHERE run.id >= ?1 AND run.finished_at >= ?2
	67	AND run.kind = 'check' AND run.issue_id IS NOT NULL AND run.state = 'failed'
	68	AND (?3 IS NULL OR run.repo_id = ?3)
	69	AND n.type = 'text'
	70	GROUP BY run.repo_id, n.value`;
	71
	72	function pullsPerTest(db: Database, now: number, repoId: string \| null): Row[] {
	73	return db
	74	.query<Row, [string, string, string \| null]>(PULLS_PER_TEST)
	75	.all(
	76	idFloor(now - SHARED_WINDOW_MS - QUEUED_SLACK_MS),
	77	new Date(now - SHARED_WINDOW_MS).toISOString(),
	78	repoId,
	79	);
	80	}
	81
	82	/**
	83	* Whether a failed check fails only what the lane fails: it named at least
	84	* one test, and every one was failed by {@link SHARED_PULLS} or more pulls
	85	* (this one counted, once it is recorded) in the window. A check that named
	86	* nothing, or one test few others fail, is the pull's own.
	87	*/
	88	export function sharedWithPulls(
	89	db: Database,
	90	run: Pick<Run, "repoId" \| "checkDetail"> \| undefined,
	91	now: number = Date.now(),
	92	): boolean {
	93	const names = run?.checkDetail?.names ?? [];
	94	if (run === undefined \|\| names.length === 0) return false;
	95	const pulls = new Map(pullsPerTest(db, now, run.repoId).map((r) => [r.test, r.pulls]));
	96	return names.every((n) => (pulls.get(n) ?? 0) >= SHARED_PULLS);
	97	}
	98
	99	/** Every shared test across the forge's repositories, oldest first: one query. */
	100	export function sharedTests(db: Database, now: number = Date.now()): SharedTest[] {
	101	return pullsPerTest(db, now, null)
	102	.filter((r) => r.pulls >= SHARED_PULLS)
	103	.sort((a, b) => a.since.localeCompare(b.since))
	104	.map((r) => ({ repoId: r.repo_id, repo: r.repo, test: r.test, pulls: r.pulls, since: r.since }));
	105	}
	106
	107	/**
	108	* The vitals cause for one shared test: `shared:<repoId>:<hash>`, stable
	109	* for the test, short enough that the sentinel's finding key
	110	* ({@link flakeKey}) keeps all of it.
	111	*/
	112	export function sharedCause(t: Pick<SharedTest, "repoId" \| "test">): string {
	113	return `shared:${t.repoId}:${Bun.hash(t.test).toString(36).slice(0, 10)}`;
	114	}
	115
	116	/**
	117	* The sentinel's finding key for a cause, `degraded:${keyPart(cause)}`
	118	* (`crew/sentinel.ts`, which reads the vitals and so is not imported here):
	119	* a shared cause is already lowercase base32 and base36, so only `:` moves.
	120	*/
	121	export const flakeKey = (cause: string): string => `degraded:${cause.replaceAll(":", "-")}`;
	122
	123	/**
	124	* Where the card points: the open flake issue the sentinel filed for this
	125	* test, to follow it, else the repository's new-issue page, to file one.
	126	*/
	127	export function flakeHref(db: Database, t: SharedTest): string {
	128	const filed = db
	129	.query<{ repo: string; number: number }, [string]>(
	130	`SELECT u.username \|\| '/' \|\| repo.name AS repo, i.number
	131	FROM sentinel_finding f
	132	JOIN issue i ON i.id = f.issue_id
	133	JOIN repo ON repo.id = i.repo_id
	134	JOIN user u ON u.id = repo.owner_id
	135	WHERE f.key = ?1 AND i.state = 'open'
	136	LIMIT 1`,
	137	)
	138	.get(flakeKey(sharedCause(t)));
	139	return filed === null ? `/${t.repo}/issues/new` : `/${filed.repo}/issues/${filed.number}`;
	140	}
	141
	142	/** The card's sentence: which test, on how many pulls, and that they wait. */
	143	export function sharedSentence(t: SharedTest): string {
	144	return `${t.repo}: "${t.test}" failed on ${t.pulls} pulls in the last hour — the lane's red, not theirs; they wait for main's next green check, unparked.`;
	145	}
		@@ -84,6 +84,7 @@ import {
84	84	} from "./autonomy.ts";
85	85	import { checksFor } from "../checks/service.ts";
86	86	import { redMain, sharedWithMain } from "../checks/red_main.ts";
	87	import { sharedWithPulls } from "../checks/shared_red.ts";
87	88	import { certain } from "./decide.ts";
88	89	import { readRunLog } from "../runs/log.ts";
89	90	import { whenText } from "./limits.ts";
		@@ -588,8 +589,9 @@ function route(app: App, event: ForgeEvent): void {
588	589	// is the agent's to look at.
589	590	const agent = agentFor(app, issue);
590	591	if (!agent) return;
591		// A red that main shares is not the pull's: no rework, no park, no
592		// strike; it waits and is checked again when main is green (#1971).
	592	// A red that main shares (#1971), or that three pulls share within
	593	// the hour (#2266), is not the pull's: no rework, no park, no
	594	// strike; it waits and is checked again when main is green.
593	595	if (waitOnMain(app, repo, issue, event.payload)) return;
594	596	// Bounded: after a few failed checks in a row the agent stops and a
595	597	// person must speak (docs/APPS.md, rule 7).
		@@ -674,13 +676,14 @@ function route(app: App, event: ForgeEvent): void {
674	676	}
675	677
676	678	/**
677		* Pulls whose check failed only on what main fails, per repository, with
	679	* Pulls whose check failed only on what main fails, or on what the lane
	680	* fails across pulls (`checks/shared_red.ts`), per repository, with
678	681	* the head that was checked. Memory only: a restart forgets them, and the
679	682	* pull's next push or a person's ask checks it again.
680	683	*/
681	684	const waitingOnMain = new Map<string, Map<string, { env: string; commitOid: string }>>();
682	685
683		/** Whether a failed check is main's red, shared: if so, the pull is noted as waiting. */
	686	/** Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting. */
684	687	function waitOnMain(
685	688	app: App,
686	689	repo: Repo,
		@@ -689,15 +692,20 @@ function waitOnMain(
689	692	): boolean {
690	693	const { runId, commitOid, env } = payload;
691	694	if (payload["kind"] !== "check" \|\| typeof runId !== "string") return false;
692		if (!sharedWithMain(redMain(app.db, repo.id), getRun(app.db, runId))) return false;
	695	const run = getRun(app.db, runId);
	696	const main = sharedWithMain(redMain(app.db, repo.id), run);
	697	if (!main && !sharedWithPulls(app.db, run)) return false;
693	698	if (typeof commitOid === "string" && typeof env === "string") {
694	699	const waiting = waitingOnMain.get(repo.id) ?? new Map();
695	700	waiting.set(issue.id, { env, commitOid });
696	701	waitingOnMain.set(repo.id, waiting);
697	702	}
698		log.info("check failed only what main fails; waiting for main to be green", {
699		issue: issue.number,
700		});
	703	log.info(
	704	main
	705	? "check failed only what main fails; waiting for main to be green"
	706	: "check failed only what other pulls fail this hour; waiting for main to be green",
	707	{ issue: issue.number },
	708	);
701	709	return true;
702	710	}
703	711
		@@ -407,10 +407,12 @@ export function degradedCauses(db: Database, now: number): Finding[] {
407	407
408	408	/**
409	409	* The causes that are a budget breached: the lane, a silent run, a run
410		* killed twice, a red main holding its pulls' checks. The last gets one fix
411		* issue (#1971), and `crew/red_fix.ts` keeps other builders off its test.
	410	* killed twice, a red main holding its pulls' checks, a test red on many
	411	* pulls at once. A red main gets one fix issue (#1971), and
	412	* `crew/red_fix.ts` keeps other builders off its test; a shared test gets
	413	* one flake issue (#2266), which its vitals card links to.
412	414	*/
413		const BREACH_CAUSE = /^(queue:check\|stuck:\|budget:\|red:)/;
	415	const BREACH_CAUSE = /^(queue:check\|stuck:\|budget:\|red:\|shared:)/;
414	416
415	417	/**
416	418	* Each budget breach that holds right now, however briefly: the crew hears
		@@ -435,7 +437,9 @@ export function budgetBreaches(db: Database, now: number): Finding[] {
435	437	lookFor: `\`${s.cause}\` is gone from the degraded list and the cause is fixed, not waited out.`,
436	438	fix: s.cause.startsWith("red:")
437	439	? "Fix the failing test on main in one pull. This is the only issue for it: pulls red on the same test wait for main, and no other builder is handed it while this is open."
438		: "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
	440	: s.cause.startsWith("shared:")
	441	? "Make the named test's verdict depend only on the code under test, not on the check machine's load, disk or clock, in one pull. Pulls red on it alone wait, unparked, for main's next green check."
	442	: "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
439	443	urgent: true,
440	444	});
441	445	}
		@@ -51,6 +51,16 @@ export function id(now: number = Date.now()): string {
51	51	return out;
52	52	}
53	53
	54	/**
	55	* The least id `id()` hands out at `ms`: every id made at or after it sorts
	56	* at or above this, so a time window over an id-indexed table is a range.
	57	*/
	58	export function idFloor(ms: number): string {
	59	let out = "";
	60	for (let shift = 45; shift >= 0; shift -= 5) out += alphabet[Math.floor(Math.max(0, ms) / 2 ** shift) % 32];
	61	return out + "0".repeat(16);
	62	}
	63
54	64	/** 256 bits of entropy, base64url, ~43 chars. For tokens and session ids. */
55	65	export function secret(): string {
56	66	const bytes = new Uint8Array(32);
		@@ -28,6 +28,7 @@ import { newestDrills } from "../core/domain_drills.ts";
28	28	import { DOMAINS } from "./domains.ts";
29	29	import { domainDrillNeeds } from "./domain_drill.ts";
30	30	import { heldChecks, redSentence } from "../checks/red_main.ts";
	31	import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
31	32
32	33	/** Free disk under this share of the volume is something to say. */
33	34	export const DISK_FREE_MIN = 0.1;
		@@ -49,7 +50,7 @@ export interface EnvCount {
49	50	/**
50	51	* One cause of the forge being off: the `needs` sentence, a `cause` key that
51	52	* holds for as long as its condition does (`disk`, `errors`, `internal`,
52		* `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`), when it began by the forge's own evidence
	53	* `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
53	54	* (the oldest evidence the forge holds, else the first time this process saw
54	55	* it, held until the cause clears; `at` is never the answer to a repeat
55	56	* call), and where to look.
		@@ -363,6 +364,10 @@ export function vitals(input: VitalsInput): Vitals {
363	364	`/${red.repo}/runs/${red.runId}`,
364	365	);
365	366
	367	// A test failing on many pulls in the hour is one card, not a park per pull.
	368	for (const shared of sharedTests(input.db, now))
	369	need(sharedCause(shared), sharedSentence(shared), shared.since, flakeHref(input.db, shared));
	370
366	371	// A red restore drill, or none in eight days, is a card until a drill passes.
367	372	const drill = lastVerdict(input.dataDir);
368	373	const owed = drillNeed(drill, now, () => drillFirstSeen(input.dataDir));
		@@ -0,0 +1,122 @@
	1	/**
	2	* A red many pulls share never parks them (#2266): through the crew's event
	3	* path, with main green, three pulls whose checks fail one test in the hour
	4	* are neither parked nor reworked and are checked again on main's next green
	5	* check; a pull red on a test only it fails is parked as ever. Each pull has
	6	* already spent its automatic tries, so any red that is its own parks it.
	7	*/
	8
	9	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	10	import type { App } from "../src/app.ts";
	11	import { announceRun } from "../src/actions/runs.ts";
	12	import { commitToBranch } from "../src/actions/repos.ts";
	13	import { openPull } from "../src/actions/pulls.ts";
	14	import { CHECK_FAILED_LIMIT } from "../src/crew/dispatch.ts";
	15	import { getIssueById } from "../src/core/issues.ts";
	16	import { getPull, type Pull } from "../src/core/pulls.ts";
	17	import type { Repo } from "../src/core/repos.ts";
	18	import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
	19	import type { User } from "../src/core/users.ts";
	20	import { vitals } from "../src/kernel/vitals.ts";
	21	import { ErrorLedger } from "../src/kernel/log.ts";
	22	import { RefusedLedger } from "../src/kernel/internal_refused.ts";
	23	import { startStandInCrew, stopStandInCrew, type StandInCrew } from "./helpers/crew-standin.ts";
	24
	25	describe("the crew, with a red many pulls share", () => {
	26	let standIn: StandInCrew;
	27	let app: App;
	28	let alice: User;
	29	let bot: User;
	30	let repo: Repo;
	31	let requested: unknown[][] = [];
	32	let n = 0;
	33
	34	beforeAll(async () => {
	35	standIn = await startStandInCrew();
	36	({ app, alice, bot, repo } = standIn);
	37	standIn.checks.request = async (...args: unknown[]) => {
	38	requested.push(args);
	39	return undefined;
	40	};
	41	// Parking is autonomy's: the repository runs on its own, on alice's word.
	42	app.db.run("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2", [alice.id, repo.id]);
	43	}, 30_000);
	44	afterAll(() => stopStandInCrew(standIn));
	45
	46	/** A pull of the crew's whose automatic tries are spent: its own next red parks it. */
	47	const spentPull = async (): Promise<Pull> => {
	48	const branch = `feat-${++n}`;
	49	const c = await commitToBranch(app, alice, repo, {
	50	message: branch,
	51	branch,
	52	files: [{ path: `${branch}.md`, content: "x\n" }],
	53	});
	54	if (!c.ok) throw new Error(JSON.stringify(c.error));
	55	const o = await openPull(app, alice, repo, { title: branch, baseBranch: "main", headBranch: branch });
	56	if (!o.ok) throw new Error(String(o.error));
	57	app.db.run("UPDATE issue SET assignee_id = ?1 WHERE id = ?2", [bot.id, o.value.id]);
	58	for (let i = 0; i < CHECK_FAILED_LIMIT; i++) {
	59	const tried = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: o.value.id, reason: "check_failed" });
	60	claimRun(app.db, tried.id);
	61	finishRun(app.db, tried.id, { state: "ok", summary: "reworked" });
	62	}
	63	return getPull(app.db, repo.id, o.value.number)!;
	64	};
	65	const failed = (pull: Pull, names: string[]): Run => {
	66	const run = createRun(app.db, {
	67	repoId: repo.id,
	68	kind: "check",
	69	env: `pull-${pull.number}`,
	70	commitOid: pull.headOid,
	71	reason: "push",
	72	issueId: pull.id,
	73	});
	74	claimRun(app.db, run.id);
	75	return finishRun(app.db, run.id, {
	76	state: "failed",
	77	summary: "check failed",
	78	checkDetail: { passed: 1, failed: names.length, names },
	79	})!;
	80	};
	81	const mainGreen = () => {
	82	const run = createRun(app.db, { repoId: repo.id, kind: "check", env: "main", commitOid: "a".repeat(40), reason: "push" });
	83	claimRun(app.db, run.id);
	84	announceRun(app, finishRun(app.db, run.id, { state: "ok", summary: "check passed" })!);
	85	};
	86	const open = (pull: Pull) => getIssueById(app.db, pull.id)?.state === "open";
	87	const sharedCards = () =>
	88	vitals({
	89	db: app.db,
	90	dataDir: app.config.dataDir,
	91	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
	92	}).degraded.filter((d) => d.cause.startsWith("shared:"));
	93
	94	test("three pulls red on X in the hour: none parked, one card; a fourth red on Y only is parked", async () => {
	95	mainGreen();
	96	const pulls = [await spentPull(), await spentPull(), await spentPull()];
	97	const runs = pulls.map((p) => failed(p, ["X"]));
	98	for (const run of runs) announceRun(app, run);
	99	for (const p of pulls) expect(open(p)).toBe(true);
	100
	101	const cards = sharedCards();
	102	expect(cards).toHaveLength(1);
	103	expect(cards[0]!.sentence).toContain('"X" failed on 3 pulls in the last hour');
	104	expect(cards[0]!.href).toBe(`/${alice.username}/${repo.name}/issues/new`);
	105
	106	const own = await spentPull();
	107	announceRun(app, failed(own, ["Y"]));
	108	expect(open(own)).toBe(false);
	109	// X and Y together is not only the lane's red: Y is the pull's own.
	110	const mixed = await spentPull();
	111	announceRun(app, failed(mixed, ["X", "Y"]));
	112	expect(open(mixed)).toBe(false);
	113	expect(sharedCards()).toHaveLength(1);
	114
	115	// Main's next green check sends the three back through the lane, once.
	116	requested = [];
	117	mainGreen();
	118	expect(requested.map((r) => r[4]).sort()).toEqual(pulls.map((p) => p.id).sort());
	119	mainGreen();
	120	expect(requested).toHaveLength(3);
	121	});
	122	});
		@@ -0,0 +1,136 @@
	1	/**
	2	* A red many pulls share (#2266), on in-memory SQLite: a test failed by
	3	* three pulls in the hour is the lane's, one failed by fewer, or failed
	4	* longer ago, is the pull's own; the vitals carry one card per shared test,
	5	* linked to its flake issue once the sentinel has filed one.
	6	*/
	7	import { beforeEach, describe, expect, test } from "bun:test";
	8	import type { Database } from "bun:sqlite";
	9	import { tmpdir } from "node:os";
	10	import { migrations } from "../src/core/schema.ts";
	11	import { openDb } from "../src/kernel/db.ts";
	12	import { claimRun, createRun, finishRun, type Run } from "../src/core/runs.ts";
	13	import {
	14	SHARED_PULLS,
	15	SHARED_WINDOW_MS,
	16	flakeHref,
	17	flakeKey,
	18	sharedCause,
	19	sharedTests,
	20	sharedWithPulls,
	21	} from "../src/checks/shared_red.ts";
	22	import { budgetBreaches, keyPart } from "../src/crew/sentinel.ts";
	23	import { id, idFloor } from "../src/kernel/id.ts";
	24	import { vitals } from "../src/kernel/vitals.ts";
	25	import { ErrorLedger } from "../src/kernel/log.ts";
	26	import { RefusedLedger } from "../src/kernel/internal_refused.ts";
	27
	28	let db: Database;
	29	beforeEach(() => {
	30	db = openDb(":memory:", migrations);
	31	db.run("INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')");
	32	db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1', 'u1', 'forge')");
	33	db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r2', 'u1', 'other')");
	34	for (let i = 1; i <= 6; i++)
	35	db.run(
	36	`INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i${i}', 'r1', ${i}, 'pull', 'p', 'u1')`,
	37	);
	38	db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('j1', 'r2', 1, 'pull', 'p', 'u1')");
	39	});
	40
	41	const red = (issueId: string, names: string[], repoId = "r1"): Run => {
	42	const run = createRun(db, { repoId, kind: "check", env: `pull-${issueId}`, issueId, commitOid: "a".repeat(40), reason: "push" });
	43	claimRun(db, run.id);
	44	return finishRun(db, run.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: names.length, names } })!;
	45	};
	46	const shared = (run: Run \| undefined, now?: number) => sharedWithPulls(db, run, now);
	47
	48	describe("a red many pulls share", () => {
	49	test("three pulls on one test share it; two do not", () => {
	50	red("i1", ["X"]);
	51	const second = red("i2", ["X"]);
	52	expect(shared(second)).toBe(false);
	53	const third = red("i3", ["X"]);
	54	expect(SHARED_PULLS).toBe(3);
	55	expect(shared(third)).toBe(true);
	56	expect(shared(second)).toBe(true);
	57	});
	58
	59	test("one pull red three times is its own red, not the lane's", () => {
	60	red("i1", ["X"]);
	61	red("i1", ["X"]);
	62	expect(shared(red("i1", ["X"]))).toBe(false);
	63	});
	64
	65	test("every test the check names must be shared; naming none is the pull's own", () => {
	66	for (const i of ["i1", "i2", "i3"]) red(i, ["X"]);
	67	expect(shared(red("i4", ["X", "Y"]))).toBe(false);
	68	expect(shared(red("i5", ["Y"]))).toBe(false);
	69	expect(shared(red("i6", []))).toBe(false);
	70	expect(shared(undefined)).toBe(false);
	71	});
	72
	73	test("another repository's reds, main's own, and reds older than the hour count for nothing", () => {
	74	red("j1", ["X"], "r2");
	75	const main = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: "b".repeat(40), reason: "push" });
	76	claimRun(db, main.id);
	77	finishRun(db, main.id, { state: "failed", summary: "check failed", checkDetail: { passed: 1, failed: 1, names: ["X"] } });
	78	red("i1", ["X"]);
	79	expect(shared(red("i2", ["X"]))).toBe(false);
	80	const third = red("i3", ["X"]);
	81	expect(shared(third)).toBe(true);
	82	expect(shared(third, Date.now() + SHARED_WINDOW_MS + 1_000)).toBe(false);
	83	});
	84
	85	test("a check detail that is not JSON is skipped, not thrown on", () => {
	86	for (const i of ["i1", "i2"]) red(i, ["X"]);
	87	const broken = red("i3", ["X"]);
	88	db.run("UPDATE run SET check_detail = '{not json' WHERE id = ?1", [broken.id]);
	89	expect(shared(red("i4", ["X"]))).toBe(true);
	90	expect(sharedTests(db)[0]!.pulls).toBe(3);
	91	});
	92
	93	test("the vitals carry one card per shared test, naming it, to file its flake issue", () => {
	94	for (const i of ["i1", "i2", "i3", "i4"]) red(i, ["X", "Y"]);
	95	red("i5", ["Z"]);
	96	const v = vitals({ db, dataDir: tmpdir(), signals: { errors: new ErrorLedger(), refused: new RefusedLedger() } });
	97	const cards = v.degraded.filter((d) => d.cause.startsWith("shared:"));
	98	expect(cards.map((c) => c.cause).sort()).toEqual(
	99	[sharedCause({ repoId: "r1", test: "X" }), sharedCause({ repoId: "r1", test: "Y" })].sort(),
	100	);
	101	expect(cards.find((c) => c.cause === sharedCause({ repoId: "r1", test: "X" }))!.sentence).toBe(
	102	`trevato/forge: "X" failed on 4 pulls in the last hour — the lane's red, not theirs; they wait for main's next green check, unparked.`,
	103	);
	104	expect(cards[0]!.href).toBe("/trevato/forge/issues/new");
	105	});
	106
	107	test("once the sentinel files the flake issue, the card follows it; closed, it files again", () => {
	108	for (const i of ["i1", "i2", "i3"]) red(i, ["X"]);
	109	const [t] = sharedTests(db);
	110	db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('f1', 'r2', 9, 'issue', 'flake', 'u1')");
	111	db.run(
	112	"INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen) VALUES (?1, 'f1', '', 'x', 'x')",
	113	[flakeKey(sharedCause(t!))],
	114	);
	115	expect(flakeHref(db, t!)).toBe("/trevato/other/issues/9");
	116	db.run("UPDATE issue SET state = 'closed' WHERE id = 'f1'");
	117	expect(flakeHref(db, t!)).toBe("/trevato/forge/issues/new");
	118	});
	119
	120	test("the cause is the sentinel's key, whole, and a breach it files at once", () => {
	121	const cause = sharedCause({ repoId: "01m23vygyffx5hg2f5vm8r74sr", test: "two forges in one process keep their own error and refusal signals" });
	122	expect(flakeKey(cause)).toBe(`degraded:${keyPart(cause)}`);
	123	expect(keyPart(cause)).toBe(cause.replaceAll(":", "-"));
	124	db.run("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 's', '/x', 1000)", [cause]);
	125	const [found] = budgetBreaches(db, 2000);
	126	expect(found!.key).toBe(flakeKey(cause));
	127	expect(found!.urgent).toBe(true);
	128	});
	129
	130	test("an id made at or after a moment sorts at or above its floor", () => {
	131	const at = Date.now();
	132	expect(id(at) >= idFloor(at)).toBe(true);
	133	expect(idFloor(at - 1) < idFloor(at)).toBe(true);
	134	expect(idFloor(at)).toHaveLength(26);
	135	});
	136	});
		@@ -156,8 +156,8 @@ test("the number of queries does not grow with runs", () => {
156	156	if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
157	157	}
158	158	expect(count()).toBe(before);
159		// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, and the red mains holding pulls' checks. and the newest drill of every domain.
160		expect(before).toBe(7);
	159	// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain.
	160	expect(before).toBe(8);
161	161	});
162	162
163	163	test("errors are counted, bounded, redacted, and a burst is said", async () => {