2626c97
@@ -420,6 +420,18 @@
A `check` that is present and wrong fails with the reason, so a commit is
never silently unchecked. A check under way when the forge restarts is
queued again, and the forge that carries on runs it from the start.
**An install failure is not a test failure** (`src/checks/install.ts`,
#5093). An install that fails with `Fail extracting tarball`, `EINTEGRITY`
or `ECONNRESET` is tried once more after 2 s. If it fails that way again,
the check is `failed: infrastructure`. It is queued again, and the merge
queue counts it as cut off, not red. If the re-run fails the same way, it
fails and says `infrastructure`. A check in a cell installs into its own
home, so its bun cache is shared with nothing. An uncontained check uses
its repository's cache (`<data>/check-install/<repo>/bun`), and installs
into it take turns under a lock file, so two never extract one package
into the same directory. A `job` check installs in its own pod and is not
retried.
**Placement.** `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check
in a cell on the forge's machine. `job` runs install and check as one
Kubernetes Job (`src/cluster/jobs.ts`) instead: `FORGE_CHECKS_IMAGE` is its
@@ -0,0 +1,194 @@
/**
 * A check's install step (#5093): the package manager's own flake is not the
 * pull's fault. Concurrent installs into one bun cache race on extracting the
 * same tarball (`Fail extracting tarball`), and a registry stream cut short
 * reads the same way, or as `EINTEGRITY` / `ECONNRESET`.
 *
 * - **No racing on a shared cache.** A check in a cell has a home of its own,
 *   so its cache is its own: nothing is shared, and a pull's install never
 *   fills a cache another check reads. A check that runs uncontained would
 *   otherwise share the forge's `~/.bun` with every check of every
 *   repository; it gets its repository's own cache instead
 *   (`<data>/check-install/<repo>/bun`), and installs into it one at a time,
 *   under a lock file beside it — across checks, gates and forges alike.
 * - **A transient failure is retried once**, after {@link INSTALL_RETRY_MS}.
 *   Failing the same way twice is {@link INFRASTRUCTURE}: the service queues
 *   the check again and the merge queue reads it as cut off, never as red.
 */
import { lstatSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { err, ok, type Result } from "../kernel/result.ts";
/** What an install prints when the machine, not the code, failed it. */
export const TRANSIENT_INSTALL = /Fail extracting tarball|EINTEGRITY|ECONNRESET/;
/** How long a transient install failure waits before its one retry. */
export const INSTALL_RETRY_MS = 2_000;
/** The error prefix of an install the machine failed twice: not the pull's red. */
export const INFRASTRUCTURE = "infrastructure";
/** A lock older than this is a holder that died: a whole install never takes so long. */
export const LOCK_STALE_MS = 15 * 60_000;
const LOCK_POLL_MS = 50;
export type InstallOutcome =
  | { readonly ok: true }
  | { readonly ok: false; readonly error: string; readonly infrastructure: boolean };
/** The first line of an install's output that says the failure was transient. */
export function transientLine(lines: readonly string[]): string | undefined {
  return lines.find((line) => TRANSIENT_INSTALL.test(line));
}
/** A repository's bun cache for its uncontained checks; the lock file sits beside it. */
export function checkInstallCache(dataDir: string, repoId: string): string {
  return join(dataDir, "check-install", repoId.replace(/[^A-Za-z0-9_-]/g, "_"), "bun");
}
/**
 * Run `attempt` (one install, its output lines handed to `log`), and once
 * more after `retryMs` when it failed with a {@link TRANSIENT_INSTALL} line.
 * A failure without one is the code's, answered at once.
 */
export async function installWithRetry(
  attempt: (log: (line: string) => void) => Promise<Result<void, string>>,
  options: {
    readonly log: (line: string) => void;
    readonly signal: AbortSignal;
    readonly retryMs?: number | undefined;
  },
): Promise<InstallOutcome> {
  let transient: string | undefined;
  for (let tries = 0; tries < 2; tries++) {
    const lines: string[] = [];
    const ran = await attempt((line) => {
      lines.push(line);
      options.log(line);
    });
    if (ran.ok) return { ok: true };
    transient = transientLine(lines);
    if (transient === undefined || options.signal.aborted)
      return { ok: false, error: ran.error, infrastructure: false };
    if (tries === 0) {
      const wait = options.retryMs ?? INSTALL_RETRY_MS;
      options.log(`install hit a transient failure (${transient.trim()}); retrying once in ${wait / 1000}s`);
      if (!(await sleep(wait, options.signal)))
        return { ok: false, error: ran.error, infrastructure: false };
    }
  }
  return {
    ok: false,
    error: `${INFRASTRUCTURE}: install failed twice — ${transient?.trim() ?? ""}`,
    infrastructure: true,
  };
}
/**
 * Run `fn` holding the lock file at `path`, made with `wx` so exactly one
 * holder at a time, across processes. A lock whose holder is gone (its pid
 * dead on this machine, or older than `staleMs`) is taken over. Answers an
 * Err, without running `fn`, when `signal` aborts the wait.
 */
export async function withLock<T>(
  path: string,
  fn: () => Promise<T>,
  options: { readonly signal: AbortSignal; readonly staleMs?: number | undefined },
): Promise<Result<T, string>> {
  // Holders in this process queue in memory, so a release hands over at
  // once; the file is what holds other processes out.
  const before = local.get(path) ?? Promise.resolve();
  let release = (): void => {};
  const mine = new Promise<void>((resolve) => {
    release = resolve;
  });
  const tail = before.then(() => mine);
  local.set(path, tail);
  try {
    if (!(await settledBefore(before, options.signal)))
      return err("cancelled waiting for the install lock");
    return await holdFile(path, fn, options);
  } finally {
    release();
    if (local.get(path) === tail) local.delete(path);
  }
}
const local = new Map<string, Promise<void>>();
async function holdFile<T>(
  path: string,
  fn: () => Promise<T>,
  options: { readonly signal: AbortSignal; readonly staleMs?: number | undefined },
): Promise<Result<T, string>> {
  mkdirSync(dirname(path), { recursive: true });
  const staleMs = options.staleMs ?? LOCK_STALE_MS;
  for (;;) {
    if (options.signal.aborted) return err("cancelled waiting for the install lock");
    try {
      writeFileSync(path, `${process.pid} ${Date.now()}\n`, { flag: "wx" });
      break;
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
    }
    // Taking over a dead holder's lock can race another waiter doing the
    // same; that only happens after a crash, and costs one overlap at worst.
    if (stale(path, staleMs)) rmSync(path, { force: true });
    else await sleep(LOCK_POLL_MS, options.signal);
  }
  try {
    return ok(await fn());
      
        junipercrew
        23d ago
        
      
      Taking over a stale lock removes the file unconditionally, so two waiters can each remove the other's fresh lock and run installs together. That is the race this change exists to stop, only rarer. Rename the stale lock to a unique name first, or re-check the contents before removing it.
    
  } finally {
    rmSync(path, { force: true });
  }
}
function stale(path: string, staleMs: number): boolean {
  const stat = lstatSync(path, { throwIfNoEntry: false });
  if (stat === undefined) return false;
  if (Date.now() - stat.mtimeMs > staleMs) return true;
  let pid = Number.NaN;
  try {
    pid = Number.parseInt(readFileSync(path, "utf8"), 10);
  } catch {
    return false;
  }
  if (!Number.isSafeInteger(pid) || pid <= 0) return false;
  try {
    process.kill(pid, 0);
    return false;
  } catch (error) {
    return (error as NodeJS.ErrnoException).code === "ESRCH";
  }
}
/** Wait for `before`; false when `signal` aborted it first. */
function settledBefore(before: Promise<void>, signal: AbortSignal): Promise<boolean> {
  if (signal.aborted) return Promise.resolve(false);
  return new Promise((resolve) => {
    const stop = (): void => resolve(false);
    signal.addEventListener("abort", stop, { once: true });
    void before.then(() => {
      signal.removeEventListener("abort", stop);
      resolve(true);
    });
  });
}
/** Wait `ms`; false when `signal` aborted it first. */
function sleep(ms: number, signal: AbortSignal): Promise<boolean> {
  if (signal.aborted) return Promise.resolve(false);
  return new Promise((resolve) => {
    const timer = setTimeout(() => {
      signal.removeEventListener("abort", stop);
      resolve(true);
    }, ms);
    const stop = (): void => {
      clearTimeout(timer);
      resolve(false);
    };
    signal.addEventListener("abort", stop, { once: true });
  });
}
@@ -37,6 +37,8 @@
import {
  setRunBase,
  vanishedBefore,
  VANISHED_SUMMARY,
  infraBefore,
  INFRA_SUMMARY,
  requeueCheck,
  REQUEUED_SUMMARY,
  sweepAbandonedChecks,
@@ -84,6 +86,7 @@
import { summariseCheckFailure } from "./summary.ts";
import type { Result } from "../kernel/result.ts";
import { placementFromEnv, type Placement } from "./placement.ts";
import { JOB_VANISHED, runJobStep } from "./job_step.ts";
import { checkInstallCache, installWithRetry, withLock } from "./install.ts";
import { extraGates } from "./queue.ts";
import type { JobOptions } from "../cluster/jobs.ts";
import type { CheckDetail } from "../core/runs.ts";
@@ -128,6 +131,8 @@
export interface ChecksOptions {
  readonly placement?: Placement | undefined;
  /** The kube client a `job` placement speaks through; tests stand one in. */
  readonly job?: JobOptions | undefined;
  /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */
  readonly installRetryMs?: number | undefined;
}
export interface ChecksHandle {
@@ -594,6 +599,16 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
          return;
        }
      }
      // The install failed twice on what the machine did, not the code: the
      // check is queued once more and nothing reads it as red (#5093).
      if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) {
        const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY);
        if (moved) {
          write(INFRA_SUMMARY);
          announceRun(app, moved.again, run.actorId ?? undefined);
          return;
        }
      }
      const seconds = Math.round((Date.now() - started) / 1000);
      const done = finishRun(app.db, run.id, {
        state: outcome.ok ? "ok" : "failed",
@@ -620,7 +635,7 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
    signal: AbortSignal,
  ): Promise<
    | { ok: true; proven?: string }
    | { ok: false; error: string; detail?: CheckDetail | undefined }
    | { ok: false; error: string; detail?: CheckDetail | undefined; infrastructure?: boolean }
  > {
    const bare = await runDir(app.db, app.reposRoot, repo, run.issueId);
    // A pull's check is the fast tier, judged against where the pull left
@@ -670,7 +685,7 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
    x: Execution,
  ): Promise<
    | { ok: true; proven?: string }
    | { ok: false; error: string; detail?: CheckDetail | undefined }
    | { ok: false; error: string; detail?: CheckDetail | undefined; infrastructure?: boolean }
  > {
    const { bare, repo, write, signal } = x;
    // A pull's fast check is of the pull as it would land: its head merged
@@ -896,15 +911,31 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
        });
      } else {
        if (manifest.install !== null) {
          const installed = await runStep("install", manifest.install, {
            cwd: dir,
            env,
            log: write,
            signal,
            timeoutMs: manifest.timeout * 1000,
            cell,
          });
          if (!installed.ok) return { ok: false, error: installed.error };
          const install = manifest.install;
          // A cell's home is its own cache. Uncontained, the repository's
          // cache is shared, so installs into it take turns (`checks/install.ts`).
          const shared =
            cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined;
          const once = (log: (line: string) => void): Promise<Result<void, string>> =>
            runStep("install", install, {
              cwd: dir,
              env: shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared },
              log,
              signal,
              timeoutMs: manifest.timeout * 1000,
              cell,
            });
          const installed = await installWithRetry(
            async (log) => {
              if (shared === undefined) return once(log);
              mkdirSync(shared, { recursive: true });
              const held = await withLock(`${shared}.lock`, () => once(log), { signal });
              return held.ok ? held.value : held;
            },
            { log: write, signal, retryMs: options.installRetryMs },
          );
          if (!installed.ok)
            return { ok: false, error: installed.error, infrastructure: installed.infrastructure };
        }
        // The check step's own lines are kept beside what `write` already
        // logs, so a failure can be summarised once here — never by reading
@@ -1185,9 +1216,15 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
          signal: controller.signal,
        });
        if (done.ok) return { ok: true, error: "", lines };
        // Cut off by this forge stopping, or the Job lost under it: the
        // code was not judged, so this is no red.
        if (stopped || controller.signal.aborted || done.error === JOB_VANISHED)
        // Cut off by this forge stopping, the Job lost under it, or an
        // install the machine failed twice: the code was not judged, so
        // this is no red.
        if (
          stopped ||
          controller.signal.aborted ||
          done.error === JOB_VANISHED ||
          done.infrastructure === true
        )
          return { ok: false, error: done.error, lines, interrupted: true };
        return { ok: false, error: done.error, lines };
      } finally {
@@ -459,6 +459,9 @@
export const REQUEUED_SUMMARY = "interrupted by a restart; queued again";
/** What a check whose cluster Job vanished says as it is queued again. */
export const VANISHED_SUMMARY = "interrupted: its cluster job vanished; queued again";
/** What a check whose install the machine failed twice says as it is queued again (#5093). */
export const INFRA_SUMMARY = "failed: infrastructure — the install failed twice on a transient error; queued again";
/**
 * How many times one check may be cut off by its forge going away and be
 * queued again. Past it the check fails once, saying so: a check that kills
@@ -521,6 +524,11 @@
export function vanishedBefore(db: Database, run: Run): boolean {
  return cutOffBefore(db, run, VANISHED_SUMMARY);
}
/** Whether the attempt just before this one failed on infrastructure: it is queued again once. */
export function infraBefore(db: Database, run: Run): boolean {
  return cutOffBefore(db, run, INFRA_SUMMARY);
}
function cutOffBefore(db: Database, run: Run, prefix: string): boolean {
  return (
    db
@@ -0,0 +1,230 @@
/**
 * A check's install (#5093): installs into one cache take turns, a transient
 * install failure is retried once, and failing the same way twice is the
 * machine's — queued again, never the pull's red.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import {
  checkInstallCache,
  INFRASTRUCTURE,
  installWithRetry,
  transientLine,
  withLock,
} from "../src/checks/install.ts";
import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
import { INFRA_SUMMARY, listRuns, type Run } from "../src/core/runs.ts";
import { createUser } from "../src/core/users.ts";
import type { Repo } from "../src/core/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { err, ok, type Result } from "../src/kernel/result.ts";
const never = new AbortController().signal;
describe("transientLine", () => {
  test("names the three failures the machine causes, and nothing the code does", () => {
    expect(transientLine(['error: Fail extracting tarball for "@typescript/typescript-linux-x64"'])).toContain("Fail extracting");
    expect(transientLine(["npm ERR! code EINTEGRITY"])).toContain("EINTEGRITY");
    expect(transientLine(["error: ECONNRESET reading registry.npmjs.org"])).toContain("ECONNRESET");
    expect(transientLine(["error: No version matching \"9.9.9\" found", "exit 1"])).toBeUndefined();
  });
});
describe("installWithRetry", () => {
  const scripted = (outputs: (string | null)[]) => {
    let calls = 0;
    const attempt = async (log: (line: string) => void): Promise<Result<void, string>> => {
      const line = outputs[calls++] ?? null;
      if (line === null) return ok();
      log(line);
      return err(`install exited with code 1 — ${line}`);
    };
    return { attempt, calls: () => calls };
  };
  test("a transient failure is retried once and the retry's green is the answer", async () => {
    const s = scripted(['error: Fail extracting tarball for "x"', null]);
    const lines: string[] = [];
    const done = await installWithRetry(s.attempt, { log: (l) => lines.push(l), signal: never, retryMs: 1 });
    expect(done).toEqual({ ok: true });
    expect(s.calls()).toBe(2);
    expect(lines.some((l) => l.includes("retrying once"))).toBe(true);
  });
  test("the code's own failure is answered at once, and is not infrastructure", async () => {
    const s = scripted(['error: No version matching "9.9.9"']);
    const done = await installWithRetry(s.attempt, { log: () => {}, signal: never, retryMs: 1 });
    expect(s.calls()).toBe(1);
    expect(done.ok).toBe(false);
    if (!done.ok) expect(done.infrastructure).toBe(false);
  });
  test("failing transiently twice is infrastructure, tried exactly twice", async () => {
    const s = scripted(["EINTEGRITY", "ECONNRESET", null]);
    const done = await installWithRetry(s.attempt, { log: () => {}, signal: never, retryMs: 1 });
    expect(s.calls()).toBe(2);
    expect(done.ok).toBe(false);
    if (!done.ok) {
      expect(done.infrastructure).toBe(true);
      expect(done.error.startsWith(`${INFRASTRUCTURE}:`)).toBe(true);
    }
  });
  test("a cancel during the wait does not retry", async () => {
    const s = scripted(["ECONNRESET", null]);
    const abort = new AbortController();
    const pending = installWithRetry(s.attempt, { log: () => {}, signal: abort.signal, retryMs: 60_000 });
    abort.abort();
    const done = await pending;
    expect(s.calls()).toBe(1);
    expect(done.ok).toBe(false);
    if (!done.ok) expect(done.infrastructure).toBe(false);
  });
});
describe("withLock", () => {
  let dir: string;
  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), "forge-install-lock-"));
  });
  afterAll(async () => {
    await rm(dir, { recursive: true, force: true });
  });
  test("8 parallel installs on one cache, 20 rounds: none ever extracts beside another", async () => {
    // Each install "extracts" the same package: a directory that exists only
    // while one install is in it. `mkdir` fails when another is in there —
    // the collision bun reports as `Fail extracting tarball`.
    const cache = join(dir, "cache");
    mkdirSync(cache, { recursive: true });
    const install = async (): Promise<number> => {
      const proc = Bun.spawn(
        ["/bin/sh", "-c", 'mkdir "$1/pkg.extracting" && sleep 0.005 && rmdir "$1/pkg.extracting"', "sh", cache],
        { stdin: "ignore", stdout: "ignore", stderr: "ignore" },
      );
      return proc.exited;
    };
    let failures = 0;
    for (let round = 0; round < 20; round++) {
      const codes = await Promise.all(
        Array.from({ length: 8 }, () => withLock(`${cache}.lock`, install, { signal: never })),
      );
      for (const code of codes) if (!code.ok || code.value !== 0) failures++;
    }
    expect(failures).toBe(0);
    expect(existsSync(`${cache}.lock`)).toBe(false);
  }, 60_000);
  test("a lock left by a holder that died is taken over", async () => {
    const lock = join(dir, "dead.lock");
    // A pid no process holds: the kernel's pid_max is far below it.
    writeFileSync(lock, `${2 ** 30} ${Date.now()}\n`);
    const held = await withLock(lock, async () => "ran", { signal: never });
    expect(held).toEqual({ ok: true, value: "ran" });
  });
  test("a live holder's lock is waited for, and a cancel ends the wait without running", async () => {
    const lock = join(dir, "live.lock");
    writeFileSync(lock, `${process.pid} ${Date.now()}\n`);
    const abort = new AbortController();
    let ran = false;
    const pending = withLock(lock, async () => {
      ran = true;
    }, { signal: abort.signal });
    await Bun.sleep(120);
    expect(ran).toBe(false);
    abort.abort();
    const held = await pending;
    expect(held.ok).toBe(false);
    expect(ran).toBe(false);
    expect(existsSync(lock)).toBe(true);
  });
});
describe("a check whose install fails on the machine", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  let checks: ChecksHandle;
  let flaky: Repo;
  let broken: Repo;
  // Fails once with bun's own line, then installs: the marker lives in the
  // checkout, which the retry shares and the next check does not.
  const once =
    'test -f .tried && exit 0; touch .tried; echo \'error: Fail extracting tarball for "@typescript/typescript-linux-x64"\'; exit 1\n';
  const always = "echo 'error: Fail extracting tarball for \"x\"'; exit 1\n";
  const forgeJson = '{"check": {"run": "exit 0", "install": "sh install.sh"}}\n';
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-check-install-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    checks = startChecks(app, { forgeUrl: servers.url, concurrency: 2, installRetryMs: 10 });
    const a = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!a.ok) throw new Error(a.error);
    const repoWith = async (name: string, script: string): Promise<Repo> => {
      const made = await createRepo(app, a.value, { name, visibility: "public" });
      if (!made.ok) throw new Error(String(made.error));
      const c = await commitToBranch(app, a.value, made.value, {
        message: "Declare a check",
        files: [
          { path: "forge.json", content: forgeJson },
          { path: "install.sh", content: script },
        ],
      });
      if (!c.ok) throw new Error(JSON.stringify(c.error));
      return made.value;
    };
    flaky = await repoWith("flaky", once);
    broken = await repoWith("broken", always);
  });
  afterAll(async () => {
    await checks.stop();
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  const settled = async (repo: Repo, done: (runs: Run[]) => boolean): Promise<Run[]> => {
    const deadline = Date.now() + 30_000;
    let runs: Run[] = [];
    while (Date.now() < deadline) {
      runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
      if (done(runs)) break;
      await Bun.sleep(50);
    }
    return runs;
  };
  test("one transient failure is retried in the same check, which passes", async () => {
    const runs = await settled(flaky, (rs) => rs.some((r) => r.state === "ok" || r.state === "failed"));
    expect(runs.length).toBe(1);
    expect(runs[0]!.state).toBe("ok");
    // Uncontained, the repository's own cache is the one bun is pointed at.
    expect(existsSync(checkInstallCache(dataDir, flaky.id))).toBe(true);
  });
  test("failing twice is infrastructure: queued again, not red; a second time it fails saying so", async () => {
    const runs = await settled(broken, (rs) => rs.some((r) => r.state === "failed"));
    const first = runs.find((r) => r.attempt === 0)!;
    const second = runs.find((r) => r.attempt === 1)!;
    expect(first.state).toBe("cancelled");
    expect(first.summary).toBe(INFRA_SUMMARY);
    expect(second.state).toBe("failed");
    expect(second.summary).toContain(`${INFRASTRUCTURE}:`);
    expect(runs.length).toBe(2);
  });
});
		@@ -420,6 +420,18 @@ A `check` that is present and wrong fails with the reason, so a commit is
420	420	never silently unchecked. A check under way when the forge restarts is
421	421	queued again, and the forge that carries on runs it from the start.
422	422
	423	An install failure is not a test failure (`src/checks/install.ts`,
	424	#5093). An install that fails with `Fail extracting tarball`, `EINTEGRITY`
	425	or `ECONNRESET` is tried once more after 2 s. If it fails that way again,
	426	the check is `failed: infrastructure`. It is queued again, and the merge
	427	queue counts it as cut off, not red. If the re-run fails the same way, it
	428	fails and says `infrastructure`. A check in a cell installs into its own
	429	home, so its bun cache is shared with nothing. An uncontained check uses
	430	its repository's cache (`<data>/check-install/<repo>/bun`), and installs
	431	into it take turns under a lock file, so two never extract one package
	432	into the same directory. A `job` check installs in its own pod and is not
	433	retried.
	434
423	435	Placement. `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check
424	436	in a cell on the forge's machine. `job` runs install and check as one
425	437	Kubernetes Job (`src/cluster/jobs.ts`) instead: `FORGE_CHECKS_IMAGE` is its
		@@ -0,0 +1,194 @@
	1	/**
	2	* A check's install step (#5093): the package manager's own flake is not the
	3	* pull's fault. Concurrent installs into one bun cache race on extracting the
	4	* same tarball (`Fail extracting tarball`), and a registry stream cut short
	5	* reads the same way, or as `EINTEGRITY` / `ECONNRESET`.
	6	*
	7	* - No racing on a shared cache. A check in a cell has a home of its own,
	8	* so its cache is its own: nothing is shared, and a pull's install never
	9	* fills a cache another check reads. A check that runs uncontained would
	10	* otherwise share the forge's `~/.bun` with every check of every
	11	* repository; it gets its repository's own cache instead
	12	* (`<data>/check-install/<repo>/bun`), and installs into it one at a time,
	13	* under a lock file beside it — across checks, gates and forges alike.
	14	* - A transient failure is retried once, after {@link INSTALL_RETRY_MS}.
	15	* Failing the same way twice is {@link INFRASTRUCTURE}: the service queues
	16	* the check again and the merge queue reads it as cut off, never as red.
	17	*/
	18
	19	import { lstatSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
	20	import { dirname, join } from "node:path";
	21	import { err, ok, type Result } from "../kernel/result.ts";
	22
	23	/** What an install prints when the machine, not the code, failed it. */
	24	export const TRANSIENT_INSTALL = /Fail extracting tarball\|EINTEGRITY\|ECONNRESET/;
	25
	26	/** How long a transient install failure waits before its one retry. */
	27	export const INSTALL_RETRY_MS = 2_000;
	28
	29	/** The error prefix of an install the machine failed twice: not the pull's red. */
	30	export const INFRASTRUCTURE = "infrastructure";
	31
	32	/** A lock older than this is a holder that died: a whole install never takes so long. */
	33	export const LOCK_STALE_MS = 15 * 60_000;
	34
	35	const LOCK_POLL_MS = 50;
	36
	37	export type InstallOutcome =
	38	\| { readonly ok: true }
	39	\| { readonly ok: false; readonly error: string; readonly infrastructure: boolean };
	40
	41	/** The first line of an install's output that says the failure was transient. */
	42	export function transientLine(lines: readonly string[]): string \| undefined {
	43	return lines.find((line) => TRANSIENT_INSTALL.test(line));
	44	}
	45
	46	/** A repository's bun cache for its uncontained checks; the lock file sits beside it. */
	47	export function checkInstallCache(dataDir: string, repoId: string): string {
	48	return join(dataDir, "check-install", repoId.replace(/[^A-Za-z0-9_-]/g, "_"), "bun");
	49	}
	50
	51	/**
	52	* Run `attempt` (one install, its output lines handed to `log`), and once
	53	* more after `retryMs` when it failed with a {@link TRANSIENT_INSTALL} line.
	54	* A failure without one is the code's, answered at once.
	55	*/
	56	export async function installWithRetry(
	57	attempt: (log: (line: string) => void) => Promise<Result<void, string>>,
	58	options: {
	59	readonly log: (line: string) => void;
	60	readonly signal: AbortSignal;
	61	readonly retryMs?: number \| undefined;
	62	},
	63	): Promise<InstallOutcome> {
	64	let transient: string \| undefined;
	65	for (let tries = 0; tries < 2; tries++) {
	66	const lines: string[] = [];
	67	const ran = await attempt((line) => {
	68	lines.push(line);
	69	options.log(line);
	70	});
	71	if (ran.ok) return { ok: true };
	72	transient = transientLine(lines);
	73	if (transient === undefined \|\| options.signal.aborted)
	74	return { ok: false, error: ran.error, infrastructure: false };
	75	if (tries === 0) {
	76	const wait = options.retryMs ?? INSTALL_RETRY_MS;
	77	options.log(`install hit a transient failure (${transient.trim()}); retrying once in ${wait / 1000}s`);
	78	if (!(await sleep(wait, options.signal)))
	79	return { ok: false, error: ran.error, infrastructure: false };
	80	}
	81	}
	82	return {
	83	ok: false,
	84	error: `${INFRASTRUCTURE}: install failed twice — ${transient?.trim() ?? ""}`,
	85	infrastructure: true,
	86	};
	87	}
	88
	89	/**
	90	* Run `fn` holding the lock file at `path`, made with `wx` so exactly one
	91	* holder at a time, across processes. A lock whose holder is gone (its pid
	92	* dead on this machine, or older than `staleMs`) is taken over. Answers an
	93	* Err, without running `fn`, when `signal` aborts the wait.
	94	*/
	95	export async function withLock<T>(
	96	path: string,
	97	fn: () => Promise<T>,
	98	options: { readonly signal: AbortSignal; readonly staleMs?: number \| undefined },
	99	): Promise<Result<T, string>> {
	100	// Holders in this process queue in memory, so a release hands over at
	101	// once; the file is what holds other processes out.
	102	const before = local.get(path) ?? Promise.resolve();
	103	let release = (): void => {};
	104	const mine = new Promise<void>((resolve) => {
	105	release = resolve;
	106	});
	107	const tail = before.then(() => mine);
	108	local.set(path, tail);
	109	try {
	110	if (!(await settledBefore(before, options.signal)))
	111	return err("cancelled waiting for the install lock");
	112	return await holdFile(path, fn, options);
	113	} finally {
	114	release();
	115	if (local.get(path) === tail) local.delete(path);
	116	}
	117	}
	118
	119	const local = new Map<string, Promise<void>>();
	120
	121	async function holdFile<T>(
	122	path: string,
	123	fn: () => Promise<T>,
	124	options: { readonly signal: AbortSignal; readonly staleMs?: number \| undefined },
	125	): Promise<Result<T, string>> {
	126	mkdirSync(dirname(path), { recursive: true });
	127	const staleMs = options.staleMs ?? LOCK_STALE_MS;
	128	for (;;) {
	129	if (options.signal.aborted) return err("cancelled waiting for the install lock");
	130	try {
	131	writeFileSync(path, `${process.pid} ${Date.now()}\n`, { flag: "wx" });
	132	break;
	133	} catch (error) {
	134	if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
	135	}
	136	// Taking over a dead holder's lock can race another waiter doing the
	137	// same; that only happens after a crash, and costs one overlap at worst.
	138	if (stale(path, staleMs)) rmSync(path, { force: true });
	139	else await sleep(LOCK_POLL_MS, options.signal);
	140	}
	141	try {
	142	return ok(await fn());
		junipercrew 23d ago Taking over a stale lock removes the file unconditionally, so two waiters can each remove the other's fresh lock and run installs together. That is the race this change exists to stop, only rarer. Rename the stale lock to a unique name first, or re-check the contents before removing it.
	143	} finally {
	144	rmSync(path, { force: true });
	145	}
	146	}
	147
	148	function stale(path: string, staleMs: number): boolean {
	149	const stat = lstatSync(path, { throwIfNoEntry: false });
	150	if (stat === undefined) return false;
	151	if (Date.now() - stat.mtimeMs > staleMs) return true;
	152	let pid = Number.NaN;
	153	try {
	154	pid = Number.parseInt(readFileSync(path, "utf8"), 10);
	155	} catch {
	156	return false;
	157	}
	158	if (!Number.isSafeInteger(pid) \|\| pid <= 0) return false;
	159	try {
	160	process.kill(pid, 0);
	161	return false;
	162	} catch (error) {
	163	return (error as NodeJS.ErrnoException).code === "ESRCH";
	164	}
	165	}
	166
	167	/** Wait for `before`; false when `signal` aborted it first. */
	168	function settledBefore(before: Promise<void>, signal: AbortSignal): Promise<boolean> {
	169	if (signal.aborted) return Promise.resolve(false);
	170	return new Promise((resolve) => {
	171	const stop = (): void => resolve(false);
	172	signal.addEventListener("abort", stop, { once: true });
	173	void before.then(() => {
	174	signal.removeEventListener("abort", stop);
	175	resolve(true);
	176	});
	177	});
	178	}
	179
	180	/** Wait `ms`; false when `signal` aborted it first. */
	181	function sleep(ms: number, signal: AbortSignal): Promise<boolean> {
	182	if (signal.aborted) return Promise.resolve(false);
	183	return new Promise((resolve) => {
	184	const timer = setTimeout(() => {
	185	signal.removeEventListener("abort", stop);
	186	resolve(true);
	187	}, ms);
	188	const stop = (): void => {
	189	clearTimeout(timer);
	190	resolve(false);
	191	};
	192	signal.addEventListener("abort", stop, { once: true });
	193	});
	194	}
		@@ -37,6 +37,8 @@ import {
37	37	setRunBase,
38	38	vanishedBefore,
39	39	VANISHED_SUMMARY,
	40	infraBefore,
	41	INFRA_SUMMARY,
40	42	requeueCheck,
41	43	REQUEUED_SUMMARY,
42	44	sweepAbandonedChecks,
		@@ -84,6 +86,7 @@ import { summariseCheckFailure } from "./summary.ts";
84	86	import type { Result } from "../kernel/result.ts";
85	87	import { placementFromEnv, type Placement } from "./placement.ts";
86	88	import { JOB_VANISHED, runJobStep } from "./job_step.ts";
	89	import { checkInstallCache, installWithRetry, withLock } from "./install.ts";
87	90	import { extraGates } from "./queue.ts";
88	91	import type { JobOptions } from "../cluster/jobs.ts";
89	92	import type { CheckDetail } from "../core/runs.ts";
		@@ -128,6 +131,8 @@ export interface ChecksOptions {
128	131	readonly placement?: Placement \| undefined;
129	132	/** The kube client a `job` placement speaks through; tests stand one in. */
130	133	readonly job?: JobOptions \| undefined;
	134	/** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */
	135	readonly installRetryMs?: number \| undefined;
131	136	}
132	137
133	138	export interface ChecksHandle {
		@@ -594,6 +599,16 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
594	599	return;
595	600	}
596	601	}
	602	// The install failed twice on what the machine did, not the code: the
	603	// check is queued once more and nothing reads it as red (#5093).
	604	if (!outcome.ok && outcome.infrastructure === true && !infraBefore(app.db, claimed)) {
	605	const moved = requeueCheck(app.db, run.id, undefined, INFRA_SUMMARY);
	606	if (moved) {
	607	write(INFRA_SUMMARY);
	608	announceRun(app, moved.again, run.actorId ?? undefined);
	609	return;
	610	}
	611	}
597	612	const seconds = Math.round((Date.now() - started) / 1000);
598	613	const done = finishRun(app.db, run.id, {
599	614	state: outcome.ok ? "ok" : "failed",
		@@ -620,7 +635,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
620	635	signal: AbortSignal,
621	636	): Promise<
622	637	\| { ok: true; proven?: string }
623		\| { ok: false; error: string; detail?: CheckDetail \| undefined }
	638	\| { ok: false; error: string; detail?: CheckDetail \| undefined; infrastructure?: boolean }
624	639	> {
625	640	const bare = await runDir(app.db, app.reposRoot, repo, run.issueId);
626	641	// A pull's check is the fast tier, judged against where the pull left
		@@ -670,7 +685,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
670	685	x: Execution,
671	686	): Promise<
672	687	\| { ok: true; proven?: string }
673		\| { ok: false; error: string; detail?: CheckDetail \| undefined }
	688	\| { ok: false; error: string; detail?: CheckDetail \| undefined; infrastructure?: boolean }
674	689	> {
675	690	const { bare, repo, write, signal } = x;
676	691	// A pull's fast check is of the pull as it would land: its head merged
		@@ -896,15 +911,31 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
896	911	});
897	912	} else {
898	913	if (manifest.install !== null) {
899		const installed = await runStep("install", manifest.install, {
900		cwd: dir,
901		env,
902		log: write,
903		signal,
904		timeoutMs: manifest.timeout * 1000,
905		cell,
906		});
907		if (!installed.ok) return { ok: false, error: installed.error };
	914	const install = manifest.install;
	915	// A cell's home is its own cache. Uncontained, the repository's
	916	// cache is shared, so installs into it take turns (`checks/install.ts`).
	917	const shared =
	918	cell === undefined ? checkInstallCache(app.config.dataDir, repo.id) : undefined;
	919	const once = (log: (line: string) => void): Promise<Result<void, string>> =>
	920	runStep("install", install, {
	921	cwd: dir,
	922	env: shared === undefined ? env : { ...env, BUN_INSTALL_CACHE_DIR: shared },
	923	log,
	924	signal,
	925	timeoutMs: manifest.timeout * 1000,
	926	cell,
	927	});
	928	const installed = await installWithRetry(
	929	async (log) => {
	930	if (shared === undefined) return once(log);
	931	mkdirSync(shared, { recursive: true });
	932	const held = await withLock(`${shared}.lock`, () => once(log), { signal });
	933	return held.ok ? held.value : held;
	934	},
	935	{ log: write, signal, retryMs: options.installRetryMs },
	936	);
	937	if (!installed.ok)
	938	return { ok: false, error: installed.error, infrastructure: installed.infrastructure };
908	939	}
909	940	// The check step's own lines are kept beside what `write` already
910	941	// logs, so a failure can be summarised once here — never by reading
		@@ -1185,9 +1216,15 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
1185	1216	signal: controller.signal,
1186	1217	});
1187	1218	if (done.ok) return { ok: true, error: "", lines };
1188		// Cut off by this forge stopping, or the Job lost under it: the
1189		// code was not judged, so this is no red.
1190		if (stopped \|\| controller.signal.aborted \|\| done.error === JOB_VANISHED)
	1219	// Cut off by this forge stopping, the Job lost under it, or an
	1220	// install the machine failed twice: the code was not judged, so
	1221	// this is no red.
	1222	if (
	1223	stopped \|\|
	1224	controller.signal.aborted \|\|
	1225	done.error === JOB_VANISHED \|\|
	1226	done.infrastructure === true
	1227	)
1191	1228	return { ok: false, error: done.error, lines, interrupted: true };
1192	1229	return { ok: false, error: done.error, lines };
1193	1230	} finally {
		@@ -459,6 +459,9 @@ export const REQUEUED_SUMMARY = "interrupted by a restart; queued again";
459	459	/** What a check whose cluster Job vanished says as it is queued again. */
460	460	export const VANISHED_SUMMARY = "interrupted: its cluster job vanished; queued again";
461	461
	462	/** What a check whose install the machine failed twice says as it is queued again (#5093). */
	463	export const INFRA_SUMMARY = "failed: infrastructure — the install failed twice on a transient error; queued again";
	464
462	465	/**
463	466	* How many times one check may be cut off by its forge going away and be
464	467	* queued again. Past it the check fails once, saying so: a check that kills
		@@ -521,6 +524,11 @@ export function vanishedBefore(db: Database, run: Run): boolean {
521	524	return cutOffBefore(db, run, VANISHED_SUMMARY);
522	525	}
523	526
	527	/** Whether the attempt just before this one failed on infrastructure: it is queued again once. */
	528	export function infraBefore(db: Database, run: Run): boolean {
	529	return cutOffBefore(db, run, INFRA_SUMMARY);
	530	}
	531
524	532	function cutOffBefore(db: Database, run: Run, prefix: string): boolean {
525	533	return (
526	534	db
		@@ -0,0 +1,230 @@
	1	/**
	2	* A check's install (#5093): installs into one cache take turns, a transient
	3	* install failure is retried once, and failing the same way twice is the
	4	* machine's — queued again, never the pull's red.
	5	*/
	6	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	7	import { existsSync, mkdirSync, writeFileSync } from "node:fs";
	8	import { mkdtemp, rm } from "node:fs/promises";
	9	import { tmpdir } from "node:os";
	10	import { join } from "node:path";
	11	import { createApp, type App } from "../src/app.ts";
	12	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	13	import {
	14	checkInstallCache,
	15	INFRASTRUCTURE,
	16	installWithRetry,
	17	transientLine,
	18	withLock,
	19	} from "../src/checks/install.ts";
	20	import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
	21	import { INFRA_SUMMARY, listRuns, type Run } from "../src/core/runs.ts";
	22	import { createUser } from "../src/core/users.ts";
	23	import type { Repo } from "../src/core/repos.ts";
	24	import { serve, type Servers } from "../src/http/server.ts";
	25	import { err, ok, type Result } from "../src/kernel/result.ts";
	26
	27	const never = new AbortController().signal;
	28
	29	describe("transientLine", () => {
	30	test("names the three failures the machine causes, and nothing the code does", () => {
	31	expect(transientLine(['error: Fail extracting tarball for "@typescript/typescript-linux-x64"'])).toContain("Fail extracting");
	32	expect(transientLine(["npm ERR! code EINTEGRITY"])).toContain("EINTEGRITY");
	33	expect(transientLine(["error: ECONNRESET reading registry.npmjs.org"])).toContain("ECONNRESET");
	34	expect(transientLine(["error: No version matching \"9.9.9\" found", "exit 1"])).toBeUndefined();
	35	});
	36	});
	37
	38	describe("installWithRetry", () => {
	39	const scripted = (outputs: (string \| null)[]) => {
	40	let calls = 0;
	41	const attempt = async (log: (line: string) => void): Promise<Result<void, string>> => {
	42	const line = outputs[calls++] ?? null;
	43	if (line === null) return ok();
	44	log(line);
	45	return err(`install exited with code 1 — ${line}`);
	46	};
	47	return { attempt, calls: () => calls };
	48	};
	49
	50	test("a transient failure is retried once and the retry's green is the answer", async () => {
	51	const s = scripted(['error: Fail extracting tarball for "x"', null]);
	52	const lines: string[] = [];
	53	const done = await installWithRetry(s.attempt, { log: (l) => lines.push(l), signal: never, retryMs: 1 });
	54	expect(done).toEqual({ ok: true });
	55	expect(s.calls()).toBe(2);
	56	expect(lines.some((l) => l.includes("retrying once"))).toBe(true);
	57	});
	58
	59	test("the code's own failure is answered at once, and is not infrastructure", async () => {
	60	const s = scripted(['error: No version matching "9.9.9"']);
	61	const done = await installWithRetry(s.attempt, { log: () => {}, signal: never, retryMs: 1 });
	62	expect(s.calls()).toBe(1);
	63	expect(done.ok).toBe(false);
	64	if (!done.ok) expect(done.infrastructure).toBe(false);
	65	});
	66
	67	test("failing transiently twice is infrastructure, tried exactly twice", async () => {
	68	const s = scripted(["EINTEGRITY", "ECONNRESET", null]);
	69	const done = await installWithRetry(s.attempt, { log: () => {}, signal: never, retryMs: 1 });
	70	expect(s.calls()).toBe(2);
	71	expect(done.ok).toBe(false);
	72	if (!done.ok) {
	73	expect(done.infrastructure).toBe(true);
	74	expect(done.error.startsWith(`${INFRASTRUCTURE}:`)).toBe(true);
	75	}
	76	});
	77
	78	test("a cancel during the wait does not retry", async () => {
	79	const s = scripted(["ECONNRESET", null]);
	80	const abort = new AbortController();
	81	const pending = installWithRetry(s.attempt, { log: () => {}, signal: abort.signal, retryMs: 60_000 });
	82	abort.abort();
	83	const done = await pending;
	84	expect(s.calls()).toBe(1);
	85	expect(done.ok).toBe(false);
	86	if (!done.ok) expect(done.infrastructure).toBe(false);
	87	});
	88	});
	89
	90	describe("withLock", () => {
	91	let dir: string;
	92	beforeAll(async () => {
	93	dir = await mkdtemp(join(tmpdir(), "forge-install-lock-"));
	94	});
	95	afterAll(async () => {
	96	await rm(dir, { recursive: true, force: true });
	97	});
	98
	99	test("8 parallel installs on one cache, 20 rounds: none ever extracts beside another", async () => {
	100	// Each install "extracts" the same package: a directory that exists only
	101	// while one install is in it. `mkdir` fails when another is in there —
	102	// the collision bun reports as `Fail extracting tarball`.
	103	const cache = join(dir, "cache");
	104	mkdirSync(cache, { recursive: true });
	105	const install = async (): Promise<number> => {
	106	const proc = Bun.spawn(
	107	["/bin/sh", "-c", 'mkdir "$1/pkg.extracting" && sleep 0.005 && rmdir "$1/pkg.extracting"', "sh", cache],
	108	{ stdin: "ignore", stdout: "ignore", stderr: "ignore" },
	109	);
	110	return proc.exited;
	111	};
	112	let failures = 0;
	113	for (let round = 0; round < 20; round++) {
	114	const codes = await Promise.all(
	115	Array.from({ length: 8 }, () => withLock(`${cache}.lock`, install, { signal: never })),
	116	);
	117	for (const code of codes) if (!code.ok \|\| code.value !== 0) failures++;
	118	}
	119	expect(failures).toBe(0);
	120	expect(existsSync(`${cache}.lock`)).toBe(false);
	121	}, 60_000);
	122
	123	test("a lock left by a holder that died is taken over", async () => {
	124	const lock = join(dir, "dead.lock");
	125	// A pid no process holds: the kernel's pid_max is far below it.
	126	writeFileSync(lock, `${2 ** 30} ${Date.now()}\n`);
	127	const held = await withLock(lock, async () => "ran", { signal: never });
	128	expect(held).toEqual({ ok: true, value: "ran" });
	129	});
	130
	131	test("a live holder's lock is waited for, and a cancel ends the wait without running", async () => {
	132	const lock = join(dir, "live.lock");
	133	writeFileSync(lock, `${process.pid} ${Date.now()}\n`);
	134	const abort = new AbortController();
	135	let ran = false;
	136	const pending = withLock(lock, async () => {
	137	ran = true;
	138	}, { signal: abort.signal });
	139	await Bun.sleep(120);
	140	expect(ran).toBe(false);
	141	abort.abort();
	142	const held = await pending;
	143	expect(held.ok).toBe(false);
	144	expect(ran).toBe(false);
	145	expect(existsSync(lock)).toBe(true);
	146	});
	147	});
	148
	149	describe("a check whose install fails on the machine", () => {
	150	let dataDir: string;
	151	let app: App;
	152	let servers: Servers;
	153	let checks: ChecksHandle;
	154	let flaky: Repo;
	155	let broken: Repo;
	156
	157	// Fails once with bun's own line, then installs: the marker lives in the
	158	// checkout, which the retry shares and the next check does not.
	159	const once =
	160	'test -f .tried && exit 0; touch .tried; echo \'error: Fail extracting tarball for "@typescript/typescript-linux-x64"\'; exit 1\n';
	161	const always = "echo 'error: Fail extracting tarball for \"x\"'; exit 1\n";
	162	const forgeJson = '{"check": {"run": "exit 0", "install": "sh install.sh"}}\n';
	163
	164	beforeAll(async () => {
	165	dataDir = await mkdtemp(join(tmpdir(), "forge-check-install-"));
	166	app = createApp({ dataDir, port: 0 });
	167	servers = await serve(app);
	168	checks = startChecks(app, { forgeUrl: servers.url, concurrency: 2, installRetryMs: 10 });
	169	const a = await createUser(app.db, {
	170	origin: "seed",
	171	trusted: true,
	172	username: "alice",
	173	email: "[email protected]",
	174	password: "a-good-passphrase",
	175	adminIfFirst: true,
	176	});
	177	if (!a.ok) throw new Error(a.error);
	178	const repoWith = async (name: string, script: string): Promise<Repo> => {
	179	const made = await createRepo(app, a.value, { name, visibility: "public" });
	180	if (!made.ok) throw new Error(String(made.error));
	181	const c = await commitToBranch(app, a.value, made.value, {
	182	message: "Declare a check",
	183	files: [
	184	{ path: "forge.json", content: forgeJson },
	185	{ path: "install.sh", content: script },
	186	],
	187	});
	188	if (!c.ok) throw new Error(JSON.stringify(c.error));
	189	return made.value;
	190	};
	191	flaky = await repoWith("flaky", once);
	192	broken = await repoWith("broken", always);
	193	});
	194	afterAll(async () => {
	195	await checks.stop();
	196	servers.stop();
	197	app.close();
	198	await rm(dataDir, { recursive: true, force: true });
	199	});
	200
	201	const settled = async (repo: Repo, done: (runs: Run[]) => boolean): Promise<Run[]> => {
	202	const deadline = Date.now() + 30_000;
	203	let runs: Run[] = [];
	204	while (Date.now() < deadline) {
	205	runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
	206	if (done(runs)) break;
	207	await Bun.sleep(50);
	208	}
	209	return runs;
	210	};
	211
	212	test("one transient failure is retried in the same check, which passes", async () => {
	213	const runs = await settled(flaky, (rs) => rs.some((r) => r.state === "ok" \|\| r.state === "failed"));
	214	expect(runs.length).toBe(1);
	215	expect(runs[0]!.state).toBe("ok");
	216	// Uncontained, the repository's own cache is the one bun is pointed at.
	217	expect(existsSync(checkInstallCache(dataDir, flaky.id))).toBe(true);
	218	});
	219
	220	test("failing twice is infrastructure: queued again, not red; a second time it fails saying so", async () => {
	221	const runs = await settled(broken, (rs) => rs.some((r) => r.state === "failed"));
	222	const first = runs.find((r) => r.attempt === 0)!;
	223	const second = runs.find((r) => r.attempt === 1)!;
	224	expect(first.state).toBe("cancelled");
	225	expect(first.summary).toBe(INFRA_SUMMARY);
	226	expect(second.state).toBe("failed");
	227	expect(second.summary).toContain(`${INFRASTRUCTURE}:`);
	228	expect(runs.length).toBe(2);
	229	});
	230	});