36f0b35
@@ -0,0 +1,10 @@
# A forge at its CPU limit is slowed, not stopped
A child forge is held to its size's cores (small is 1). Where the kernel's `cpu.max` is ours to write, the cell's cgroup holds it. Where it is not (`apps/cgroup.ts`, `cpuCapped` false), the watch (`watchTree`, `apps/contain.ts`) slows the cell itself: `apps/throttle.ts` pauses the whole process tree (SIGSTOP) for a share of each tick and lets it go (SIGCONT), so over a few seconds it gets `limit` cores. Using exactly the limit is not over it.
- **Stopping** for CPU is the last resort: more than twice the limit for five minutes, in a cell at least ten minutes old (`throttleFailed`). Memory, disk and process-count stops are unchanged.
- **The owner is told**: a stop by the host writes `host-stopped.json` in the forge's data directory (`spawn/stop_note.ts`); the forge's home page says what happened and why for a day.
- **Work fits the size** (`spawn/fit.ts`): on a 1-core forge, while a crew run is running, checks are not started, scheduled snapshots are not taken, and looks wait (their age restarts while they wait). One browser at a time: pictures and the accessibility check share `looks/browser_gate.ts`.
- **A run cut off by a stop** is queued again in the same clone and session (`requeue`, `crew/dispatch.ts`). Cut off twice in a row (`crew/interrupt_loop.ts`) the issue is parked with a plain note instead of looping.
Not done here: the sizes' measured first-change times (the plan lines say what each size is for, not a number), the production probe, and the real-1-CPU-cell three-minute test, which needs a cell the sandbox cannot make.
@@ -38,6 +38,7 @@
import { dirname, join } from "node:path";
import { portInsteadOfSocket } from "../git/hook-address.ts";
import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
import { pauseMs, signalAll, THROTTLE_WINDOW_MS, throttleFailed, throttleFailure, type OverState } from "./throttle.ts";
import {
  cgroupHost,
  diskDevice,
@@ -717,7 +718,7 @@
export function openCell(
      breach = why;
      log(`stopped: ${why}`);
      stop(why);
    });
    }, { throttle: spec.cpuQuota === true && cgroup?.cpuCapped !== true });
  };
  return {
    argv,
@@ -875,6 +876,21 @@
interface Watched {
  readonly cap: MemoryCap;
  /** The tree's CPU ticks so far and resident KiB, at the last tick. */
  last?: { ticks: number; kib: number };
  /** Slowed (SIGSTOP/SIGCONT, `apps/throttle.ts`) rather than stopped when over its cores. */
  readonly throttle: boolean;
  readonly born: number;
  readonly over: OverState;
  /** The pids paused right now, and the timer that lets them go. */
  paused: number[];
  resume?: ReturnType<typeof setTimeout> | undefined;
}
/** Let a throttled tree run again: always, before it is stopped or unwatched. */
function release(entry: Watched): void {
  if (entry.resume !== undefined) clearTimeout(entry.resume);
  entry.resume = undefined;
  if (entry.paused.length > 0) signalAll(entry.paused, "SIGCONT");
  entry.paused = [];
}
const watched = new Set<Watched>();
@@ -1043,8 +1059,13 @@
export function watchTree(
  writable: readonly string[],
  label: string | undefined,
  breach: (why: string) => void,
  options: { readonly throttle?: boolean | undefined } = {},
): () => void {
  const entry: Watched = {
    throttle: options.throttle === true,
    born: Date.now(),
    over: {},
    paused: [],
    root,
    limits,
    writable,
@@ -1058,6 +1079,7 @@
export function watchTree(
  watched.add(entry);
  timer ??= setInterval(() => during("cell-watch", tick), WATCH.tickMs);
  return () => {
    release(entry);
    watched.delete(entry);
    if (watched.size === 0 && timer !== undefined) {
      clearInterval(timer);
@@ -1243,7 +1265,28 @@
function tick(): void {
    )
      entry.samples.shift();
    const oldest = entry.samples[0];
    if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
    if (entry.throttle) {
      // Slow it, never stop it for being busy: pause the tree for a share of the next tick.
      release(entry);
      const base = [...entry.samples].reverse().find((s) => now - s.at >= THROTTLE_WINDOW_MS);
      if (base !== undefined) {
        const cores = (ticks - base.ticks) / CLK_TCK / ((now - base.at) / 1000);
        const ms = pauseMs(cores, limits.cpus, WATCH.tickMs);
        if (ms > 0) {
      
        sentinelcrew
        6m ago
        
      
      SIGSTOPped pids are resumed by a timer only. A host crash leaves the tree frozen, and the pids may be reused by the time the timer fires. Re-check before signalling and resume on boot.
    
          entry.paused = tree;
          signalAll(tree, "SIGSTOP");
          entry.resume = setTimeout(() => release(entry), ms);
        }
      }
      if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
        const cores = (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
        if (throttleFailed(entry.over, cores, limits.cpus, entry.born, now)) {
          release(entry);
          entry.breach(throttleFailure(cores, limits.cpus));
          continue;
        }
      }
    } else if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
      const cores =
        (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
      if (cores > limits.cpus) {
@@ -0,0 +1,68 @@
/**
 * Holding a cell to its cores by slowing it, where the kernel's `cpu.max`
 * is not ours to write (#11539). A forge at its limit runs slower; nothing
 * is killed for being busy. The cell's whole process tree is paused
 * (SIGSTOP) for a share of each watch tick and let go (SIGCONT), so that
 * over a few seconds it gets `limit` cores: duty = limit / cores.
 *
 * Stopping the cell for CPU remains only as a last resort, for a tree the
 * pause cannot hold: over twice its limit for five minutes, and never in
 * the cell's first ten minutes.
 */
/** The window the pause is sized from, ms: short, so a burst is slowed within seconds. */
export const THROTTLE_WINDOW_MS = 5_000;
/** A pause shorter than this share of a tick is not worth the signals; none is longer than the most share. */
export const PAUSE_MIN_SHARE = 0.05;
export const PAUSE_MAX_SHARE = 0.9;
/** Last resort: over `LAST_RESORT_FACTOR` × limit, for this long, in a cell at least this old. */
export const LAST_RESORT_FACTOR = 2;
export const LAST_RESORT_MS = 5 * 60_000;
export const LAST_RESORT_MIN_AGE_MS = 10 * 60_000;
/**
 * How long to pause the tree over the next tick, ms (0: let it run).
 * It used `cores` over the throttle window against `limit`; running a
 * share `limit / cores` of each tick averages `limit`. Using exactly the
 * limit is not over it.
 */
export function pauseMs(cores: number, limit: number, tickMs: number): number {
  if (!(cores > limit) || !(limit > 0) || !Number.isFinite(limit)) return 0;
  const ms = Math.round(tickMs * (1 - limit / cores));
  return ms < tickMs * PAUSE_MIN_SHARE ? 0 : Math.min(ms, Math.floor(tickMs * PAUSE_MAX_SHARE));
}
/** What the last-resort rule remembers between ticks: since when the tree has been over twice its limit. */
export interface OverState {
  since?: number | undefined;
}
/**
 * Whether the throttle has failed: `cores` (over the 30 s CPU window) above
 * `LAST_RESORT_FACTOR` × `limit` since `LAST_RESORT_MS` ago, in a cell born
 * `LAST_RESORT_MIN_AGE_MS` ago. Dropping under the line forgets the clock.
 */
export function throttleFailed(state: OverState, cores: number, limit: number, born: number, now: number): boolean {
  if (!(cores > limit * LAST_RESORT_FACTOR)) {
    state.since = undefined;
    return false;
  }
  state.since ??= now;
  return now - state.since >= LAST_RESORT_MS && now - born >= LAST_RESORT_MIN_AGE_MS;
}
/** The line a cell is stopped with when the throttle could not hold it. */
export function throttleFailure(cores: number, limit: number): string {
  return `used ${cores.toFixed(1)} cores for ${LAST_RESORT_MS / 60_000} minutes while slowed, over twice its limit of ${limit}`;
}
/** Signal `pids`, ignoring the ones already gone or not ours. */
export function signalAll(pids: readonly number[], signal: "SIGSTOP" | "SIGCONT"): void {
  for (const pid of pids) {
    try {
      process.kill(pid, signal);
    } catch {
      // gone, or not ours to signal
    }
  }
}
@@ -11,6 +11,7 @@
 * is that command's business.
 */
import { crewHasTheCore, crewRunActive } from "../spawn/fit.ts";
import { pullDir, runDir } from "../core/pulls.ts";
import { cellHides } from "../spawn/hosted-mark.ts";
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
@@ -626,6 +627,8 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
    if (stopped) return;
    const limit = capacity();
    if (options.gate === undefined) gate.resize(limit);
    // On one core the crew run has it: a check waits until no run is active (`spawn/fit.ts`).
    if (crewHasTheCore(() => crewRunActive(app.db))) return;
      
        sentinelcrew
        6m ago
        
      
      Unbounded wait: checks never start while a crew run is active on a 1-core forge. Add a maximum wait so the queue cannot starve, with a test.
    
    startVerify();
    // Every merge-queue gate in flight holds a slot, a line's first too (#6376).
    while (slotted() < limit - gatesInFlight()) {
@@ -112,6 +112,7 @@
import { crewWeekFor, weekFor, weekScope } from "./week_read.ts";
import { announceGovernor, governorFor } from "./governed.ts";
import { BREAKER_NOTE, breakerTripped } from "./breaker.ts";
import { REPEAT_NOTE } from "./repeat_note.ts";
import { INTERRUPTED_TWICE_NOTE } from "./interrupt_loop.ts";
import { given, pendingReplans } from "../core/limb_replan.ts";
import { handedOverPulls, SKIP_SUMMARY, skipsReplan } from "./guard.ts";
import { atRunCap, type Held, heldBy, heldWords, replannable } from "./held.ts";
@@ -990,7 +991,7 @@
export function parkForBreaker(
  const last = listComments(app.db, issue.id).at(-1);
  if (last?.body === note) return false;
  addComment(app, author, repo, issue, note);
  log.info(note === BREAKER_NOTE ? "parked by the breaker" : note === REPEAT_NOTE ? "parked by the repeat stop" : "parked at the run cap", {
  log.info(note === BREAKER_NOTE ? "parked by the breaker" : note === REPEAT_NOTE ? "parked by the repeat stop" : note === INTERRUPTED_TWICE_NOTE ? "parked: cut off twice in a row" : "parked at the run cap", {
    repo: `${repo.ownerName}/${repo.name}`,
    issue: issue.number,
  });
@@ -80,6 +80,7 @@
import { detectClaude } from "./claude.ts";
import { followDivergence, settleAll, settleOnPush } from "./converge.ts";
import { breakerTripped } from "./breaker.ts";
import { REPEAT_NOTE, repeatTripped } from "./repeat.ts";
import { INTERRUPTED_TWICE_NOTE, INTERRUPTIONS_IN_A_ROW, interruptedInARow } from "./interrupt_loop.ts";
import { unstickOwed } from "./stalls.ts";
import {
  CAP_NOTE,
@@ -1385,6 +1386,12 @@
function requeue(app: App, run: Run, why: string): void {
  cancelRun(app.db, run.id, why);
  const task = taskOf(app, run);
  if (task === undefined) return;
  // Cut off twice in a row: the forge says so plainly, and does not loop (#11539).
  if (run.issueId !== null && interruptedInARow(app.db, run.issueId) >= INTERRUPTIONS_IN_A_ROW) {
    parkForBreaker(app, task.repo, task.issue, task.agent, INTERRUPTED_TWICE_NOTE);
    log.info("run not queued again: cut off twice in a row", { run: run.id });
    return;
  }
  // The run that starts carries on in the clone this one left, in the
  // same session — an interruption is a pause, not a fresh start.
  queueAgentRun(
@@ -0,0 +1,39 @@
/**
 * A run cut off by its forge being stopped goes back on the queue and
 * carries on in the same clone and session. Cut off twice in a row, the
 * forge says so plainly rather than looping (#11539): the cycle of stop,
 * queue again, stop again is what spent a forge's credit.
 */
import type { Database } from "bun:sqlite";
/** The words a restart's cancel is recorded with (`requeue` in `dispatch.ts`). */
export const RESTART_CANCEL = "interrupted by a restart";
/** How many restarts in a row end the carrying on. */
export const INTERRUPTIONS_IN_A_ROW = 2;
/** What the issue is parked with, once. */
export const INTERRUPTED_TWICE_NOTE =
  "Parked by the crew: this forge was stopped twice in a row while the crew worked here, so the run is not started again by itself. What the crew had done is kept in its clone; say go and it carries on from there.";
/**
 * How many of the issue's latest ended agent runs, newest first, were cut
 * off by a restart, without a run that ended another way between them.
 */
export function interruptedInARow(db: Database, issueId: string): number {
  const rows = db
    .query<{ summary: string }, [string, number]>(
      `SELECT summary FROM run
       WHERE issue_id = ?1 AND kind = 'agent'
         AND state IN ('ok', 'failed', 'cancelled')
       ORDER BY id DESC LIMIT ?2`,
    )
    .all(issueId, INTERRUPTIONS_IN_A_ROW + 1);
  let n = 0;
  for (const row of rows) {
    if (!row.summary.includes(RESTART_CANCEL)) break;
    n++;
  }
  return n;
}
@@ -0,0 +1,7 @@
/**
 * At most one browser at a time in this forge (#11539): a picture and an
 * accessibility check share it, so two Chromiums never compete for a core.
 */
import { Gate } from "../git/env.ts";
export const browserGate = new Gate(1);
@@ -38,7 +38,7 @@
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no
import { tmpdir } from "node:os";
import { join } from "node:path";
import { childEnv } from "../apps/process.ts";
import { Gate } from "../git/env.ts";
import { browserGate } from "./browser_gate.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import type { Entries } from "../release/webvitals.ts";
@@ -107,7 +107,7 @@
export function findChromium(): string | undefined {
  return undefined;
}
const gate = new Gate(1);
const gate = browserGate;
/** Take one picture. Serialised: the second caller waits for the first. */
export async function takeLook(
@@ -15,6 +15,7 @@
 * environment does — a preview's go when its pull closes.
 */
import { crewHasTheCore, crewRunActive } from "../spawn/fit.ts";
import { sealedPreview } from "../core/sealed.ts";
import {
  closeSync,
@@ -88,6 +89,11 @@
export interface LooksOptions {
   * while two agent runs are at their peak rather than being the straw.
   */
  readonly pressure?: (() => number | undefined) | undefined;
  /**
   * Whether the crew has the core: on a forge of one CPU a look waits while
   * a run is active (`spawn/fit.ts`). Read from the forge's size unless a test says otherwise.
   */
  readonly crewBusy?: (() => boolean) | undefined;
  /** The clock a failed look's backoff waits on: the real one unless a test says otherwise. */
  readonly clock?: BackoffClock | undefined;
}
@@ -140,8 +146,13 @@
export interface LooksHandle {
  stop(): Promise<void>;
}
/** A look that has waited longer than this is stale, and is dropped. */
/**
 * A look that has waited longer than this is stale, and is dropped. A look
 * held for the crew keeps its place: its age is counted from the run's end.
 */
const MAX_WAIT_MS = 10 * 60_000;
/** How long a look waits before it asks again whether the crew is still at work. */
const CREW_WAIT_MS = 5_000;
/** How often production is checked for a picture it lacks. */
const SWEEP_MS = 10 * 60_000;
/** The first check, once a booted forge's apps have had a moment. */
@@ -349,6 +360,13 @@
export function startLooks(app: App, options: LooksOptions): LooksHandle {
    for (;;) {
      const next = queue.shift();
      if (next === undefined || stopped) return;
      // On one core the crew run has it: the look waits, and its age restarts when the run ends.
      if ((options.crewBusy ?? (() => crewHasTheCore(() => crewRunActive(app.db))))()) {
        if (stopped) return;
        queue.unshift({ ...next, at: Date.now() });
        await Bun.sleep(CREW_WAIT_MS);
        continue;
      }
      if (Date.now() - next.at > MAX_WAIT_MS) {
        log.warn("look dropped: waited too long", {
          app: next.appId,
@@ -6,11 +6,14 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { browserGate } from "../looks/browser_gate.ts";
const STEP_MS = 15000;
/** A DevTools connection to one page of a private headless Chromium. */
export async function withPage<T>(binary: string, use: (send: (method: string, params?: object) => Promise<any>, once: (event: string) => Promise<void>) => Promise<T>): Promise<T> {
  // One browser at a time: a picture being taken has it first (`looks/browser_gate.ts`).
  const release = await browserGate.acquire();
      
        sentinelcrew
        6m ago
        
      
      High: the gate is acquired at line 10 of the function, but spawn (and mkdtemp) are outside the try. A throw leaks the gate and blocks every later browser use. Acquire inside the try or wrap the spawn.
    
  const profile = mkdtempSync(join(tmpdir(), "forge-axe-"));
  const proc = Bun.spawn(
    [binary, "--headless=new", "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-background-networking", "--no-first-run", `--user-data-dir=${profile}`, "--remote-debugging-port=0", "about:blank"],
@@ -87,5 +90,6 @@
export async function withPage<T>(binary: string, use: (send: (method: string, p
    proc.kill("SIGKILL");
    await proc.exited.catch(() => {});
    rmSync(profile, { recursive: true, force: true });
    release();
  }
}
@@ -10,6 +10,7 @@
 * is within the hour, not the next night.
 */
import { crewHasTheCore, crewRunActive } from "../spawn/fit.ts";
import type { App } from "../app.ts";
import { createLog } from "../kernel/log.ts";
import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts";
@@ -267,6 +268,8 @@
export function startSnapshots(
  // The manifest is written last, so a snapshot cut short is not one.
  const takeIfDue = async (): Promise<boolean> => {
    if (stopped || active > 0) return false;
    // On one core the crew run has it: the scheduled snapshot waits for the next pass (`spawn/fit.ts`).
    if (crewHasTheCore(() => crewRunActive(app.db))) return false;
    const store = current();
    if (store === undefined) return false;
    let newest: Manifest | undefined;
@@ -8,6 +8,7 @@
 * (`spawn/placement.ts` `fallbackPlacement`), and the ledger says which.
 */
import { noteStop } from "./stop_note.ts";
import { Database } from "bun:sqlite";
import { existsSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -356,7 +357,11 @@
export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
    });
    const exited = proc.exited.then((code) => {
      const why = cell.breach();
      if (why !== undefined) parent.log.warn("child forge stopped", { name: input.name, why });
      if (why !== undefined) {
        parent.log.warn("child forge stopped", { name: input.name, why });
        // The owner reads it on their home page (`spawn/stop_note.ts`).
        noteStop(dir, why);
      }
      cell.close();
      door.close();
      release();
@@ -0,0 +1,35 @@
/**
 * Work scheduled to fit the forge's size (#11539). On one CPU a crew run is
 * the product and has the core: looks, checks and snapshots wait until no
 * run is active, so nothing competes with it for the time the host gives.
 * A forge that is not a cell child (no size in its environment) and one of
 * two or more cores are not held: they have the room.
 */
import type { Database } from "bun:sqlite";
import { isSize, SIZES } from "./sizes.ts";
import { SIZE_ENV } from "./usage.ts";
/** The cores this forge is held to, or undefined where it is not a sized cell child. */
export function forgeCores(env: Readonly<Record<string, string | undefined>> = process.env): number | undefined {
  const size = env[SIZE_ENV];
  return isSize(size) ? SIZES[size].cpu : undefined;
}
/** Whether a crew run is working now: one `running` agent run. */
export function crewRunActive(db: Database): boolean {
  return db.query("SELECT 1 FROM run WHERE kind = 'agent' AND state = 'running' LIMIT 1").get() !== null;
}
/**
 * Whether the other work (a look, a check, a snapshot) waits for the crew:
 * on a forge of one core, while a run is active. `active` is read only when
 * the forge has one core, so a larger forge pays no query.
 */
export function crewHasTheCore(
  active: () => boolean,
  env: Readonly<Record<string, string | undefined>> = process.env,
): boolean {
  const cores = forgeCores(env);
  return cores !== undefined && cores <= 1 && active();
}
@@ -55,8 +55,8 @@
export const isSize = (v: unknown): v is Size => typeof v === "string" && Object
// What each size is for, never how big: the figures live only in SIZES.
const SAYS: Readonly<Record<Size, string>> = {
  small: "For a notes app, a small tool, a place to try things.",
  medium: "For a busy app, or a few apps side by side.",
  small: "For a notes app, a small tool, a place to try things. The crew has its only core while it builds, so a change takes longer and checks and pictures wait until the crew is idle.",
  medium: "For a busy app, or a few apps side by side; checks and pictures run beside the crew.",
      
        sentinelcrew
        6m ago
        
      
      Principle 1: one confident sentence per size. This line now explains scheduling twice over.
    
  large: "For a crew's working forge, with room to build.",
};
@@ -0,0 +1,49 @@
/**
 * What a forge tells its owner when the host stopped it (#11539): the host
 * writes one small file in the forge's own data directory, the forge reads
 * it on its home page for a day. Plain data in, plain data out.
 */
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
const FILE = "host-stopped.json";
/** How long the note stays on the home page, ms. */
export const NOTE_MS = 24 * 60 * 60_000;
export interface StopNote {
  readonly at: string;
  readonly why: string;
}
/** Record that the host stopped the forge whose data is `dataDir`, and why. Best effort: a note never stops a stop. */
export function noteStop(dataDir: string, why: string, now = new Date()): void {
  try {
    mkdirSync(dataDir, { recursive: true });
    const tmp = join(dataDir, `${FILE}.tmp`);
    writeFileSync(tmp, JSON.stringify({ at: now.toISOString(), why: why.slice(0, 400) }));
    renameSync(tmp, join(dataDir, FILE));
  } catch {
    // the stop stands without its note
  }
}
/** The note, while it is a day old at most; undefined when there is none or it is not well formed. */
export function readStop(dataDir: string, now = Date.now()): StopNote | undefined {
  try {
    const raw: unknown = JSON.parse(readFileSync(join(dataDir, FILE), "utf8"));
    if (typeof raw !== "object" || raw === null) return undefined;
    const { at, why } = raw as Record<string, unknown>;
    if (typeof at !== "string" || typeof why !== "string") return undefined;
    const when = Date.parse(at);
    if (!Number.isFinite(when) || now - when > NOTE_MS || when > now + 60_000) return undefined;
    return { at, why: why.slice(0, 400) };
  } catch {
    return undefined;
  }
}
/** The line on the owner's home page. */
export function stopWords(note: StopNote): string {
  const time = note.at.slice(11, 16);
  return `This forge was stopped by its host at ${time} UTC: ${note.why}. It started again, and the crew carries on where it stopped.`;
}
@@ -6,6 +6,7 @@
 * best of what is public (`actions/front.ts`).
 */
import { readStop, stopWords } from "../../spawn/stop_note.ts";
import { UPSTREAM_REPO } from "../../core/upstream.ts";
import { foundryHref } from "../../actions/foundry_door.ts";
import { forgeCeiling } from "../../flow/ceiling.ts";
@@ -368,7 +369,10 @@
export async function studioHome(
  // The first page after Allow says so, once.
  const allowed = untoldAnswer(ctx.app.db, user.id) === "allowed";
  if (allowed) markTold(ctx.app.db, user.id, new Date().toISOString());
  const needsYou = html`${requests}${waiting}${held}${alarms}`;
  // A stop by the host is said to the owner, plainly, for a day (`spawn/stop_note.ts`, #11539).
  const stopped = user.isAdmin ? readStop(ctx.app.config.dataDir) : undefined;
  const stopLine = stopped === undefined ? null : html`<p class="small" id="host-stopped" role="status">${stopWords(stopped)}</p>`;
      
        sentinelcrew
        6m ago
        
      
      Both doors: the TUI home has no equivalent of this line. Also 'host' and 'slowed' are not plain words (principle 8).
    
  const needsYou = html`${requests}${waiting}${held}${alarms}${stopLine}`;
  const mine = writableAppsOf(ctx, user).repos;
  const body = html`<h1 class="sr-only">Home</h1>
    ${allowed ? html`<div class="toast" data-tone="success" role="status" id="youre-in"><strong>${YOURE_IN_MESSAGE}</strong> Your account is open: tell the crew what you want to build.</div>` : null}
@@ -0,0 +1,84 @@
/**
 * A cell at its CPU limit is slowed, not stopped (#11539): paused for a
 * share of each tick so it averages its cores, and stopped only when the
 * pause cannot hold it, after five minutes over twice the limit, in a cell
 * ten minutes old.
 */
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { watchTree, WATCH, type Limits } from "../src/apps/contain.ts";
import { LAST_RESORT_MIN_AGE_MS, LAST_RESORT_MS, pauseMs, throttleFailed } from "../src/apps/throttle.ts";
describe("the pause", () => {
  test("at or under the limit nothing is paused: using exactly the limit is not over it", () => {
    expect(pauseMs(1, 1, 1000)).toBe(0);
    expect(pauseMs(0.4, 1, 1000)).toBe(0);
    expect(pauseMs(1.02, 1, 1000)).toBe(0);
  });
  test("over it the tree runs limit/cores of each tick (hand calculation: 2 cores on 1 is half)", () => {
    expect(pauseMs(2, 1, 1000)).toBe(500);
    expect(pauseMs(4, 1, 1000)).toBe(750);
    expect(pauseMs(3, 2, 1000)).toBe(333);
  });
  test("never the whole tick, whatever the demand", () => {
    expect(pauseMs(1000, 1, 1000)).toBe(900);
  });
});
describe("the last resort", () => {
  const born = 0;
  test("five minutes over twice the limit, in a cell past ten minutes, is a failed throttle", () => {
    const s = {};
    const t0 = LAST_RESORT_MIN_AGE_MS;
    expect(throttleFailed(s, 2.5, 1, born, t0)).toBe(false);
    expect(throttleFailed(s, 2.5, 1, born, t0 + LAST_RESORT_MS - 1)).toBe(false);
    expect(throttleFailed(s, 2.5, 1, born, t0 + LAST_RESORT_MS)).toBe(true);
  });
  test("at or near the limit it never is, however long", () => {
    const s = {};
    for (let at = 0; at < 60 * 60_000; at += 10_000) expect(throttleFailed(s, 2, 1, born, at + LAST_RESORT_MIN_AGE_MS)).toBe(false);
  });
  test("a cell younger than ten minutes is never stopped for CPU", () => {
    const s = {};
    expect(throttleFailed(s, 9, 1, 0, 0)).toBe(false);
    expect(throttleFailed(s, 9, 1, 0, LAST_RESORT_MS + 1)).toBe(false);
    expect(throttleFailed(s, 9, 1, 0, LAST_RESORT_MIN_AGE_MS - 1)).toBe(false);
  });
  test("dropping under the line forgets the clock", () => {
    const s = {};
    const t0 = LAST_RESORT_MIN_AGE_MS;
    throttleFailed(s, 3, 1, born, t0);
    throttleFailed(s, 1, 1, born, t0 + 60_000);
    expect(throttleFailed(s, 3, 1, born, t0 + LAST_RESORT_MS)).toBe(false);
  });
});
const cpuSeconds = (pid: number): number => {
  const f = readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]!.split(" ");
  return (Number(f[11]) + Number(f[12])) / 100;
};
test("a CPU-bound process under a throttle runs near its quota and is not stopped", async () => {
  const saved = { ...WATCH };
      
        sentinelcrew
        6m ago
        
      
      A 16 s test measuring wall-clock CPU share will flake on a loaded CI machine. Inject the clock and the signal function and test tick behaviour deterministically.
    
  WATCH.tickMs = 100;
  const hog = Bun.spawn(["bun", "-e", "const end = Date.now() + 30000; while (Date.now() < end) {}"], { stdout: "ignore", stderr: "ignore" });
  let stopped: string | undefined;
  const limits: Limits = { memoryMiB: 4096, cpus: 0.5, diskMiB: 1 << 20 };
  const unwatch = watchTree(hog.pid, limits, { bytes: 4096 << 20, by: "watch" }, [], undefined, (why) => (stopped = why), { throttle: true });
  try {
    await Bun.sleep(10_000); // the window fills and the pause settles
    const from = cpuSeconds(hog.pid);
    const t0 = Date.now();
    await Bun.sleep(6_000);
    const cores = (cpuSeconds(hog.pid) - from) / ((Date.now() - t0) / 1000);
    expect(stopped).toBeUndefined();
    expect(cores).toBeGreaterThan(0.2);
    expect(cores).toBeLessThan(0.8); // unthrottled it is 1.0
    expect(hog.killed).toBe(false);
  } finally {
    unwatch();
    hog.kill("SIGCONT");
    hog.kill();
    Object.assign(WATCH, saved);
  }
}, 40_000);
@@ -0,0 +1,43 @@
/** On a forge of one core the crew run has it; looks, checks and snapshots wait (#11539). */
import { expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { crewHasTheCore, crewRunActive, forgeCores } from "../src/spawn/fit.ts";
import { browserGate } from "../src/looks/browser_gate.ts";
import { withPage } from "../src/release/devtools.ts";
test("a small forge holds other work while a run is active; larger and unsized forges do not", () => {
  expect(forgeCores({ FORGE_CELL_SIZE: "small" })).toBe(1);
  expect(forgeCores({ FORGE_CELL_SIZE: "large" })).toBe(4);
  expect(forgeCores({})).toBeUndefined();
  expect(crewHasTheCore(() => true, { FORGE_CELL_SIZE: "small" })).toBe(true);
  expect(crewHasTheCore(() => false, { FORGE_CELL_SIZE: "small" })).toBe(false);
  expect(crewHasTheCore(() => true, { FORGE_CELL_SIZE: "medium" })).toBe(false);
  expect(crewHasTheCore(() => true, {})).toBe(false);
});
test("a larger forge never asks whether a run is active", () => {
  let asked = 0;
  crewHasTheCore(() => (asked++, true), { FORGE_CELL_SIZE: "large" });
  expect(asked).toBe(0);
});
test("a run is active only while one is running", () => {
  const db = new Database(":memory:");
  db.run("CREATE TABLE run (kind TEXT, state TEXT)");
  expect(crewRunActive(db)).toBe(false);
  db.run("INSERT INTO run VALUES ('agent','queued'), ('check','running')");
  expect(crewRunActive(db)).toBe(false);
  db.run("INSERT INTO run VALUES ('agent','running')");
  expect(crewRunActive(db)).toBe(true);
});
test("one browser at a time: a page waits while a picture holds the gate", async () => {
  const held = await browserGate.acquire();
  let started = false;
  const waiting = withPage("/nonexistent/chromium", async () => 1).catch(() => (started = true));
  await Bun.sleep(100);
  expect(started).toBe(false);
  held();
  await waiting;
  expect(started).toBe(true);
});
@@ -0,0 +1,74 @@
/**
 * A run cut off by a forge stop carries on once; cut off twice in a row the
 * forge says so on the issue and does not queue it again (#11539).
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { openIssue } from "../src/actions/issues.ts";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { claimRun, createRun, getRun, listQueued } from "../src/core/runs.ts";
import { listComments } from "../src/core/issues.ts";
import { requeueInterrupted } from "../src/crew/dispatch.ts";
import { INTERRUPTED_TWICE_NOTE, interruptedInARow } from "../src/crew/interrupt_loop.ts";
import { STALE_AFTER_MS } from "../src/runs/lease.ts";
import { createUser } from "../src/core/users.ts";
let dataDir = "";
let app: App;
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-twice-"));
  app = createApp({ dataDir, port: 0 });
});
afterAll(async () => {
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
test("the first stop queues the run again; the second says so and does not", async () => {
  const made = await createUser(app.db, {
    origin: "seed", trusted: true, username: "alice", email: "[email protected]",
    password: "a-good-passphrase", adminIfFirst: true,
  });
  if (!made.ok) throw new Error(made.error);
  const alice = made.value;
  const bot = await createAgent(app, alice, { username: "bot" });
  if (!bot.ok) throw new Error(String(bot.error));
  const created = await createRepo(app, alice, { name: "loop", visibility: "private" });
  if (!created.ok) throw new Error(String(created.error));
  const repo = created.value;
  expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
  const issued = openIssue(app, alice, repo, { title: "Build it", body: "" });
  if (!issued.ok) throw new Error(String(issued.error));
  const issue = issued.value;
  const lapse = (id: string) =>
    app.db.query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1").run(id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString());
  const cutOff = (id: string) => {
    claimRun(app.db, id);
    lapse(id);
    requeueInterrupted(app);
  };
  const first = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: issue.id, actorId: bot.value.user.id, reason: "assigned" });
  cutOff(first.id);
  expect(getRun(app.db, first.id)!.state).toBe("cancelled");
  const again = listQueued(app.db, "agent").filter((r) => r.issueId === issue.id && r.state === "queued");
  expect(again).toHaveLength(1);
  expect(interruptedInARow(app.db, issue.id)).toBe(1);
  cutOff(again[0]!.id);
  expect(interruptedInARow(app.db, issue.id)).toBe(2);
  expect(listQueued(app.db, "agent").filter((r) => r.issueId === issue.id && r.state === "queued")).toHaveLength(0);
  expect(listComments(app.db, issue.id).some((c) => c.body === INTERRUPTED_TWICE_NOTE)).toBe(true);
});
test("a run that ended another way between two stops breaks the row", () => {
  const rows = [{ summary: "interrupted by a restart; queued again" }, { summary: "built it" }, { summary: "interrupted by a restart; queued again" }];
  const fake = { query: () => ({ all: () => rows }) } as never;
  expect(interruptedInARow(fake, "x")).toBe(1);
});
@@ -0,0 +1,42 @@
/** The owner is told on their home page that the host stopped their forge, and why (#11539). */
import { expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { NOTE_MS, noteStop, readStop, stopWords } from "../src/spawn/stop_note.ts";
test("a stop is noted, read back for a day, and then forgotten", () => {
  const dir = mkdtempSync(join(tmpdir(), "forge-stopnote-"));
  try {
    expect(readStop(dir)).toBeUndefined();
    const at = new Date("2026-11-11T23:36:00Z");
    noteStop(dir, "used 2.9 cores for 5 minutes while slowed, over twice its limit of 1", at);
    expect(readStop(dir, at.getTime() + 1000)?.why).toContain("over twice its limit of 1");
    expect(readStop(dir, at.getTime() + NOTE_MS + 1)).toBeUndefined();
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
});
test("the words give the time and the reason", () => {
  expect(stopWords({ at: "2026-11-11T23:36:00.000Z", why: "used too much" })).toBe(
    "This forge was stopped by its host at 23:36 UTC: used too much. It started again, and the crew carries on where it stopped.",
  );
});
test("the owner's home page says so; a forge never stopped says nothing", async () => {
  const { startAppsForge, signedInCookie, stopAppsForge } = await import("./helpers/apps-forge.ts");
  const forge = await startAppsForge();
  try {
    const cookie = await signedInCookie(forge.servers);
    const read = async () => (await fetch(`${forge.servers.url}/`, { headers: { cookie } })).text();
    expect(await read()).not.toContain("host-stopped");
    noteStop(forge.app.config.dataDir, "used 2.9 cores for 5 minutes while slowed, over twice its limit of 1");
    const page = await read();
    expect(page).toContain('id="host-stopped"');
    expect(page).toContain("This forge was stopped by its host");
    expect(page).toContain("over twice its limit of 1");
  } finally {
    await stopAppsForge(forge);
  }
}, 30_000);
		@@ -0,0 +1,10 @@
	1	# A forge at its CPU limit is slowed, not stopped
	2
	3	A child forge is held to its size's cores (small is 1). Where the kernel's `cpu.max` is ours to write, the cell's cgroup holds it. Where it is not (`apps/cgroup.ts`, `cpuCapped` false), the watch (`watchTree`, `apps/contain.ts`) slows the cell itself: `apps/throttle.ts` pauses the whole process tree (SIGSTOP) for a share of each tick and lets it go (SIGCONT), so over a few seconds it gets `limit` cores. Using exactly the limit is not over it.
	4
	5	- Stopping for CPU is the last resort: more than twice the limit for five minutes, in a cell at least ten minutes old (`throttleFailed`). Memory, disk and process-count stops are unchanged.
	6	- The owner is told: a stop by the host writes `host-stopped.json` in the forge's data directory (`spawn/stop_note.ts`); the forge's home page says what happened and why for a day.
	7	- Work fits the size (`spawn/fit.ts`): on a 1-core forge, while a crew run is running, checks are not started, scheduled snapshots are not taken, and looks wait (their age restarts while they wait). One browser at a time: pictures and the accessibility check share `looks/browser_gate.ts`.
	8	- A run cut off by a stop is queued again in the same clone and session (`requeue`, `crew/dispatch.ts`). Cut off twice in a row (`crew/interrupt_loop.ts`) the issue is parked with a plain note instead of looping.
	9
	10	Not done here: the sizes' measured first-change times (the plan lines say what each size is for, not a number), the production probe, and the real-1-CPU-cell three-minute test, which needs a cell the sandbox cannot make.
		@@ -38,6 +38,7 @@ import { dirname, join } from "node:path";
38	38	import { portInsteadOfSocket } from "../git/hook-address.ts";
39	39	import { relay, startEgress, upstreamProxy, type Egress } from "./egress.ts";
40	40	import { BACKGROUND_CPU_WEIGHT } from "./background.ts";
	41	import { pauseMs, signalAll, THROTTLE_WINDOW_MS, throttleFailed, throttleFailure, type OverState } from "./throttle.ts";
41	42	import {
42	43	cgroupHost,
43	44	diskDevice,
		@@ -717,7 +718,7 @@ export function openCell(
717	718	breach = why;
718	719	log(`stopped: ${why}`);
719	720	stop(why);
720		});
	721	}, { throttle: spec.cpuQuota === true && cgroup?.cpuCapped !== true });
721	722	};
722	723	return {
723	724	argv,
		@@ -875,6 +876,21 @@ interface Watched {
875	876	readonly cap: MemoryCap;
876	877	/** The tree's CPU ticks so far and resident KiB, at the last tick. */
877	878	last?: { ticks: number; kib: number };
	879	/** Slowed (SIGSTOP/SIGCONT, `apps/throttle.ts`) rather than stopped when over its cores. */
	880	readonly throttle: boolean;
	881	readonly born: number;
	882	readonly over: OverState;
	883	/** The pids paused right now, and the timer that lets them go. */
	884	paused: number[];
	885	resume?: ReturnType<typeof setTimeout> \| undefined;
	886	}
	887
	888	/** Let a throttled tree run again: always, before it is stopped or unwatched. */
	889	function release(entry: Watched): void {
	890	if (entry.resume !== undefined) clearTimeout(entry.resume);
	891	entry.resume = undefined;
	892	if (entry.paused.length > 0) signalAll(entry.paused, "SIGCONT");
	893	entry.paused = [];
878	894	}
879	895
880	896	const watched = new Set<Watched>();
		@@ -1043,8 +1059,13 @@ export function watchTree(
1043	1059	writable: readonly string[],
1044	1060	label: string \| undefined,
1045	1061	breach: (why: string) => void,
	1062	options: { readonly throttle?: boolean \| undefined } = {},
1046	1063	): () => void {
1047	1064	const entry: Watched = {
	1065	throttle: options.throttle === true,
	1066	born: Date.now(),
	1067	over: {},
	1068	paused: [],
1048	1069	root,
1049	1070	limits,
1050	1071	writable,
		@@ -1058,6 +1079,7 @@ export function watchTree(
1058	1079	watched.add(entry);
1059	1080	timer ??= setInterval(() => during("cell-watch", tick), WATCH.tickMs);
1060	1081	return () => {
	1082	release(entry);
1061	1083	watched.delete(entry);
1062	1084	if (watched.size === 0 && timer !== undefined) {
1063	1085	clearInterval(timer);
		@@ -1243,7 +1265,28 @@ function tick(): void {
1243	1265	)
1244	1266	entry.samples.shift();
1245	1267	const oldest = entry.samples[0];
1246		if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
	1268	if (entry.throttle) {
	1269	// Slow it, never stop it for being busy: pause the tree for a share of the next tick.
	1270	release(entry);
	1271	const base = [...entry.samples].reverse().find((s) => now - s.at >= THROTTLE_WINDOW_MS);
	1272	if (base !== undefined) {
	1273	const cores = (ticks - base.ticks) / CLK_TCK / ((now - base.at) / 1000);
	1274	const ms = pauseMs(cores, limits.cpus, WATCH.tickMs);
	1275	if (ms > 0) {
		sentinelcrew 6m ago SIGSTOPped pids are resumed by a timer only. A host crash leaves the tree frozen, and the pids may be reused by the time the timer fires. Re-check before signalling and resume on boot.
	1276	entry.paused = tree;
	1277	signalAll(tree, "SIGSTOP");
	1278	entry.resume = setTimeout(() => release(entry), ms);
	1279	}
	1280	}
	1281	if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
	1282	const cores = (ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
	1283	if (throttleFailed(entry.over, cores, limits.cpus, entry.born, now)) {
	1284	release(entry);
	1285	entry.breach(throttleFailure(cores, limits.cpus));
	1286	continue;
	1287	}
	1288	}
	1289	} else if (oldest !== undefined && now - oldest.at >= WATCH.cpuWindowMs) {
1247	1290	const cores =
1248	1291	(ticks - oldest.ticks) / CLK_TCK / ((now - oldest.at) / 1000);
1249	1292	if (cores > limits.cpus) {
		@@ -0,0 +1,68 @@
	1	/**
	2	* Holding a cell to its cores by slowing it, where the kernel's `cpu.max`
	3	* is not ours to write (#11539). A forge at its limit runs slower; nothing
	4	* is killed for being busy. The cell's whole process tree is paused
	5	* (SIGSTOP) for a share of each watch tick and let go (SIGCONT), so that
	6	* over a few seconds it gets `limit` cores: duty = limit / cores.
	7	*
	8	* Stopping the cell for CPU remains only as a last resort, for a tree the
	9	* pause cannot hold: over twice its limit for five minutes, and never in
	10	* the cell's first ten minutes.
	11	*/
	12
	13	/** The window the pause is sized from, ms: short, so a burst is slowed within seconds. */
	14	export const THROTTLE_WINDOW_MS = 5_000;
	15	/** A pause shorter than this share of a tick is not worth the signals; none is longer than the most share. */
	16	export const PAUSE_MIN_SHARE = 0.05;
	17	export const PAUSE_MAX_SHARE = 0.9;
	18	/** Last resort: over `LAST_RESORT_FACTOR` × limit, for this long, in a cell at least this old. */
	19	export const LAST_RESORT_FACTOR = 2;
	20	export const LAST_RESORT_MS = 5 * 60_000;
	21	export const LAST_RESORT_MIN_AGE_MS = 10 * 60_000;
	22
	23	/**
	24	* How long to pause the tree over the next tick, ms (0: let it run).
	25	* It used `cores` over the throttle window against `limit`; running a
	26	* share `limit / cores` of each tick averages `limit`. Using exactly the
	27	* limit is not over it.
	28	*/
	29	export function pauseMs(cores: number, limit: number, tickMs: number): number {
	30	if (!(cores > limit) \|\| !(limit > 0) \|\| !Number.isFinite(limit)) return 0;
	31	const ms = Math.round(tickMs * (1 - limit / cores));
	32	return ms < tickMs * PAUSE_MIN_SHARE ? 0 : Math.min(ms, Math.floor(tickMs * PAUSE_MAX_SHARE));
	33	}
	34
	35	/** What the last-resort rule remembers between ticks: since when the tree has been over twice its limit. */
	36	export interface OverState {
	37	since?: number \| undefined;
	38	}
	39
	40	/**
	41	* Whether the throttle has failed: `cores` (over the 30 s CPU window) above
	42	* `LAST_RESORT_FACTOR` × `limit` since `LAST_RESORT_MS` ago, in a cell born
	43	* `LAST_RESORT_MIN_AGE_MS` ago. Dropping under the line forgets the clock.
	44	*/
	45	export function throttleFailed(state: OverState, cores: number, limit: number, born: number, now: number): boolean {
	46	if (!(cores > limit * LAST_RESORT_FACTOR)) {
	47	state.since = undefined;
	48	return false;
	49	}
	50	state.since ??= now;
	51	return now - state.since >= LAST_RESORT_MS && now - born >= LAST_RESORT_MIN_AGE_MS;
	52	}
	53
	54	/** The line a cell is stopped with when the throttle could not hold it. */
	55	export function throttleFailure(cores: number, limit: number): string {
	56	return `used ${cores.toFixed(1)} cores for ${LAST_RESORT_MS / 60_000} minutes while slowed, over twice its limit of ${limit}`;
	57	}
	58
	59	/** Signal `pids`, ignoring the ones already gone or not ours. */
	60	export function signalAll(pids: readonly number[], signal: "SIGSTOP" \| "SIGCONT"): void {
	61	for (const pid of pids) {
	62	try {
	63	process.kill(pid, signal);
	64	} catch {
	65	// gone, or not ours to signal
	66	}
	67	}
	68	}
		@@ -11,6 +11,7 @@
11	11	* is that command's business.
12	12	*/
13	13
	14	import { crewHasTheCore, crewRunActive } from "../spawn/fit.ts";
14	15	import { pullDir, runDir } from "../core/pulls.ts";
15	16	import { cellHides } from "../spawn/hosted-mark.ts";
16	17	import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
		@@ -626,6 +627,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
626	627	if (stopped) return;
627	628	const limit = capacity();
628	629	if (options.gate === undefined) gate.resize(limit);
	630	// On one core the crew run has it: a check waits until no run is active (`spawn/fit.ts`).
	631	if (crewHasTheCore(() => crewRunActive(app.db))) return;
		sentinelcrew 6m ago Unbounded wait: checks never start while a crew run is active on a 1-core forge. Add a maximum wait so the queue cannot starve, with a test.
629	632	startVerify();
630	633	// Every merge-queue gate in flight holds a slot, a line's first too (#6376).
631	634	while (slotted() < limit - gatesInFlight()) {
		@@ -112,6 +112,7 @@ import { crewWeekFor, weekFor, weekScope } from "./week_read.ts";
112	112	import { announceGovernor, governorFor } from "./governed.ts";
113	113	import { BREAKER_NOTE, breakerTripped } from "./breaker.ts";
114	114	import { REPEAT_NOTE } from "./repeat_note.ts";
	115	import { INTERRUPTED_TWICE_NOTE } from "./interrupt_loop.ts";
115	116	import { given, pendingReplans } from "../core/limb_replan.ts";
116	117	import { handedOverPulls, SKIP_SUMMARY, skipsReplan } from "./guard.ts";
117	118	import { atRunCap, type Held, heldBy, heldWords, replannable } from "./held.ts";
		@@ -990,7 +991,7 @@ export function parkForBreaker(
990	991	const last = listComments(app.db, issue.id).at(-1);
991	992	if (last?.body === note) return false;
992	993	addComment(app, author, repo, issue, note);
993		log.info(note === BREAKER_NOTE ? "parked by the breaker" : note === REPEAT_NOTE ? "parked by the repeat stop" : "parked at the run cap", {
	994	log.info(note === BREAKER_NOTE ? "parked by the breaker" : note === REPEAT_NOTE ? "parked by the repeat stop" : note === INTERRUPTED_TWICE_NOTE ? "parked: cut off twice in a row" : "parked at the run cap", {
994	995	repo: `${repo.ownerName}/${repo.name}`,
995	996	issue: issue.number,
996	997	});
		@@ -80,6 +80,7 @@ import { detectClaude } from "./claude.ts";
80	80	import { followDivergence, settleAll, settleOnPush } from "./converge.ts";
81	81	import { breakerTripped } from "./breaker.ts";
82	82	import { REPEAT_NOTE, repeatTripped } from "./repeat.ts";
	83	import { INTERRUPTED_TWICE_NOTE, INTERRUPTIONS_IN_A_ROW, interruptedInARow } from "./interrupt_loop.ts";
83	84	import { unstickOwed } from "./stalls.ts";
84	85	import {
85	86	CAP_NOTE,
		@@ -1385,6 +1386,12 @@ function requeue(app: App, run: Run, why: string): void {
1385	1386	cancelRun(app.db, run.id, why);
1386	1387	const task = taskOf(app, run);
1387	1388	if (task === undefined) return;
	1389	// Cut off twice in a row: the forge says so plainly, and does not loop (#11539).
	1390	if (run.issueId !== null && interruptedInARow(app.db, run.issueId) >= INTERRUPTIONS_IN_A_ROW) {
	1391	parkForBreaker(app, task.repo, task.issue, task.agent, INTERRUPTED_TWICE_NOTE);
	1392	log.info("run not queued again: cut off twice in a row", { run: run.id });
	1393	return;
	1394	}
1388	1395	// The run that starts carries on in the clone this one left, in the
1389	1396	// same session — an interruption is a pause, not a fresh start.
1390	1397	queueAgentRun(
		@@ -0,0 +1,39 @@
	1	/**
	2	* A run cut off by its forge being stopped goes back on the queue and
	3	* carries on in the same clone and session. Cut off twice in a row, the
	4	* forge says so plainly rather than looping (#11539): the cycle of stop,
	5	* queue again, stop again is what spent a forge's credit.
	6	*/
	7
	8	import type { Database } from "bun:sqlite";
	9
	10	/** The words a restart's cancel is recorded with (`requeue` in `dispatch.ts`). */
	11	export const RESTART_CANCEL = "interrupted by a restart";
	12
	13	/** How many restarts in a row end the carrying on. */
	14	export const INTERRUPTIONS_IN_A_ROW = 2;
	15
	16	/** What the issue is parked with, once. */
	17	export const INTERRUPTED_TWICE_NOTE =
	18	"Parked by the crew: this forge was stopped twice in a row while the crew worked here, so the run is not started again by itself. What the crew had done is kept in its clone; say go and it carries on from there.";
	19
	20	/**
	21	* How many of the issue's latest ended agent runs, newest first, were cut
	22	* off by a restart, without a run that ended another way between them.
	23	*/
	24	export function interruptedInARow(db: Database, issueId: string): number {
	25	const rows = db
	26	.query<{ summary: string }, [string, number]>(
	27	`SELECT summary FROM run
	28	WHERE issue_id = ?1 AND kind = 'agent'
	29	AND state IN ('ok', 'failed', 'cancelled')
	30	ORDER BY id DESC LIMIT ?2`,
	31	)
	32	.all(issueId, INTERRUPTIONS_IN_A_ROW + 1);
	33	let n = 0;
	34	for (const row of rows) {
	35	if (!row.summary.includes(RESTART_CANCEL)) break;
	36	n++;
	37	}
	38	return n;
	39	}
		@@ -0,0 +1,7 @@
	1	/**
	2	* At most one browser at a time in this forge (#11539): a picture and an
	3	* accessibility check share it, so two Chromiums never compete for a core.
	4	*/
	5	import { Gate } from "../git/env.ts";
	6
	7	export const browserGate = new Gate(1);
		@@ -38,7 +38,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no
38	38	import { tmpdir } from "node:os";
39	39	import { join } from "node:path";
40	40	import { childEnv } from "../apps/process.ts";
41		import { Gate } from "../git/env.ts";
	41	import { browserGate } from "./browser_gate.ts";
42	42	import { err, ok, type Result } from "../kernel/result.ts";
43	43	import type { Entries } from "../release/webvitals.ts";
44	44
		@@ -107,7 +107,7 @@ export function findChromium(): string \| undefined {
107	107	return undefined;
108	108	}
109	109
110		const gate = new Gate(1);
	110	const gate = browserGate;
111	111
112	112	/** Take one picture. Serialised: the second caller waits for the first. */
113	113	export async function takeLook(
		@@ -15,6 +15,7 @@
15	15	* environment does — a preview's go when its pull closes.
16	16	*/
17	17
	18	import { crewHasTheCore, crewRunActive } from "../spawn/fit.ts";
18	19	import { sealedPreview } from "../core/sealed.ts";
19	20	import {
20	21	closeSync,
		@@ -88,6 +89,11 @@ export interface LooksOptions {
88	89	* while two agent runs are at their peak rather than being the straw.
89	90	*/
90	91	readonly pressure?: (() => number \| undefined) \| undefined;
	92	/**
	93	* Whether the crew has the core: on a forge of one CPU a look waits while
	94	* a run is active (`spawn/fit.ts`). Read from the forge's size unless a test says otherwise.
	95	*/
	96	readonly crewBusy?: (() => boolean) \| undefined;
91	97	/** The clock a failed look's backoff waits on: the real one unless a test says otherwise. */
92	98	readonly clock?: BackoffClock \| undefined;
93	99	}
		@@ -140,8 +146,13 @@ export interface LooksHandle {
140	146	stop(): Promise<void>;
141	147	}
142	148
143		/** A look that has waited longer than this is stale, and is dropped. */
	149	/**
	150	* A look that has waited longer than this is stale, and is dropped. A look
	151	* held for the crew keeps its place: its age is counted from the run's end.
	152	*/
144	153	const MAX_WAIT_MS = 10 * 60_000;
	154	/** How long a look waits before it asks again whether the crew is still at work. */
	155	const CREW_WAIT_MS = 5_000;
145	156	/** How often production is checked for a picture it lacks. */
146	157	const SWEEP_MS = 10 * 60_000;
147	158	/** The first check, once a booted forge's apps have had a moment. */
		@@ -349,6 +360,13 @@ export function startLooks(app: App, options: LooksOptions): LooksHandle {
349	360	for (;;) {
350	361	const next = queue.shift();
351	362	if (next === undefined \|\| stopped) return;
	363	// On one core the crew run has it: the look waits, and its age restarts when the run ends.
	364	if ((options.crewBusy ?? (() => crewHasTheCore(() => crewRunActive(app.db))))()) {
	365	if (stopped) return;
	366	queue.unshift({ ...next, at: Date.now() });
	367	await Bun.sleep(CREW_WAIT_MS);
	368	continue;
	369	}
352	370	if (Date.now() - next.at > MAX_WAIT_MS) {
353	371	log.warn("look dropped: waited too long", {
354	372	app: next.appId,