4a1c76b · b10a99e · 4a922fd · 9623f9a · a702933
@@ -0,0 +1,8 @@
# Interrupted out
When the host cuts a crew run off five times (its lease lapses), each time the task is queued again after a wait of 15 s, doubling to 4 min; the count is the run's `interruptions` column. The sixth time the run is marked failed with "the host stopped this run five times; not tried again". A graceful stop for a restart neither counts nor waits. Runs failed under the older "stopped twice by its forge restarting" text still show. Home names each such run (last 24 hours, up to five, repositories you may write) in one line with **Try again**; so does the terminal.
- **Web**: Home, under the held line. Try again posts (same origin, signed in) to `/:owner/:name/runs/:id/requeue`, which queues the run's task again and returns to Home. Writers with the crew ability only; others are refused.
- **Terminal**: the palette's "Stopped runs". `j`/`k` move, `Enter` or `r` tries again. The screen reads the database on every draw, so a retry on either door shows on the other.
- **Gone when**: a newer agent run of the same issue exists, the run is over a day old, or you cannot write the repository.
- Both doors read `interruptedOutRuns` (`src/runs/interrupted_out.ts`) and write through `requeueRepoRun`. The forge may refuse a retry (one already waiting, the day's ceiling); it says so.
@@ -1,8 +0,0 @@
# Stopped twice
When the forge restarts under a crew run twice, the run is marked failed with "stopped twice by its forge restarting; not tried again". Home names each such run (last 24 hours, up to five, repositories you may write) in one line with **Try again**; so does the terminal.
- **Web**: Home, under the held line. Try again posts (same origin, signed in) to `/:owner/:name/runs/:id/requeue`, which queues the run's task again and returns to Home. Writers with the crew ability only; others are refused.
- **Terminal**: the palette's "Stopped runs". `j`/`k` move, `Enter` or `r` tries again. The screen reads the database on every draw, so a retry on either door shows on the other.
- **Gone when**: a newer agent run of the same issue exists, the run is over a day old, or you cannot write the repository.
- Both doors read `stoppedTwiceRuns` (`src/runs/stopped_twice.ts`) and write through `requeueRepoRun`. The forge may refuse a retry (one already waiting, the day's ceiling); it says so.
@@ -0,0 +1,1 @@
bfdb3d075979d9c2b9612d7bc755bb9b1496235488462eea76aa6aa4666e1bbb
@@ -0,0 +1,7 @@
// How many times the host cut a run's task off and it was queued again (#11693): a count kept as
// data, so no one parses a summary to learn it. Each queued-again run carries its parent's count + 1.
// New column with a default. Additive. No backfill: a run in flight at deploy time starts from 0 and
// so gets at most one more try than the limit, which is harmless.
export default `
  ALTER TABLE run ADD COLUMN interruptions INTEGER NOT NULL DEFAULT 0;
  `;
@@ -78,6 +78,8 @@
export interface Run {
  readonly notBefore: string | null;
  /** How many times this task has been paused and queued again. */
  readonly attempt: number;
  /** How many times the host cut this task off and it was queued again (#11693). */
  readonly interruptions: number;
  /** When the process working this run last said it still was. */
  readonly heartbeatAt: string | null;
  /** For an agent run: how many turns the model took. */
@@ -125,6 +127,7 @@
interface RunRow {
  cost_usd: number;
  not_before: string | null;
  attempt: number;
  interruptions: number;
  heartbeat_at: string | null;
  turns: number;
  context_tokens: number;
@@ -159,6 +162,7 @@
const toRun = (r: RunRow): Run => ({
  costUsd: r.cost_usd,
  notBefore: r.not_before,
  attempt: r.attempt,
  interruptions: r.interruptions,
  heartbeatAt: r.heartbeat_at,
  turns: r.turns,
  contextTokens: r.context_tokens,
@@ -190,6 +194,8 @@
export interface CreateRunInput {
  /** For a run queued again after a pause: when it may start, and the count. */
  notBefore?: string | undefined;
  attempt?: number | undefined;
  /** How many times the host interrupted this task before (#11693). */
  interruptions?: number | undefined;
  /** What the card says while it waits. */
  summary?: string | undefined;
  /** The variant group the run belongs to, when it is one of several tries. */
@@ -200,8 +206,8 @@
export function createRun(db: Database, input: CreateRunInput): Run {
  const runId = id();
  db.query(
    `INSERT INTO run (id, repo_id, kind, issue_id, env, commit_oid, branch, actor_id, reason,
                      not_before, attempt, summary, variant_group)
     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)`,
                      not_before, attempt, summary, variant_group, interruptions)
     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)`,
  ).run(
    runId,
    input.repoId,
@@ -216,6 +222,7 @@
export function createRun(db: Database, input: CreateRunInput): Run {
    input.attempt ?? 0,
    input.summary ?? "",
    input.variantGroup ?? null,
    input.interruptions ?? 0,
  );
  return getRun(db, runId)!;
}
@@ -360,7 +360,7 @@
export async function startCrew(
      for (const runId of inFlight.keys()) abortAgentRun(runId);
      await Promise.all(inFlight.values());
      for (const run of interrupted)
        requeue(app, run, "stopped for a restart; queued again");
        requeue(app, run, "stopped for a restart; queued again", false);
      handles.delete(app);
    },
  };
@@ -458,7 +458,7 @@
export function queueAgentRun(
  agent: Agent,
  reason: RunReason,
  actorId?: string,
  again?: { notBefore: string; attempt: number; summary: string },
  again?: { notBefore: string; attempt: number; summary: string; interruptions?: number },
  variantGroup?: string,
  retry = false,
  /** The run is this agent's own on the issue: another agent's run does not stand in its way (team reviews). */
@@ -1385,14 +1385,27 @@
function forgetWorkdir(app: App, repo: Repo, pull: Pull): void {
  });
}
/** What a run says when a restart cut it off a second time in a row. */
export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried again";
/** What runs said before the count was kept (#11583): still read as given up on. */
export const STOPPED_TWICE_LEGACY = "stopped twice by its forge restarting; not tried again";
/** What a run says when the host cut it off every time it was tried. */
export const INTERRUPTED_OUT = "the host stopped this run five times; not tried again";
/** How many interruptions a task takes before it is given up (#11693). */
export const MAX_INTERRUPTIONS = 5;
const BACKOFF_FIRST_MS = 15_000;
const BACKOFF_MAX_MS = 4 * 60_000;
/** The wait before the nth queued-again try (n from 1): 15 s, doubling, at most 4 min. */
export function interruptionBackoffMs(n: number): number {
  return Math.min(BACKOFF_MAX_MS, BACKOFF_FIRST_MS * 2 ** Math.max(0, n - 1));
}
/**
 * A run a dead forge left `running` was cut off with it: its lease has
 * lapsed. Each ends here as cancelled, and the same task goes back on the
 * queue under the same reason — the run that starts reads the whole
 * conversation, so nothing a person said is lost. A run whose lease is
 * queue after a backoff — the run that starts reads the whole
 * conversation, so nothing a person said is lost. The interruptions are
 * counted on the run (`interruptions`); after `MAX_INTERRUPTIONS` the task
 * fails with `INTERRUPTED_OUT` and Try again. A run whose lease is
 * fresh is being worked by a forge still alive beside this one — the old
 * process of a release, finishing — and is left to it: that forge queues
 * the task again itself when it stops. Runs at boot and on every tick: a
@@ -1401,19 +1414,22 @@
export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried a
export function requeueInterrupted(app: App): void {
  for (const run of listActive(app.db, "agent")) {
    if (run.state !== "running" || !leaseLapsed(run, STALE_AFTER_MS)) continue;
    // Queued by an interruption and cut off again: a forge stopped every
    // 90 s would loop it until its credit is gone. Fail it, saying why.
    if ((run.summary ?? "").startsWith(REQUEUED_SUMMARY)) {
      finishRun(app.db, run.id, { state: "failed", summary: STOPPED_TWICE });
      log.info("run failed, stopped twice", { run: run.id });
      continue;
    }
    requeue(app, run, REQUEUED_SUMMARY);
    requeue(app, run, REQUEUED_SUMMARY, true);
  }
}
/** End a run that cannot go on, and put its task back on the queue. */
function requeue(app: App, run: Run, why: string): void {
/**
 * End a run that cannot go on, and put its task back on the queue. Only a
 * run the host cut off (`interrupted`) spends the budget and waits; a
 * graceful stop for a restart carries the count over and starts at once.
 */
export function requeue(app: App, run: Run, why: string, interrupted: boolean, now = Date.now()): void {
  const interruptions = run.interruptions + (interrupted ? 1 : 0);
  if (interrupted && interruptions > MAX_INTERRUPTIONS) {
    finishRun(app.db, run.id, { state: "failed", summary: INTERRUPTED_OUT });
    log.info("run given up after interruptions", { run: run.id, interruptions: run.interruptions });
    return;
  }
  cancelRun(app.db, run.id, why);
  const task = taskOf(app, run);
  if (task === undefined) return;
@@ -1427,8 +1443,9 @@
function requeue(app: App, run: Run, why: string): void {
    task.reason,
    undefined,
    {
      notBefore: new Date().toISOString(),
      notBefore: new Date(now + (interrupted ? interruptionBackoffMs(interruptions) : 0)).toISOString(),
      attempt: run.attempt + 1,
      interruptions,
      summary: `${why}; carries on where it stopped`,
    },
  );
@@ -1,17 +1,17 @@
/**
 * Runs the forge's restarts stopped twice (#11583), as plain data: the last
 * Runs the forge's host interruptions gave up on (#11583, #11693), as plain data: the last
 * 24 hours, newest first, one per issue, none once a newer run of that issue
 * exists (Try again made one). Home's line and the terminal screen share it.
 */
import type { Database } from "bun:sqlite";
import type { Repo } from "../core/repos.ts";
import { STOPPED_TWICE } from "../crew/dispatch.ts";
import { INTERRUPTED_OUT, STOPPED_TWICE_LEGACY } from "../crew/dispatch.ts";
const WINDOW_MS = 24 * 60 * 60 * 1000;
export const MAX_STOPPED = 5;
export interface StoppedTwice {
export interface InterruptedOut {
  runId: string;
  repo: Repo;
  number: number;
@@ -25,23 +25,24 @@
interface Row {
  repo_id: string;
  number: number;
  title: string;
  summary: string;
}
/** One bounded query over the repositories given (the ones this person may write). */
export function stoppedTwiceRuns(db: Database, writable: readonly Repo[], now = Date.now()): StoppedTwice[] {
export function interruptedOutRuns(db: Database, writable: readonly Repo[], now = Date.now()): InterruptedOut[] {
  const byId = new Map<string, Repo>(writable.map((r) => [r.id, r]));
  if (byId.size === 0) return [];
  const since = new Date(now - WINDOW_MS).toISOString();
  const marks = [...byId.keys()].map(() => "?").join(",");
  return db
    .query<Row, (string | number)[]>(
      `SELECT r.id, r.repo_id, i.number, i.title
      `SELECT r.id, r.repo_id, i.number, i.title, r.summary
         FROM run r JOIN issue i ON i.id = r.issue_id
        WHERE r.kind = 'agent' AND r.state = 'failed' AND r.summary = ?
        WHERE r.kind = 'agent' AND r.state = 'failed' AND r.summary IN (?, ?)
          AND r.finished_at >= ? AND r.repo_id IN (${marks})
          AND NOT EXISTS (SELECT 1 FROM run n WHERE n.issue_id = r.issue_id AND n.kind = 'agent' AND n.id > r.id)
        ORDER BY r.finished_at DESC LIMIT ${MAX_STOPPED}`,
    )
    .all(STOPPED_TWICE, since, ...byId.keys())
    .map((row) => ({ runId: row.id, repo: byId.get(row.repo_id)!, number: row.number, title: row.title, reason: STOPPED_TWICE }));
    .all(INTERRUPTED_OUT, STOPPED_TWICE_LEGACY, since, ...byId.keys())
    .map((row) => ({ runId: row.id, repo: byId.get(row.repo_id)!, number: row.number, title: row.title, reason: row.summary }));
}
@@ -18,7 +18,7 @@
import { sourceFor } from "../view/source.ts";
import { fleetCommand } from "./screens/fleet.ts";
import { dataCommand } from "./screens/data.ts";
import { packsCommand } from "./screens/packs.ts";
import { stoppedTwiceCommand } from "./screens/stopped_twice.ts";
import { interruptedOutCommand } from "./screens/interrupted_out.ts";
import { getForgeCommand } from "./screens/getforge.ts";
import { startForgeCommand } from "./screens/startforge.ts";
import { pricingCommand } from "./screens/pricing.ts";
@@ -250,7 +250,7 @@
export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
        run: () => openOnWeb(session, ctx, "/explore.atom", "The explore feed"),
      },
      packsCommand(session, ctx),
      stoppedTwiceCommand(session, ctx),
      interruptedOutCommand(session, ctx),
      getForgeCommand(session, ctx),
      ...startForgeCommand(session, ctx),
      pricingCommand(session, ctx),
@@ -1,6 +1,6 @@
/**
 * Runs the forge's restarts stopped twice, in the terminal (#11583): the same
 * lines Home prints on the web, from `stoppedTwiceRuns`, read on every draw so
 * Runs the host stopped until it was given up on, in the terminal (#11583): the same
 * lines Home prints on the web, from `interruptedOutRuns`, read on every draw so
 * a Try again made on the page shows here at once. Enter (or r) queues the run
 * again through `requeueRepoRun`, the web route's own door, which announces the
 * new run so web viewers see it live. Gated like the page: only repositories
@@ -10,16 +10,16 @@
import { requeueRepoRun } from "../../actions/runs.ts";
import { listReposWritableTo } from "../../core/repos.ts";
import { words } from "../../fleet/verbs.ts";
import { stoppedTwiceRuns } from "../../runs/stopped_twice.ts";
import { interruptedOutRuns } from "../../runs/interrupted_out.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { truncate } from "../term/width.ts";
export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
export function interruptedOutScreen(session: Session, ctx: Ctx): Screen {
  let at = 0;
  const rows = () => stoppedTwiceRuns(session.app.db, listReposWritableTo(session.app.db, session.user, { withoutArchived: true, limit: 200 }));
  const rows = () => interruptedOutRuns(session.app.db, listReposWritableTo(session.app.db, session.user, { withoutArchived: true, limit: 200 }));
  return {
    title: "Stopped runs",
    keys: [binding(["j", "k"], "move"), binding(["enter", "r"], "try again"), binding(["esc"], "back")],
@@ -27,7 +27,7 @@
export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
      const c = ctx.theme.c;
      const list = rows();
      at = Math.min(at, Math.max(0, list.length - 1));
      if (list.length === 0) s.text(area.x + 1, area.y, "No run was stopped twice by restarts in the last day.", style(c.mutedFg, c.bg), area.x + area.w);
      if (list.length === 0) s.text(area.x + 1, area.y, "No run was stopped by the host in the last day.", style(c.mutedFg, c.bg), area.x + area.w);
      list.forEach((r, i) => {
        if (i >= area.h) return;
        const text = `${i === at ? "> " : "  "}${r.repo.name}#${r.number} ${r.title}: ${r.reason}`;
@@ -53,9 +53,9 @@
export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
  };
}
export const stoppedTwiceCommand = (session: Session, ctx: Ctx) => ({
  id: "stopped-twice",
export const interruptedOutCommand = (session: Session, ctx: Ctx) => ({
  id: "interrupted-out",
  title: "Stopped runs",
  hint: "runs restarts stopped twice, with Try again",
  run: () => ctx.push(stoppedTwiceScreen(session, ctx)),
  hint: "runs the host stopped five times, with Try again",
  run: () => ctx.push(interruptedOutScreen(session, ctx)),
});
@@ -72,7 +72,7 @@
import { needsYouSection } from "./asks_page.ts";
import { flowCards } from "./flow_card.ts";
import { composer, feedJs, feedNote, feedRegion, registerWorkerJs } from "./feed.ts";
import { heldLine } from "./held.ts";
import { stoppedTwiceLine } from "./stopped_twice.ts";
import { interruptedOutLine } from "./interrupted_out.ts";
import { appChips, movingRegion, tellBox } from "./living.ts";
import { alarmLines, cachedAlarms } from "./alarm_lines.ts";
import { kpiRow, waitingLine } from "./kpi_row.ts";
@@ -357,7 +357,7 @@
export async function studioHome(
  const waiting = await homeRegion(ctx.app.db, ctx.request, user.id, "waiting", homeVariant(ctx.url), access, draws.waiting);
  const held = timePhase(ctx.request, "held", () => heldLine(ctx, user));
  const mine = writableAppsOf(ctx, user).repos;
  const stopped = timePhase(ctx.request, "stopped", () => stoppedTwiceLine(ctx, mine));
  const stopped = timePhase(ctx.request, "stopped", () => interruptedOutLine(ctx, mine));
  const alarms = user.isAdmin
    ? alarmLines(cachedAlarms(ctx.app.db, new Date(), forgeCeiling(ctx.app.config), ctx.app.config.dataDir))
    : null;
@@ -1,5 +1,5 @@
/**
 * Home names a run the forge's restarts stopped twice (#11583): one plain
 * Home names a run the host interrupted until it was given up on (#11583): one plain
 * line per such run in the last 24 hours — its issue, the reason in the
 * words the run carries, and Try again, which posts to the run's re-queue
 * route. One bounded query; no line when there is none, and none once a
@@ -8,16 +8,16 @@
import type { Repo } from "../../core/repos.ts";
import type { Ctx } from "../../http/context.ts";
import { stoppedTwiceRuns } from "../../runs/stopped_twice.ts";
import { interruptedOutRuns } from "../../runs/interrupted_out.ts";
import { repoBase } from "../chrome.ts";
import { html, type Html } from "../html.ts";
import { btn } from "./../ui.ts";
/** The lines for runs stopped twice that this person may try again; nothing where none is. */
export function stoppedTwiceLine(ctx: Ctx, writable: readonly Repo[], now = Date.now()): Html | null {
  const rows = stoppedTwiceRuns(ctx.app.db, writable, now);
/** The lines for runs given up after interruptions that this person may try again; nothing where none is. */
export function interruptedOutLine(ctx: Ctx, writable: readonly Repo[], now = Date.now()): Html | null {
  const rows = interruptedOutRuns(ctx.app.db, writable, now);
  if (rows.length === 0) return null;
  return html`<section class="stack stack-tight" id="stopped-twice" aria-label="Stopped runs">
  return html`<section class="stack stack-tight" id="interrupted-out" aria-label="Stopped runs">
    ${rows.map((row) => {
      const repo = row.repo;
      return html`<form class="bar small" method="post" action="${repoBase(repo)}/runs/${row.runId}/requeue">
@@ -1,4 +1,6 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
import { listActive, type RunKind } from "../src/core/runs.ts";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -6,15 +8,18 @@
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { openIssue } from "../src/actions/issues.ts";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { claimRun, createRun, getRun, listQueued } from "../src/core/runs.ts";
import { claimRun, createRun, getRun } from "../src/core/runs.ts";
import { createUser } from "../src/core/users.ts";
import { requeueInterrupted } from "../src/crew/dispatch.ts";
import { INTERRUPTED_OUT, MAX_INTERRUPTIONS, interruptionBackoffMs, requeue, requeueInterrupted } from "../src/crew/dispatch.ts";
import { daySpent } from "../src/core/account_ceiling.ts";
import { noteRunProgress } from "../src/core/runs.ts";
import { recordUsage, usageTotals } from "../src/core/usage.ts";
import { STALE_AFTER_MS } from "../src/runs/lease.ts";
import type { User } from "../src/core/users.ts";
/** Every queued run of one kind, those still waiting out a backoff included. */
const queuedIncludingWaiting = (db: Database, kind: RunKind) => listActive(db, kind).filter((r) => r.state === "queued");
describe("a run a restart cuts off", () => {
  let dataDir: string;
  let app: App;
@@ -23,7 +28,7 @@
describe("a run a restart cuts off", () => {
  let n = 0;
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-stopped-twice-"));
    dataDir = await mkdtemp(join(tmpdir(), "forge-interrupted-out-"));
    app = createApp({ dataDir, port: 0 });
    const a = await createUser(app.db, {
      origin: "seed", trusted: true, username: "alice",
@@ -60,8 +65,7 @@
describe("a run a restart cuts off", () => {
      .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1")
      .run(id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString());
  };
  const forIssue = (issueId: string) =>
    listQueued(app.db, "agent").filter((r) => r.issueId === issueId);
  const forIssue = (issueId: string) => queuedIncludingWaiting(app.db, "agent").filter((r) => r.issueId === issueId);
  test("once: cancelled and queued again", async () => {
    const { issue, run } = await setup();
@@ -73,16 +77,44 @@
describe("a run a restart cuts off", () => {
    expect(again[0]!.summary).toStartWith("interrupted by a restart; queued again");
  });
  test("twice in a row: failed with its reason, nothing queued", async () => {
  test("each interruption is counted as data and waits 15 s doubling to 4 min", async () => {
    expect([1, 2, 3, 4, 5, 6, 9].map(interruptionBackoffMs)).toEqual([15_000, 30_000, 60_000, 120_000, 240_000, 240_000, 240_000]);
    const { issue, run } = await setup();
    let cur = run;
    for (let i = 1; i <= MAX_INTERRUPTIONS; i++) {
      lapse(cur.id);
      const lo = Date.now();
      requeueInterrupted(app);
      const hi = Date.now();
      expect(getRun(app.db, cur.id)!.state).toBe("cancelled");
      cur = forIssue(issue.id)[0]!;
      expect(cur.interruptions).toBe(i);
      const due = Date.parse(cur.notBefore!);
      expect(due).toBeGreaterThanOrEqual(lo + interruptionBackoffMs(i));
      expect(due).toBeLessThanOrEqual(hi + interruptionBackoffMs(i));
    }
  });
  test("a graceful stop leaves the count and the wait alone, even past the budget", async () => {
    const { issue, run } = await setup();
    app.db.query("UPDATE run SET interruptions = ?2 WHERE id = ?1").run(run.id, MAX_INTERRUPTIONS);
    const now = Date.parse("2026-01-01T00:00:00.000Z");
    requeue(app, getRun(app.db, run.id)!, "stopped for a restart; queued again", false, now);
    expect(getRun(app.db, run.id)!.state).toBe("cancelled");
    const again = forIssue(issue.id);
    expect(again).toHaveLength(1);
    expect(again[0]!.interruptions).toBe(MAX_INTERRUPTIONS);
    expect(again[0]!.notBefore).toBe(new Date(now).toISOString());
  });
  test("the sixth interruption fails the run with its reason, nothing queued", async () => {
    const { issue, run } = await setup();
    app.db.query("UPDATE run SET interruptions = ?2 WHERE id = ?1").run(run.id, MAX_INTERRUPTIONS);
    lapse(run.id);
    requeueInterrupted(app);
    const second = forIssue(issue.id)[0]!;
    lapse(second.id);
    requeueInterrupted(app);
    const after = getRun(app.db, second.id)!;
    const after = getRun(app.db, run.id)!;
    expect(after.state).toBe("failed");
    expect(after.summary).toBe("stopped twice by its forge restarting; not tried again");
    expect(after.summary).toBe(INTERRUPTED_OUT);
    expect(forIssue(issue.id)).toEqual([]);
  });
@@ -1,4 +1,6 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
import { listActive, type RunKind } from "../src/core/runs.ts";
import { chmodSync, existsSync, readdirSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
@@ -13,7 +15,6 @@
import {
  claimRun,
  createRun,
  getRun,
  listQueued,
  listRuns,
} from "../src/core/runs.ts";
import { listPulls } from "../src/core/pulls.ts";
@@ -25,11 +26,16 @@
import { createUser, type User } from "../src/core/users.ts";
import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
import { serve, type Servers } from "../src/http/server.ts";
/** Every queued run of one kind, those still waiting out a backoff included. */
const queuedIncludingWaiting = (db: Database, kind: RunKind) => listActive(db, kind).filter((r) => r.state === "queued");
/** The crew's tick, injected short: every wait below is for the next tick (#11053). */
const TICK_MS = 50;
/** How often a test looks again at a run it waits on. */
const POLL_MS = 20;
const waiting = (app: App) => queuedIncludingWaiting(app.db, "agent");
describe("a run the last forge left running", () => {
  let dataDir: string;
  let app: App;
@@ -159,7 +165,7 @@
describe("a run the last forge left running", () => {
    const after = getRun(app.db, orphan.id)!;
    expect(after.state).toBe("cancelled");
    expect(after.summary).toContain("interrupted by a restart");
    const queued = listQueued(app.db, "agent").filter(
    const queued = waiting(app).filter(
      (r) => r.issueId === issue.id && r.state === "queued",
    );
    expect(queued).toHaveLength(1);
@@ -206,7 +212,7 @@
describe("a run the last forge left running", () => {
    const after = getRun(app.db, orphan.id)!;
    expect(after.state).toBe("cancelled");
    expect(after.summary).toContain("interrupted by a restart");
    const queued = listQueued(app.db, "agent").filter(
    const queued = waiting(app).filter(
      (r) => r.issueId === issue.id && r.state === "queued",
    );
    expect(queued).toHaveLength(1);
@@ -276,7 +282,7 @@
sleep 120
    const ended = getRun(app.db, queuedRun.id)!;
    expect(ended.state === "cancelled" || ended.state === "failed").toBe(true);
    const again = listQueued(app.db, "agent").filter(
    const again = waiting(app).filter(
      (r) => r.issueId === issue.id && r.state === "queued",
    );
    expect(again).toHaveLength(1);
@@ -339,7 +345,7 @@
sleep 120
    const ended = getRun(app.db, queuedRun.id)!;
    expect(ended.state === "cancelled" || ended.state === "failed").toBe(true);
    const again = listQueued(app.db, "agent").filter((r) => r.issueId === issue.id && r.state === "queued");
    const again = waiting(app).filter((r) => r.issueId === issue.id && r.state === "queued");
    expect(again).toHaveLength(1);
    expect(again[0]!.summary).toContain("carries on");
  }, 40_000);
@@ -409,7 +415,7 @@
echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while
    expect(ended.state).toBe("ok");
    expect(ended.summary).toMatch(/^opened pull #\d+/);
    expect(
      listQueued(app.db, "agent").filter((r) => r.issueId === issue.id),
      waiting(app).filter((r) => r.issueId === issue.id),
    ).toHaveLength(0);
  }, 40_000);
@@ -650,7 +656,7 @@
describe("an issue handed to someone else", () => {
    const talk = addComment(app, alice, repo, issue, "I have this one");
    expect(talk.ok).toBe(true);
    expect(
      listQueued(app.db, "agent").some((r) => r.issueId === issue.id),
      waiting(app).some((r) => r.issueId === issue.id),
    ).toBe(false);
  });
@@ -660,7 +666,7 @@
describe("an issue handed to someone else", () => {
    claimRun(app.db, run.id);
    expect(assignIssue(app, alice, repo, issue, other).ok).toBe(true);
    expect(getRun(app.db, run.id)?.state).toBe("cancelled");
    const queued = listQueued(app.db, "agent").filter(
    const queued = waiting(app).filter(
      (r) => r.issueId === issue.id,
    );
    expect(queued).toHaveLength(1);
@@ -1,4 +1,4 @@
/** Home names a run stopped twice by restarts, with Try again (#11583). */
/** Home names a run given up after interruptions, with Try again (#11583). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
@@ -6,10 +6,10 @@
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { newApp } from "../src/actions/apps.ts";
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { stoppedTwiceRuns } from "../src/runs/stopped_twice.ts";
import { stoppedTwiceScreen } from "../src/tui/screens/stopped_twice.ts";
import { interruptedOutRuns } from "../src/runs/interrupted_out.ts";
import { interruptedOutScreen } from "../src/tui/screens/interrupted_out.ts";
import { getRepo } from "../src/core/repos.ts";
import { STOPPED_TWICE } from "../src/crew/dispatch.ts";
import { INTERRUPTED_OUT, STOPPED_TWICE_LEGACY } from "../src/crew/dispatch.ts";
import { createIssue } from "../src/core/issues.ts";
import { createSession } from "../src/core/sessions.ts";
import { createUser, type User } from "../src/core/users.ts";
@@ -37,7 +37,7 @@
function failedRun(title: string, summary: string, finishedAt: string) {
}
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-stopped-twice-"));
  dataDir = await mkdtemp(join(tmpdir(), "forge-interrupted-out-"));
  app = createApp({ dataDir, port: 0 });
  servers = await serve(app);
  const r = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
@@ -61,18 +61,18 @@
afterAll(async () => {
});
test("no line when no run was stopped twice", async () => {
  expect(await home()).not.toContain("stopped-twice");
  expect(await home()).not.toContain("interrupted-out");
});
test("a run stopped twice lately shows one line with its reason and Try again; old, other, or retried runs show none", async () => {
  failedRun("Old one", STOPPED_TWICE, new Date(Date.now() - 25 * 3600_000).toISOString());
  failedRun("Old one", INTERRUPTED_OUT, new Date(Date.now() - 25 * 3600_000).toISOString());
  failedRun("Other failure", "tests failed", new Date().toISOString());
  const again = failedRun("Already retried", STOPPED_TWICE, new Date(Date.now() - 1000).toISOString());
  const again = failedRun("Already retried", INTERRUPTED_OUT, new Date(Date.now() - 1000).toISOString());
  app.db.query("INSERT INTO run (id, repo_id, kind, issue_id, commit_oid, state) VALUES ('zzzzz-newer', ?, 'agent', ?, 'c', 'queued')").run(repoId, again.issue.id);
  const hit = failedRun("Fix <b>the</b> thing", STOPPED_TWICE, new Date().toISOString());
  const hit = failedRun("Fix <b>the</b> thing", INTERRUPTED_OUT, new Date().toISOString());
  const page = await home();
  expect(page).toContain('id="stopped-twice"');
  expect(page).toContain(STOPPED_TWICE);
  expect(page).toContain('id="interrupted-out"');
  expect(page).toContain(INTERRUPTED_OUT);
  expect(page).toContain(`action="/alice/shop/runs/${hit.id}/requeue"`);
  expect(page).toContain("Try again");
  expect(page).toContain("Fix &lt;b&gt;the&lt;/b&gt; thing");
@@ -80,15 +80,23 @@
test("a run stopped twice lately shows one line with its reason and Try again; o
  expect(page.match(/\/requeue"/g)?.length).toBe(1);
});
test("a run failed under the old \"stopped twice\" text still shows", async () => {
  const hit = failedRun("Old words", STOPPED_TWICE_LEGACY, new Date().toISOString());
  const repo = getRepo(app.db, "alice", "shop")!;
  const row = interruptedOutRuns(app.db, [repo]).find((r) => r.runId === hit.id);
  expect(row?.reason).toBe(STOPPED_TWICE_LEGACY);
  expect(await home()).toContain(`/runs/${hit.id}/requeue`);
});
test("a stranger sees no line", async () => {
  const r = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
  if (!r.ok) throw new Error(r.error);
  const bob = `forge_session=${createSession(app.db, r.value.id).token}`;
  expect(await (await fetch(`${servers.url}/`, { headers: { cookie: bob } })).text()).not.toContain("stopped-twice");
  expect(await (await fetch(`${servers.url}/`, { headers: { cookie: bob } })).text()).not.toContain("interrupted-out");
});
test("the Try again route refuses a cross-origin post", async () => {
  const run = app.db.query<{ id: string }, []>("SELECT id FROM run WHERE summary LIKE 'stopped twice%' ORDER BY id DESC LIMIT 1").get()!;
  const run = app.db.query<{ id: string }, []>("SELECT id FROM run WHERE summary LIKE 'the host stopped%' ORDER BY id DESC LIMIT 1").get()!;
  const res = await fetch(`${servers.url}/alice/shop/runs/${run.id}/requeue`, {
    method: "POST",
    headers: { cookie, origin: "https://evil.example", "content-type": "application/x-www-form-urlencoded" },
@@ -108,7 +116,7 @@
const post = (path: string, who: string) =>
const runsOf = (issueId: string) => app.db.query<{ n: number }, [string]>("SELECT count(*) AS n FROM run WHERE issue_id = ? AND kind = 'agent'").get(issueId)!.n;
test("a signed-in non-writer's same-origin post queues nothing", async () => {
  const hit = failedRun("Not for bob", STOPPED_TWICE, new Date().toISOString());
  const hit = failedRun("Not for bob", INTERRUPTED_OUT, new Date().toISOString());
  const r = await createUser(app.db, { origin: "seed", trusted: true, username: "carol", email: "[email protected]", password: "a-good-passphrase" });
  if (!r.ok) throw new Error(r.error);
  const carol = `forge_session=${createSession(app.db, r.value.id).token}`;
@@ -119,7 +127,7 @@
test("a signed-in non-writer's same-origin post queues nothing", async () => {
});
test("a writer's same-origin post queues a new run and the line goes away", async () => {
  const hit = failedRun("Retry me", STOPPED_TWICE, new Date().toISOString());
  const hit = failedRun("Retry me", INTERRUPTED_OUT, new Date().toISOString());
  expect(await home()).toContain(`/runs/${hit.id}/requeue`);
  const res = await post(`/alice/shop/runs/${hit.id}/requeue`, cookie);
  expect(res.status).toBe(303);
@@ -128,12 +136,12 @@
test("a writer's same-origin post queues a new run and the line goes away", asyn
});
test("the terminal screen lists the same runs and Enter tries one again", async () => {
  const hit = failedRun("Retry from terminal", STOPPED_TWICE, new Date().toISOString());
  const hit = failedRun("Retry from terminal", INTERRUPTED_OUT, new Date().toISOString());
  const repo = getRepo(app.db, "alice", "shop")!;
  expect(stoppedTwiceRuns(app.db, [repo]).some((r) => r.runId === hit.id)).toBe(true);
  expect(interruptedOutRuns(app.db, [repo]).some((r) => r.runId === hit.id)).toBe(true);
  const toasts: string[] = [];
  const ctx = { theme: { c: {} }, pop() {}, push() {}, invalidate() {}, toast: (t: string) => toasts.push(t) };
  const screen = stoppedTwiceScreen({ app, user: alice } as never, ctx as never);
  const screen = interruptedOutScreen({ app, user: alice } as never, ctx as never);
  // Newest first: the run just made is the first row.
  expect(screen.key!({ key: "enter", ctrl: false, alt: false } as never)).toBe(true);
  expect(toasts[0]).toContain("again");
@@ -51,7 +51,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["mirror-packs", "Mirror the packs' upstreams", "bring each pack's upstream projects in as read-only repositories"],
    ["pack-repos", "Pack repositories", "which repositories /packs lists"],
    ["stopped-twice", "Stopped runs", "runs restarts stopped twice, with Try again"],
    ["interrupted-out", "Stopped runs", "runs the host stopped five times, with Try again"],
    ["get-a-forge", "Get your own forge", "the plans, where to get a hosted forge, how to run one yourself"],
    ["pricing", "Get a forge (plans)", "pick a plan, name it, pay at the checkout address"],
    ["data", "Data", "the SQLite files you may read as a tree; / filters, : asks in SQL, y yanks a row"],
@@ -208,7 +208,7 @@
test("a site-admin agent is offered the roster without the person-only entries",
    ["explore", "Explore", "on the web"],
    ["explore-feed", "Explore feed", "Atom address"],
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["stopped-twice", "Stopped runs", "runs restarts stopped twice, with Try again"],
    ["interrupted-out", "Stopped runs", "runs the host stopped five times, with Try again"],
    ["get-a-forge", "Get your own forge", "the plans, where to get a hosted forge, how to run one yourself"],
    ["pricing", "Get a forge (plans)", "pick a plan, name it, pay at the checkout address"],
    ["data", "Data", "the SQLite files you may read as a tree; / filters, : asks in SQL, y yanks a row"],
@@ -296,7 +296,7 @@
test("a member person is offered the signed-in roster without the admin entries"
    ["explore", "Explore", "on the web"],
    ["explore-feed", "Explore feed", "Atom address"],
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["stopped-twice", "Stopped runs", "runs restarts stopped twice, with Try again"],
    ["interrupted-out", "Stopped runs", "runs the host stopped five times, with Try again"],
    ["get-a-forge", "Get your own forge", "the plans, where to get a hosted forge, how to run one yourself"],
    ["pricing", "Get a forge (plans)", "pick a plan, name it, pay at the checkout address"],
    ["fleet", "Fleet rows", "the rows a fleet query line selects"],
@@ -379,7 +379,7 @@
test("signed out, the palette offers sign-in and the public entries", () => {
    ["explore", "Explore", "on the web"],
    ["explore-feed", "Explore feed", "Atom address"],
    ["packs", "Forge packs", "premade forges to deploy, and how to help build their apps"],
    ["stopped-twice", "Stopped runs", "runs restarts stopped twice, with Try again"],
    ["interrupted-out", "Stopped runs", "runs the host stopped five times, with Try again"],
    ["get-a-forge", "Get your own forge", "the plans, where to get a hosted forge, how to run one yourself"],
    ["start-forge", "Start my forge", "a free forge of your own; you choose a username and password next"],
    ["pricing", "Get a forge (plans)", "pick a plan, name it, pay at the checkout address"],
		@@ -0,0 +1,8 @@
	1	# Interrupted out
	2
	3	When the host cuts a crew run off five times (its lease lapses), each time the task is queued again after a wait of 15 s, doubling to 4 min; the count is the run's `interruptions` column. The sixth time the run is marked failed with "the host stopped this run five times; not tried again". A graceful stop for a restart neither counts nor waits. Runs failed under the older "stopped twice by its forge restarting" text still show. Home names each such run (last 24 hours, up to five, repositories you may write) in one line with Try again; so does the terminal.
	4
	5	- Web: Home, under the held line. Try again posts (same origin, signed in) to `/:owner/:name/runs/:id/requeue`, which queues the run's task again and returns to Home. Writers with the crew ability only; others are refused.
	6	- Terminal: the palette's "Stopped runs". `j`/`k` move, `Enter` or `r` tries again. The screen reads the database on every draw, so a retry on either door shows on the other.
	7	- Gone when: a newer agent run of the same issue exists, the run is over a day old, or you cannot write the repository.
	8	- Both doors read `interruptedOutRuns` (`src/runs/interrupted_out.ts`) and write through `requeueRepoRun`. The forge may refuse a retry (one already waiting, the day's ceiling); it says so.
		@@ -1,8 +0,0 @@
1		# Stopped twice
2
3		When the forge restarts under a crew run twice, the run is marked failed with "stopped twice by its forge restarting; not tried again". Home names each such run (last 24 hours, up to five, repositories you may write) in one line with Try again; so does the terminal.
4
5		- Web: Home, under the held line. Try again posts (same origin, signed in) to `/:owner/:name/runs/:id/requeue`, which queues the run's task again and returns to Home. Writers with the crew ability only; others are refused.
6		- Terminal: the palette's "Stopped runs". `j`/`k` move, `Enter` or `r` tries again. The screen reads the database on every draw, so a retry on either door shows on the other.
7		- Gone when: a newer agent run of the same issue exists, the run is over a day old, or you cannot write the repository.
8		- Both doors read `stoppedTwiceRuns` (`src/runs/stopped_twice.ts`) and write through `requeueRepoRun`. The forge may refuse a retry (one already waiting, the day's ceiling); it says so.
		@@ -0,0 +1,1 @@
	1	bfdb3d075979d9c2b9612d7bc755bb9b1496235488462eea76aa6aa4666e1bbb
		@@ -0,0 +1,7 @@
	1	// How many times the host cut a run's task off and it was queued again (#11693): a count kept as
	2	// data, so no one parses a summary to learn it. Each queued-again run carries its parent's count + 1.
	3	// New column with a default. Additive. No backfill: a run in flight at deploy time starts from 0 and
	4	// so gets at most one more try than the limit, which is harmless.
	5	export default `
	6	ALTER TABLE run ADD COLUMN interruptions INTEGER NOT NULL DEFAULT 0;
	7	`;
		@@ -78,6 +78,8 @@ export interface Run {
78	78	readonly notBefore: string \| null;
79	79	/** How many times this task has been paused and queued again. */
80	80	readonly attempt: number;
	81	/** How many times the host cut this task off and it was queued again (#11693). */
	82	readonly interruptions: number;
81	83	/** When the process working this run last said it still was. */
82	84	readonly heartbeatAt: string \| null;
83	85	/** For an agent run: how many turns the model took. */
		@@ -125,6 +127,7 @@ interface RunRow {
125	127	cost_usd: number;
126	128	not_before: string \| null;
127	129	attempt: number;
	130	interruptions: number;
128	131	heartbeat_at: string \| null;
129	132	turns: number;
130	133	context_tokens: number;
		@@ -159,6 +162,7 @@ const toRun = (r: RunRow): Run => ({
159	162	costUsd: r.cost_usd,
160	163	notBefore: r.not_before,
161	164	attempt: r.attempt,
	165	interruptions: r.interruptions,
162	166	heartbeatAt: r.heartbeat_at,
163	167	turns: r.turns,
164	168	contextTokens: r.context_tokens,
		@@ -190,6 +194,8 @@ export interface CreateRunInput {
190	194	/** For a run queued again after a pause: when it may start, and the count. */
191	195	notBefore?: string \| undefined;
192	196	attempt?: number \| undefined;
	197	/** How many times the host interrupted this task before (#11693). */
	198	interruptions?: number \| undefined;
193	199	/** What the card says while it waits. */
194	200	summary?: string \| undefined;
195	201	/** The variant group the run belongs to, when it is one of several tries. */
		@@ -200,8 +206,8 @@ export function createRun(db: Database, input: CreateRunInput): Run {
200	206	const runId = id();
201	207	db.query(
202	208	`INSERT INTO run (id, repo_id, kind, issue_id, env, commit_oid, branch, actor_id, reason,
203		not_before, attempt, summary, variant_group)
204		VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)`,
	209	not_before, attempt, summary, variant_group, interruptions)
	210	VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)`,
205	211	).run(
206	212	runId,
207	213	input.repoId,
		@@ -216,6 +222,7 @@ export function createRun(db: Database, input: CreateRunInput): Run {
216	222	input.attempt ?? 0,
217	223	input.summary ?? "",
218	224	input.variantGroup ?? null,
	225	input.interruptions ?? 0,
219	226	);
220	227	return getRun(db, runId)!;
221	228	}
		@@ -360,7 +360,7 @@ export async function startCrew(
360	360	for (const runId of inFlight.keys()) abortAgentRun(runId);
361	361	await Promise.all(inFlight.values());
362	362	for (const run of interrupted)
363		requeue(app, run, "stopped for a restart; queued again");
	363	requeue(app, run, "stopped for a restart; queued again", false);
364	364	handles.delete(app);
365	365	},
366	366	};
		@@ -458,7 +458,7 @@ export function queueAgentRun(
458	458	agent: Agent,
459	459	reason: RunReason,
460	460	actorId?: string,
461		again?: { notBefore: string; attempt: number; summary: string },
	461	again?: { notBefore: string; attempt: number; summary: string; interruptions?: number },
462	462	variantGroup?: string,
463	463	retry = false,
464	464	/** The run is this agent's own on the issue: another agent's run does not stand in its way (team reviews). */
		@@ -1385,14 +1385,27 @@ function forgetWorkdir(app: App, repo: Repo, pull: Pull): void {
1385	1385	});
1386	1386	}
1387	1387
1388		/** What a run says when a restart cut it off a second time in a row. */
1389		export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried again";
	1388	/** What runs said before the count was kept (#11583): still read as given up on. */
	1389	export const STOPPED_TWICE_LEGACY = "stopped twice by its forge restarting; not tried again";
	1390	/** What a run says when the host cut it off every time it was tried. */
	1391	export const INTERRUPTED_OUT = "the host stopped this run five times; not tried again";
	1392	/** How many interruptions a task takes before it is given up (#11693). */
	1393	export const MAX_INTERRUPTIONS = 5;
	1394	const BACKOFF_FIRST_MS = 15_000;
	1395	const BACKOFF_MAX_MS = 4 * 60_000;
	1396
	1397	/** The wait before the nth queued-again try (n from 1): 15 s, doubling, at most 4 min. */
	1398	export function interruptionBackoffMs(n: number): number {
	1399	return Math.min(BACKOFF_MAX_MS, BACKOFF_FIRST_MS * 2 ** Math.max(0, n - 1));
	1400	}
1390	1401
1391	1402	/**
1392	1403	* A run a dead forge left `running` was cut off with it: its lease has
1393	1404	* lapsed. Each ends here as cancelled, and the same task goes back on the
1394		* queue under the same reason — the run that starts reads the whole
1395		* conversation, so nothing a person said is lost. A run whose lease is
	1405	* queue after a backoff — the run that starts reads the whole
	1406	* conversation, so nothing a person said is lost. The interruptions are
	1407	* counted on the run (`interruptions`); after `MAX_INTERRUPTIONS` the task
	1408	* fails with `INTERRUPTED_OUT` and Try again. A run whose lease is
1396	1409	* fresh is being worked by a forge still alive beside this one — the old
1397	1410	* process of a release, finishing — and is left to it: that forge queues
1398	1411	* the task again itself when it stops. Runs at boot and on every tick: a
		@@ -1401,19 +1414,22 @@ export const STOPPED_TWICE = "stopped twice by its forge restarting; not tried a
1401	1414	export function requeueInterrupted(app: App): void {
1402	1415	for (const run of listActive(app.db, "agent")) {
1403	1416	if (run.state !== "running" \|\| !leaseLapsed(run, STALE_AFTER_MS)) continue;
1404		// Queued by an interruption and cut off again: a forge stopped every
1405		// 90 s would loop it until its credit is gone. Fail it, saying why.
1406		if ((run.summary ?? "").startsWith(REQUEUED_SUMMARY)) {
1407		finishRun(app.db, run.id, { state: "failed", summary: STOPPED_TWICE });
1408		log.info("run failed, stopped twice", { run: run.id });
1409		continue;
1410		}
1411		requeue(app, run, REQUEUED_SUMMARY);
	1417	requeue(app, run, REQUEUED_SUMMARY, true);
1412	1418	}
1413	1419	}
1414	1420
1415		/** End a run that cannot go on, and put its task back on the queue. */
1416		function requeue(app: App, run: Run, why: string): void {
	1421	/**
	1422	* End a run that cannot go on, and put its task back on the queue. Only a
	1423	* run the host cut off (`interrupted`) spends the budget and waits; a
	1424	* graceful stop for a restart carries the count over and starts at once.
	1425	*/
	1426	export function requeue(app: App, run: Run, why: string, interrupted: boolean, now = Date.now()): void {
	1427	const interruptions = run.interruptions + (interrupted ? 1 : 0);
	1428	if (interrupted && interruptions > MAX_INTERRUPTIONS) {
	1429	finishRun(app.db, run.id, { state: "failed", summary: INTERRUPTED_OUT });
	1430	log.info("run given up after interruptions", { run: run.id, interruptions: run.interruptions });
	1431	return;
	1432	}
1417	1433	cancelRun(app.db, run.id, why);
1418	1434	const task = taskOf(app, run);
1419	1435	if (task === undefined) return;
		@@ -1427,8 +1443,9 @@ function requeue(app: App, run: Run, why: string): void {
1427	1443	task.reason,
1428	1444	undefined,
1429	1445	{
1430		notBefore: new Date().toISOString(),
	1446	notBefore: new Date(now + (interrupted ? interruptionBackoffMs(interruptions) : 0)).toISOString(),
1431	1447	attempt: run.attempt + 1,
	1448	interruptions,
1432	1449	summary: `${why}; carries on where it stopped`,
1433	1450	},
1434	1451	);
		@@ -1,17 +1,17 @@
1	1	/**
2		* Runs the forge's restarts stopped twice (#11583), as plain data: the last
	2	* Runs the forge's host interruptions gave up on (#11583, #11693), as plain data: the last
3	3	* 24 hours, newest first, one per issue, none once a newer run of that issue
4	4	* exists (Try again made one). Home's line and the terminal screen share it.
5	5	*/
6	6
7	7	import type { Database } from "bun:sqlite";
8	8	import type { Repo } from "../core/repos.ts";
9		import { STOPPED_TWICE } from "../crew/dispatch.ts";
	9	import { INTERRUPTED_OUT, STOPPED_TWICE_LEGACY } from "../crew/dispatch.ts";
10	10
11	11	const WINDOW_MS = 24 * 60 * 60 * 1000;
12	12	export const MAX_STOPPED = 5;
13	13
14		export interface StoppedTwice {
	14	export interface InterruptedOut {
15	15	runId: string;
16	16	repo: Repo;
17	17	number: number;
		@@ -25,23 +25,24 @@ interface Row {
25	25	repo_id: string;
26	26	number: number;
27	27	title: string;
	28	summary: string;
28	29	}
29	30
30	31	/** One bounded query over the repositories given (the ones this person may write). */
31		export function stoppedTwiceRuns(db: Database, writable: readonly Repo[], now = Date.now()): StoppedTwice[] {
	32	export function interruptedOutRuns(db: Database, writable: readonly Repo[], now = Date.now()): InterruptedOut[] {
32	33	const byId = new Map<string, Repo>(writable.map((r) => [r.id, r]));
33	34	if (byId.size === 0) return [];
34	35	const since = new Date(now - WINDOW_MS).toISOString();
35	36	const marks = [...byId.keys()].map(() => "?").join(",");
36	37	return db
37	38	.query<Row, (string \| number)[]>(
38		`SELECT r.id, r.repo_id, i.number, i.title
	39	`SELECT r.id, r.repo_id, i.number, i.title, r.summary
39	40	FROM run r JOIN issue i ON i.id = r.issue_id
40		WHERE r.kind = 'agent' AND r.state = 'failed' AND r.summary = ?
	41	WHERE r.kind = 'agent' AND r.state = 'failed' AND r.summary IN (?, ?)
41	42	AND r.finished_at >= ? AND r.repo_id IN (${marks})
42	43	AND NOT EXISTS (SELECT 1 FROM run n WHERE n.issue_id = r.issue_id AND n.kind = 'agent' AND n.id > r.id)
43	44	ORDER BY r.finished_at DESC LIMIT ${MAX_STOPPED}`,
44	45	)
45		.all(STOPPED_TWICE, since, ...byId.keys())
46		.map((row) => ({ runId: row.id, repo: byId.get(row.repo_id)!, number: row.number, title: row.title, reason: STOPPED_TWICE }));
	46	.all(INTERRUPTED_OUT, STOPPED_TWICE_LEGACY, since, ...byId.keys())
	47	.map((row) => ({ runId: row.id, repo: byId.get(row.repo_id)!, number: row.number, title: row.title, reason: row.summary }));
47	48	}
		@@ -18,7 +18,7 @@ import { sourceFor } from "../view/source.ts";
18	18	import { fleetCommand } from "./screens/fleet.ts";
19	19	import { dataCommand } from "./screens/data.ts";
20	20	import { packsCommand } from "./screens/packs.ts";
21		import { stoppedTwiceCommand } from "./screens/stopped_twice.ts";
	21	import { interruptedOutCommand } from "./screens/interrupted_out.ts";
22	22	import { getForgeCommand } from "./screens/getforge.ts";
23	23	import { startForgeCommand } from "./screens/startforge.ts";
24	24	import { pricingCommand } from "./screens/pricing.ts";
		@@ -250,7 +250,7 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
250	250	run: () => openOnWeb(session, ctx, "/explore.atom", "The explore feed"),
251	251	},
252	252	packsCommand(session, ctx),
253		stoppedTwiceCommand(session, ctx),
	253	interruptedOutCommand(session, ctx),
254	254	getForgeCommand(session, ctx),
255	255	...startForgeCommand(session, ctx),
256	256	pricingCommand(session, ctx),
		@@ -1,6 +1,6 @@
1	1	/**
2		* Runs the forge's restarts stopped twice, in the terminal (#11583): the same
3		* lines Home prints on the web, from `stoppedTwiceRuns`, read on every draw so
	2	* Runs the host stopped until it was given up on, in the terminal (#11583): the same
	3	* lines Home prints on the web, from `interruptedOutRuns`, read on every draw so
4	4	* a Try again made on the page shows here at once. Enter (or r) queues the run
5	5	* again through `requeueRepoRun`, the web route's own door, which announces the
6	6	* new run so web viewers see it live. Gated like the page: only repositories
		@@ -10,16 +10,16 @@
10	10	import { requeueRepoRun } from "../../actions/runs.ts";
11	11	import { listReposWritableTo } from "../../core/repos.ts";
12	12	import { words } from "../../fleet/verbs.ts";
13		import { stoppedTwiceRuns } from "../../runs/stopped_twice.ts";
	13	import { interruptedOutRuns } from "../../runs/interrupted_out.ts";
14	14	import type { Ctx, Screen } from "../app/app.ts";
15	15	import { binding } from "../app/keys.ts";
16	16	import type { Session } from "../session.ts";
17	17	import { style } from "../term/style.ts";
18	18	import { truncate } from "../term/width.ts";
19	19
20		export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
	20	export function interruptedOutScreen(session: Session, ctx: Ctx): Screen {
21	21	let at = 0;
22		const rows = () => stoppedTwiceRuns(session.app.db, listReposWritableTo(session.app.db, session.user, { withoutArchived: true, limit: 200 }));
	22	const rows = () => interruptedOutRuns(session.app.db, listReposWritableTo(session.app.db, session.user, { withoutArchived: true, limit: 200 }));
23	23	return {
24	24	title: "Stopped runs",
25	25	keys: [binding(["j", "k"], "move"), binding(["enter", "r"], "try again"), binding(["esc"], "back")],
		@@ -27,7 +27,7 @@ export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
27	27	const c = ctx.theme.c;
28	28	const list = rows();
29	29	at = Math.min(at, Math.max(0, list.length - 1));
30		if (list.length === 0) s.text(area.x + 1, area.y, "No run was stopped twice by restarts in the last day.", style(c.mutedFg, c.bg), area.x + area.w);
	30	if (list.length === 0) s.text(area.x + 1, area.y, "No run was stopped by the host in the last day.", style(c.mutedFg, c.bg), area.x + area.w);
31	31	list.forEach((r, i) => {
32	32	if (i >= area.h) return;
33	33	const text = `${i === at ? "> " : " "}${r.repo.name}#${r.number} ${r.title}: ${r.reason}`;
		@@ -53,9 +53,9 @@ export function stoppedTwiceScreen(session: Session, ctx: Ctx): Screen {
53	53	};
54	54	}
55	55
56		export const stoppedTwiceCommand = (session: Session, ctx: Ctx) => ({
57		id: "stopped-twice",
	56	export const interruptedOutCommand = (session: Session, ctx: Ctx) => ({
	57	id: "interrupted-out",
58	58	title: "Stopped runs",
59		hint: "runs restarts stopped twice, with Try again",
60		run: () => ctx.push(stoppedTwiceScreen(session, ctx)),
	59	hint: "runs the host stopped five times, with Try again",
	60	run: () => ctx.push(interruptedOutScreen(session, ctx)),
61	61	});
		@@ -72,7 +72,7 @@ import { needsYouSection } from "./asks_page.ts";
72	72	import { flowCards } from "./flow_card.ts";
73	73	import { composer, feedJs, feedNote, feedRegion, registerWorkerJs } from "./feed.ts";
74	74	import { heldLine } from "./held.ts";
75		import { stoppedTwiceLine } from "./stopped_twice.ts";
	75	import { interruptedOutLine } from "./interrupted_out.ts";
76	76	import { appChips, movingRegion, tellBox } from "./living.ts";
77	77	import { alarmLines, cachedAlarms } from "./alarm_lines.ts";
78	78	import { kpiRow, waitingLine } from "./kpi_row.ts";
		@@ -357,7 +357,7 @@ export async function studioHome(
357	357	const waiting = await homeRegion(ctx.app.db, ctx.request, user.id, "waiting", homeVariant(ctx.url), access, draws.waiting);
358	358	const held = timePhase(ctx.request, "held", () => heldLine(ctx, user));
359	359	const mine = writableAppsOf(ctx, user).repos;
360		const stopped = timePhase(ctx.request, "stopped", () => stoppedTwiceLine(ctx, mine));
	360	const stopped = timePhase(ctx.request, "stopped", () => interruptedOutLine(ctx, mine));
361	361	const alarms = user.isAdmin
362	362	? alarmLines(cachedAlarms(ctx.app.db, new Date(), forgeCeiling(ctx.app.config), ctx.app.config.dataDir))
363	363	: null;