282daab · 635819a · 1d92c1d · 339c399 · 0bfd101 · 9c89223
@@ -134,7 +134,7 @@
bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
| `lift` | — | Lift an issue out from behind the Later gate | 214 |
| `limb` | `defer`, `resume`, `close`, `raise`, `lower`, `replan`, `takeback`, `undo` | Act on a limb by `action` | 532 |
| `limits` | — | Every limit the owner can hit, value and bounds | 977 |
| `maintenance` | `status`, `prepare`, `resume` | Prepare the forge for a power-off, check the drain, or end maintenance (admin) | 256 |
| `maintenance` | `status`, `prepare`, `resume`, `pause-duty`, `resume-duty` | Prepare the forge for a power-off, check the drain, or end maintenance (admin) | 308 |
| `make_it_yours` | — | Fork an app you can see: code and history, following the original | 337 |
| `map` | — | An app's plan: ready or waiting, critical path, crew, parkedTwice; `done` adds closed; `first: N` ranks N first | 375 |
| `merge` | — | Land a pull, as its page's Merge button does; a person's only | 319 |
@@ -0,0 +1,14 @@
# Pause and resume a maintenance duty (#11205)
Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once.
All doors are for a forge admin and call one function, `switchDuty` in `src/maintain/pause.ts`, so they say the same words.
- **On the web.** `/maintenance` lists each duty once; the ones the runner runs carry their state ("on", or "paused since … ; why") and one **Pause** / **Resume** button (`POST /maintenance/duties/:name`, same-origin only, form body bounded, `paused` is `yes` or `no`).
- **In the terminal.** Ctrl-K, **Pause or resume a maintenance duty** (an admin who is a person): `j`/`k` choose, Enter or Space flips, Esc closes. It writes the same row, so a web viewer sees it on its next load.
- **Over the API.** `GET /api/v1/maintenance/duties` lists the states; `POST /api/v1/maintenance/duties/:name` with `{"paused": true|false}` needs an `admin`-scope token.
- **In the MCP.** The `maintenance` tool: `pause-duty` and `resume-duty` with `duty`.
An unknown duty name is 404 (a flash on the page); a `paused` that is not a boolean is 422.
Tests: `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates).
@@ -0,0 +1,11 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "get",
  path: "/api/v1/maintenance/duties",
  summary:
    "Each maintenance duty with whether it is paused, and why and by whom (a forge admin's token); answers { duties }",
  auth: true,
  role: "admin",
  refusals: { forbidden: "the token is not a forge admin" },
};
@@ -102,6 +102,7 @@
export { route as getApiV1Front } from "./getApiV1Front.ts";
export { route as getApiV1Health } from "./getApiV1Health.ts";
export { route as getApiV1Here } from "./getApiV1Here.ts";
export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
export { route as getApiV1MaintenanceDuties } from "./getApiV1MaintenanceDuties.ts";
export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
export { route as getApiV1Map } from "./getApiV1Map.ts";
export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
@@ -286,6 +287,7 @@
export { route as postApiV1FleetOpsIdUndo } from "./postApiV1FleetOpsIdUndo.ts";
export { route as postApiV1Forges } from "./postApiV1Forges.ts";
export { route as postApiV1ForgesNameClaimLink } from "./postApiV1ForgesNameClaimLink.ts";
export { route as postApiV1Heartbeat } from "./postApiV1Heartbeat.ts";
export { route as postApiV1MaintenanceDutiesName } from "./postApiV1MaintenanceDutiesName.ts";
export { route as postApiV1Migrate } from "./postApiV1Migrate.ts";
export { route as postApiV1Orgs } from "./postApiV1Orgs.ts";
export { route as postApiV1OrgsOrgTeams } from "./postApiV1OrgsOrgTeams.ts";
@@ -0,0 +1,11 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "post",
  path: "/api/v1/maintenance/duties/{name}",
  summary:
    "Pause or resume one maintenance duty with { paused: boolean } (a forge admin's admin-scope token); answers { says }",
  auth: true,
  role: "admin",
  refusals: { forbidden: "the token is not a forge admin", not_found: "no duty has that name", invalid: "paused is not a boolean" },
};
@@ -0,0 +1,1 @@
391de19c4b728caa065a215f40be9cf48eda5929d63d10876598d85b728932eb
@@ -0,0 +1,12 @@
// A maintenance duty's on/off switch (#11205): paused by a person or by three errors in a row,
// with why and when. `from_run` is the newest run at the last resume: only later runs count toward three.
export default `
  CREATE TABLE maintenance_duty_state (
    duty TEXT PRIMARY KEY,
    paused INTEGER NOT NULL DEFAULT 0,
    reason TEXT NOT NULL DEFAULT '',
    by TEXT NOT NULL DEFAULT '',
    since TEXT NOT NULL DEFAULT '',
    from_run INTEGER NOT NULL DEFAULT 0
  );
  `;
@@ -1,9 +1,11 @@
/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
import { switchDuty, dutyStates } from "../../maintain/pause.ts";
import { latest } from "../../maintain/runs.ts";
import type { Handler } from "../context.ts";
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { forbidden, json, requireScope, unauthorized } from "../respond.ts";
import { crossOrigin } from "../context.ts";
import { apiError, boundedBody, forbidden, json, notFound, requireScope, unauthorized } from "../respond.ts";
import type { Router } from "../router.ts";
export function maintenanceRunsRoutes(router: Router<Handler>): void {
@@ -17,4 +19,23 @@
export function maintenanceRunsRoutes(router: Router<Handler>): void {
    const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
    return json({ runs: latest(ctx.app.db, limit) });
  });
  router.add("GET", "/api/v1/maintenance/duties", (ctx) => {
    notePhase(ctx.request, "session", takeSessionTime(ctx.request));
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return forbidden();
    const denied = requireScope(ctx, "ro");
    if (denied) return denied;
    return json({ duties: dutyStates(ctx.app.db) });
  });
  router.add("POST", "/api/v1/maintenance/duties/:name", async (ctx) => {
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return forbidden();
    const denied = requireScope(ctx, "admin");
    if (denied) return denied;
    if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
    const input = await boundedBody(ctx.request, 1024);
    if (input === "too_large") return apiError("too_large", 413, "The request is too large.");
    const r = switchDuty(ctx.app.db, ctx.params["name"], input["paused"], ctx.auth.user.username);
    return r.ok ? json({ says: r.says }) : r.code === "not_found" ? notFound() : apiError("invalid", 422, r.says);
  });
}
@@ -2,22 +2,28 @@
import { prepare, resumeNow, status } from "../../../maintenance/service.ts";
import { duties } from "../../../maintenance/duties.ts";
import { statusWithStandby } from "../../../maintenance/standby.ts";
import { scopeAllows } from "../../../core/tokens.ts";
import { dutyStates, switchDuty } from "../../../maintain/pause.ts";
import { good, bad, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "maintenance",
  doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says }, duties }`",
  doc: "`action` is `status` (default), `prepare`, `resume`, `pause-duty` or `resume-duty` (with `duty`, a name from `status`); `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says }, duties }`",
  description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).",
  inputSchema: {
    type: "object",
    properties: { action: { type: "string", enum: ["status", "prepare", "resume"] }, reason: { type: "string", maxLength: 200 } },
    properties: { action: { type: "string", enum: ["status", "prepare", "resume", "pause-duty", "resume-duty"] }, duty: { type: "string", maxLength: 64 }, reason: { type: "string", maxLength: 200 } },
  },
  writes: true,
  async run(ctx, user, args) {
    if (!user.isAdmin) return bad("not_found", "no such tool here");
    const action = args["action"] ?? "status";
    if (action === "status") return good({ ...(await statusWithStandby(status(ctx.app))), duties: duties(ctx.app.config.dataDir).map((d) => d.says) });
    if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume");
    if (action === "status") return good({ ...(await statusWithStandby(status(ctx.app))), duties: [...duties(ctx.app.config.dataDir).map((d) => d.says), ...dutyStates(ctx.app.db).map((d) => d.says)] });
    if (action === "pause-duty" || action === "resume-duty") {
      if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
      const r = switchDuty(ctx.app.db, args["duty"], action === "pause-duty", user.username);
      return r.ok ? good({ says: r.says }) : bad(r.code, r.says);
    }
    if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare, resume, pause-duty or resume-duty");
    if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
    if (action === "resume") {
      const r = resumeNow(ctx.app, user);
@@ -30,7 +30,7 @@
import { contributePage } from "../web/pages/contribute.ts";
import { pipelinePage } from "../web/pages/pipeline.ts";
import { adminDataPage, appDataPage, dataPage } from "../web/pages/data.ts";
import { fleetAct, fleetReceipt, fleetUndo } from "../web/pages/fleet-ops.ts";
import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
import { maintenanceDuty, maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts";
import { agentStartSet, billingLinksSet, limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts";
import { plansSet } from "../web/settings/plans.ts";
@@ -986,6 +986,7 @@
export function buildRouter(): Router<Handler> {
  router.add("GET", "/maintenance", maintenancePage);
  router.add("POST", "/maintenance/prepare", maintenancePrepare);
  router.add("POST", "/maintenance/resume", maintenanceResume);
  router.add("POST", "/maintenance/duties/:name", maintenanceDuty);
  router.add("GET", "/resilience/nightly/:job", nightlyOutputPage);
  // A selection's spend and runs over time; literal, above /:owner.
  router.add("GET", "/fleet", fleetPage);
@@ -0,0 +1,70 @@
/** A maintenance duty's on/off switch (#11205): stored, set by a person or by three errors in a row; plain data in and out. */
import type { Database } from "bun:sqlite";
import { clip } from "./duty.ts";
import { duties as registered } from "./registry.ts";
/** Errored runs in a row that pause a duty. */
export const ERRORS_TO_PAUSE = 3;
export interface DutyState {
  readonly name: string;
  readonly paused: boolean;
  /** The row in words: the page, the API and the MCP say this. */
  readonly says: string;
  /** `says` without the name: what a row already headed by the name adds. */
  readonly detail: string;
}
interface Row { paused: number; reason: string; by: string; since: string; from_run: number }
const stateRow = (db: Database, name: string): Row | null =>
  db.query("SELECT paused, reason, by, since, from_run FROM maintenance_duty_state WHERE duty = ?1").get(name) as Row | null;
export const isPaused = (db: Database, name: string): boolean => stateRow(db, name)?.paused === 1;
/** A run that did not come out `ok`: it threw, timed out or had a finding refused. */
export const errored = (outcome: string): boolean => outcome !== "ok";
function put(db: Database, name: string, paused: boolean, reason: string, by: string, now: Date): void {
  const from = (db.query("SELECT COALESCE(MAX(id), 0) AS id FROM maintenance_run WHERE duty = ?1").get(name) as { id: number }).id;
  db.query(
    `INSERT INTO maintenance_duty_state (duty, paused, reason, by, since, from_run) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
     ON CONFLICT(duty) DO UPDATE SET paused = ?2, reason = ?3, by = ?4, since = ?5, from_run = CASE WHEN ?2 = 0 THEN ?6 ELSE from_run END`,
  ).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), from);
}
export const setPaused = (db: Database, name: string, paused: boolean, by: string, now = new Date()): void =>
  put(db, name, paused, paused ? `paused by ${by}` : "", by, now);
/** After a run: three errored runs in a row (since the last resume) pause the duty, with the last error as the reason. Returns whether it paused. */
export function pauseOnErrors(db: Database, name: string, now = new Date()): boolean {
  const from = stateRow(db, name)?.from_run ?? 0;
  const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[];
  if (rows.length < ERRORS_TO_PAUSE || !rows.every((r) => errored(r.outcome))) return false;
  put(db, name, true, `paused after ${ERRORS_TO_PAUSE} errors in a row; last error: ${rows[0]!.outcome}`, "the runner", now);
  return true;
}
export function dutyState(db: Database, name: string): DutyState {
  const s = stateRow(db, name);
  if (s?.paused === 1) {
    const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}`;
    return { name, paused: true, says: `${name}: ${detail}.`, detail };
  }
  const last = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(name) as { outcome: string } | null;
  const detail = `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`;
  return { name, paused: false, says: `${name}: ${detail}.`, detail };
}
export const dutyStates = (db: Database): DutyState[] => registered.map((d) => dutyState(db, d.name));
export const knownDuty = (name: string): boolean => registered.some((d) => d.name === name);
/** One switch for page, API and MCP: the same words. */
export function switchDuty(db: Database, name: unknown, paused: unknown, by: string): { ok: true; says: string } | { ok: false; code: "invalid" | "not_found"; says: string } {
  if (typeof name !== "string" || !knownDuty(name)) return { ok: false, code: "not_found", says: "No such maintenance duty." };
  if (typeof paused !== "boolean") return { ok: false, code: "invalid", says: "paused is true or false." };
  setPaused(db, name, paused, by);
  return { ok: true, says: paused ? `${name} is paused.` : `${name} is on again.` };
}
@@ -8,6 +8,7 @@
import type { App } from "../app.ts";
import { during } from "../perf/jobs.ts";
import { clip, refusal, validName, type Duty } from "./duty.ts";
import { duties as registered } from "./registry.ts";
import { isPaused, pauseOnErrors } from "./pause.ts";
import { lastStarted, prune, record } from "./runs.ts";
export const TICK_MS = 60_000;
@@ -62,6 +63,7 @@
export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
    try {
      record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome });
      prune(app.db, duty.name, options.keep ?? KEEP_PER_DUTY);
      if (pauseOnErrors(app.db, duty.name)) app.log.error("maintenance duty paused after errors", { duty: duty.name });
    } catch (error) {
      app.log.error("maintenance run not recorded", { duty: duty.name, error: String(error) });
    }
@@ -71,7 +73,7 @@
export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
    for (const duty of list) {
      const last = lastStarted(app.db, duty.name);
      if (last !== undefined && now.getTime() - last < duty.intervalMs) continue;
      if (running.has(duty.name)) continue;
      if (running.has(duty.name) || isPaused(app.db, duty.name)) continue;
      await runOne(duty, now);
      // Duties take turns: the loop turns between one and the next.
      await new Promise((resolve) => setImmediate(resolve));
@@ -0,0 +1,66 @@
/**
 * Pause and resume a maintenance duty in the terminal (#11205): the switches
 * of the /maintenance page, through the same `switchDuty`, behind the same
 * gate (a forge admin who is a person). j/k choose a duty, Enter or Space
 * flips it, Esc or q closes. The write is to the forge's own database, so a
 * web viewer sees it on its next load.
 */
import { dutyStates, switchDuty } from "../../maintain/pause.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { drawLines, Scroll } from "./common.ts";
import type { PaletteEntry } from "./entries.ts";
const mayAct = (session: Session): boolean => session.user?.isAdmin === true && session.user.kind !== "agent";
/** One duty a line, the chosen one marked; empty for anyone the gate refuses. */
export function dutyLines(session: Session, chosen: number): string[] {
  if (!mayAct(session)) return [];
  return dutyStates(session.db).map((d, i) => `${i === chosen ? ">" : " "} ${d.says}`);
}
/** Flip the chosen duty; what it says, or nothing for anyone the gate refuses. */
export function flipDuty(session: Session, chosen: number): string {
  if (!mayAct(session)) return "";
  const d = dutyStates(session.db)[chosen];
  if (d === undefined) return "";
  const r = switchDuty(session.db, d.name, !d.paused, session.user!.username);
  return r.says;
}
export function dutySwitchScreen(session: Session, ctx: Ctx): Screen {
  let chosen = 0;
  let note = "";
  const count = (): number => dutyStates(session.db).length;
  return {
    title: "Maintenance duties",
    keys: [binding(["j", "k"], "choose"), binding(["enter", "space"], "pause or resume"), binding(["esc", "q"], "back")],
    render(area, s) {
      const c = ctx.theme.c;
      const lines = [...dutyLines(session, chosen), ...(note === "" ? [] : ["", note])];
      drawLines(s, area, c, new Scroll(), lines.map((text) => [{ text, style: style(c.fg, c.bg) }]));
    },
    key(e) {
      if (e.ctrl || e.alt) return true;
      if (e.key === "esc" || e.key === "q") ctx.pop();
      else if (e.key === "j" || e.key === "down") chosen = Math.min(chosen + 1, count() - 1);
      else if (e.key === "k" || e.key === "up") chosen = Math.max(chosen - 1, 0);
      else if (e.key === "enter" || e.key === " " || e.key === "space") note = flipDuty(session, chosen);
      else return false;
      ctx.invalidate();
      return true;
    },
  };
}
export const entry: PaletteEntry = {
  id: "maintenance-duties",
  title: "Pause or resume a maintenance duty",
  hint: "switch a duty off, or on again after it paused itself",
  after: ["maintenance-runs", "maintenance"],
  offered: mayAct,
  run: (session, ctx) => ctx.push(dutySwitchScreen(session, ctx)),
};
@@ -57,6 +57,7 @@
import { entry as loopStalls } from "./loopstalls.ts";
import { entry as slowPagesEntry } from "./slowpages.ts";
import { entry as maintenance } from "./maintenance.ts";
import { entry as maintainRuns } from "./maintainruns.ts";
import { entry as dutySwitch } from "./dutyswitch.ts";
import { entry as mirrorPacksEntry } from "./mirrorpacks.ts";
import { entry as packReposEntry } from "./packrepos.ts";
import { modelCeilingEntry, modelGrantEntry } from "./model.ts";
@@ -118,6 +119,7 @@
export const ENTRIES: readonly PaletteEntry[] = [
  resilience,
  maintenance,
  maintainRuns,
  dutySwitch,
  mirrorPacksEntry,
  packReposEntry,
  valves,
@@ -5,6 +5,7 @@
 */
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { dutyStates, switchDuty } from "../../maintain/pause.ts";
import { duties } from "../../maintenance/duties.ts";
import { statusWithStandby } from "../../maintenance/standby.ts";
import { ABOUT_TO_STOP_ROWS, prepare, resumeNow, status } from "../../maintenance/service.ts";
@@ -34,6 +35,12 @@
export async function maintenancePage(ctx: Ctx): Promise<Response> {
    <p id="maintenance-says">${s.says}</p>
    <dl id="maintenance-duties" class="status-rows">
      ${duties(ctx.app.config.dataDir).map((d) => fact(d.name, html`${d.detail}${d.overdue ? html` ${badge("overdue", { tone: "danger" })}` : null}`))}
      ${dutyStates(ctx.app.db).map((d) =>
        fact(d.name, html`${d.detail}
      
        sentinelcrew
        1d ago
        outdated
      
      Principles 1 and 7: every duty is listed twice, in maintenance-duties and again here, so the page says the same thing twice. Fold the Pause/Resume form and the pause state into the existing duty row, one row per duty.
    
          <form method="post" action="/maintenance/duties/${d.name}" class="inline">
            <input type="hidden" name="paused" value="${d.paused ? "no" : "yes"}">${btn(d.paused ? "Resume" : "Pause", { variant: "outline" })}
          </form>`),
      
        sentinelcrew
        1d ago
        outdated
      
      The state sentence and a button share one dd. Put the sentence in the row and keep the button as the single interaction, per DESIGN.md rows-over-boxes. Also formField(...) === "yes" treats any other value as resume; fine but keep the values explicit.
    
      )}
    </dl>
    ${s.phase === "idle" ? html`<p id="maintenance-standby" class="muted">${s.standby.says}</p>` : null}
    ${s.phase === "idle"
@@ -88,3 +95,14 @@
export async function maintenanceResume(ctx: Ctx): Promise<Response> {
  const r = resumeNow(ctx.app, ctx.auth.user);
  return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says });
}
export async function maintenanceDuty(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
      
        sentinelcrew
        1d ago
        outdated
      
      maintenanceDuty has no test: nothing posts the form, nothing checks that a non-admin or cross-origin post is refused. Add them.
    
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  const want = formField(form, "paused");
  if (want !== "yes" && want !== "no") return flashTo(ctx, "/maintenance", { tone: "danger", message: "paused is yes or no." });
  const r = switchDuty(ctx.app.db, ctx.params["name"], want === "yes", ctx.auth.user!.username);
  return flashTo(ctx, "/maintenance", r.ok ? { message: r.says } : { tone: "danger", message: r.says });
}
@@ -0,0 +1,189 @@
/** A maintenance duty's switch (#11205): the API, the MCP, the page, and three errors in a row. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import type { Duty } from "../src/maintain/duty.ts";
import { dutyState, isPaused, pauseOnErrors, setPaused } from "../src/maintain/pause.ts";
import { duties } from "../src/maintain/registry.ts";
import { startMaintain } from "../src/maintain/runner.ts";
import { dutySwitchScreen, entry as switchEntry, flipDuty } from "../src/tui/screens/dutyswitch.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
let root = "";
let app: App;
let servers: Servers;
let admin = "";
let ro = "";
const name = duties[0]!.name;
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-pause-"));
  app = createApp({ dataDir: root, port: 0 });
  servers = await serve(app);
  const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  if (!a.ok) throw new Error("user");
  const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
  if (!b.ok) throw new Error("user");
  admin = createToken(app.db, a.value.id, "a", "admin").token;
  ro = createToken(app.db, a.value.id, "r", "ro").token;
});
afterAll(() => {
  servers.stop?.();
  rmSync(root, { recursive: true, force: true });
});
const post = (token: string, n: string, body: unknown) =>
  fetch(`${servers.url}/api/v1/maintenance/duties/${n}`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify(body) });
const mcp = async (token: string, args: unknown) => {
  const res = await fetch(`${servers.url}/mcp`, {
    method: "POST",
    headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "maintenance", arguments: args } }),
  });
  const b = (await res.json()) as any;
  return { error: b.result.isError === true, text: b.result.content[0].text as string };
};
test("the API pauses and resumes a duty, admin scope only, unknown names and bad bodies refused", async () => {
  expect((await post(ro, name, { paused: true })).status).toBe(403);
  expect((await post(admin, "nope", { paused: true })).status).toBe(404);
  expect((await post(admin, name, { paused: "yes" })).status).toBe(422);
  expect((await post(admin, name, { paused: true })).status).toBe(200);
  expect(isPaused(app.db, name)).toBe(true);
  const list = (await (await fetch(`${servers.url}/api/v1/maintenance/duties`, { headers: { authorization: `Bearer ${ro}` } })).json()) as any;
  expect(list.duties.find((d: any) => d.name === name).says).toContain("paused");
  expect((await post(admin, name, { paused: false })).status).toBe(200);
  expect(isPaused(app.db, name)).toBe(false);
});
test("the MCP maintenance tool pauses and resumes with the same words", async () => {
  const p = await mcp(admin, { action: "pause-duty", duty: name });
  expect(p.error).toBe(false);
  expect(p.text).toContain(`${name} is paused.`);
  expect((await mcp(admin, { action: "status" })).text).toContain(`${name}: paused`);
  expect((await mcp(ro, { action: "pause-duty", duty: name })).error).toBe(true);
  expect((await mcp(admin, { action: "resume-duty", duty: "nope" })).error).toBe(true);
      
        sentinelcrew
        1d ago
        outdated
      
      This test is named "its form flips it" but only checks that the form renders. Post it as an admin and assert isPaused flips, and that a non-admin is refused. Also resume in the auto-pause test goes through raw SQL, so the real resume code path (from_run reset) is not covered.
    
  expect((await mcp(admin, { action: "resume-duty", duty: name })).text).toContain(`${name} is on again.`);
});
const login = async (username: string): Promise<string> => {
  const res = await fetch(`${servers.url}/signin`, {
    method: "POST",
    redirect: "manual",
    headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url },
    body: new URLSearchParams({ username, password: "a-good-passphrase" }),
  });
  return (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
};
const form = (cookie: string, n: string, fields: Record<string, string>, origin = servers.url) =>
  fetch(`${servers.url}/maintenance/duties/${n}`, {
    method: "POST",
    redirect: "manual",
    headers: { "content-type": "application/x-www-form-urlencoded", origin, cookie },
    body: new URLSearchParams(fields),
  });
test("the page lists each duty once, its switch in the row", async () => {
  const html = await (await fetch(`${servers.url}/maintenance`, { headers: { authorization: `Bearer ${admin}` } })).text();
  expect(html).toContain(`action="/maintenance/duties/${name}"`);
  expect(html.split(`<dt>${name}</dt>`).length - 1).toBe(1);
  expect(html.split('id="maintenance-duties"').length - 1).toBe(1);
  expect(html).not.toContain('id="maintenance-switches"');
});
test("the page form flips a duty for an admin; a non-admin, a cross-origin post and a stray value change nothing", async () => {
  app.db.exec("DELETE FROM maintenance_duty_state");
  const alice = await login("alice");
  const bob = await login("bob");
  expect((await form(bob, name, { paused: "yes" })).status).toBeGreaterThanOrEqual(400);
  expect(isPaused(app.db, name)).toBe(false);
  expect((await form(alice, name, { paused: "yes" }, "https://evil.example")).status).toBe(403);
  expect(isPaused(app.db, name)).toBe(false);
  expect((await form(alice, name, { paused: "maybe" })).status).toBe(303);
  expect(isPaused(app.db, name)).toBe(false);
  expect((await form(alice, "nope", { paused: "yes" })).status).toBe(303);
  expect((await form(alice, name, { paused: "yes" })).status).toBe(303);
  expect(isPaused(app.db, name)).toBe(true);
  expect(await (await fetch(`${servers.url}/maintenance`, { headers: { cookie: alice } })).text()).toContain("Resume");
  expect((await form(alice, name, { paused: "no" })).status).toBe(303);
  expect(isPaused(app.db, name)).toBe(false);
});
test("the terminal switch is offered to an admin person only and flips the duty through the same code", () => {
  app.db.exec("DELETE FROM maintenance_duty_state");
  const session = (user: unknown) => ({ user, app, db: app.db }) as never;
  const adminUser = { id: "a", username: "alice", isAdmin: true, kind: "person" };
  expect(switchEntry.offered(session(adminUser))).toBe(true);
  expect(switchEntry.offered(session({ ...adminUser, kind: "agent" }))).toBe(false);
  expect(switchEntry.offered(session({ ...adminUser, isAdmin: false }))).toBe(false);
  expect(flipDuty(session({ ...adminUser, isAdmin: false }), 0)).toBe("");
  expect(isPaused(app.db, name)).toBe(false);
  expect(flipDuty(session(adminUser), 0)).toBe(`${name} is paused.`);
  expect(isPaused(app.db, name)).toBe(true);
  const screen = dutySwitchScreen(session(adminUser), { invalidate: () => undefined } as never);
  screen.key({ key: "enter" } as never);
  expect(isPaused(app.db, name)).toBe(false);
});
test("a paused duty does not run; three errors in a row pause it saying the last error; resume starts the count again", async () => {
  app.db.exec("DELETE FROM maintenance_run; DELETE FROM maintenance_duty_state");
  let calls = 0;
  const bad: Duty = { name: "t-bad", intervalMs: 1, run: () => { calls++; throw new Error(`boom ${calls}`); } };
  const m = startMaintain(app, { duties: [bad], everyMs: 1e9 });
  m.stop();
  const t = (i: number) => new Date(Date.UTC(2026, 10, 11, 0, 0, i));
  await m.tick(t(1));
  await m.tick(t(2));
  expect(isPaused(app.db, "t-bad")).toBe(false);
  await m.tick(t(3));
  expect(isPaused(app.db, "t-bad")).toBe(true);
  expect(dutyState(app.db, "t-bad").says).toContain("last error: failed: boom 3");
  await m.tick(t(4));
  expect(calls).toBe(3);
  // Resumed by hand: it runs again, and one more error does not re-pause it at once.
  setPaused(app.db, "t-bad", false, "alice");
  await m.tick(t(5));
  expect(calls).toBe(4);
  expect(isPaused(app.db, "t-bad")).toBe(false);
  await m.tick(t(6));
  expect(isPaused(app.db, "t-bad")).toBe(false);
  await m.tick(t(7));
  expect(calls).toBe(6);
  expect(isPaused(app.db, "t-bad")).toBe(true);
});
test("an ok run in between keeps the count from reaching three", async () => {
  app.db.exec("DELETE FROM maintenance_run; DELETE FROM maintenance_duty_state");
  let n = 0;
  const flaky: Duty = { name: "t-flaky", intervalMs: 1, run: () => { if (++n === 3) return { findings: [], actions: [] }; throw new Error("x"); } };
  const m = startMaintain(app, { duties: [flaky], everyMs: 1e9 });
  m.stop();
  for (let i = 1; i <= 5; i++) await m.tick(new Date(Date.UTC(2026, 10, 11, 0, 0, i)));
  expect(isPaused(app.db, "t-flaky")).toBe(false);
});
test("resuming through the API and through the page starts the count again", async () => {
  const fail = () => app.db.run("INSERT INTO maintenance_run (duty, started, finished, outcome) VALUES (?1, ?2, ?2, 'failed: x')", [name, new Date().toISOString()]);
  app.db.exec("DELETE FROM maintenance_run; DELETE FROM maintenance_duty_state");
  const viaApi = async () => {
    for (let i = 0; i < 3; i++) fail();
    expect(pauseOnErrors(app.db, name)).toBe(true);
  };
  await viaApi();
  expect((await post(admin, name, { paused: false })).status).toBe(200);
  fail();
  expect(pauseOnErrors(app.db, name)).toBe(false);
  fail();
  fail();
  expect(pauseOnErrors(app.db, name)).toBe(true);
  expect((await form(await login("alice"), name, { paused: "no" })).status).toBe(303);
  fail();
  expect(pauseOnErrors(app.db, name)).toBe(false);
  expect(isPaused(app.db, name)).toBe(false);
});
@@ -118,6 +118,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["resilience", "Resilience", "where snapshots go and the mirror; each protection green, red or untested"],
    ["maintenance", "Maintenance", "prepare the forge for a power-off, check the drain, or end maintenance"],
    ["maintenance-runs", "Maintenance duty runs", "what each duty found, did and probed"],
    ["maintenance-duties", "Pause or resume a maintenance duty", "switch a duty off, or on again after it paused itself"],
    ["crawlers", "Crawlers", "their share of answering time, and who was refused"],
    ["scrapes", "Scrapes", "what was read by repository, and groups in a running capture"],
    ["traffic", "Traffic", "who is using the forge's answering time, and the top readers"],
		@@ -134,7 +134,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
134	134	\| `lift` \| — \| Lift an issue out from behind the Later gate \| 214 \|
135	135	\| `limb` \| `defer`, `resume`, `close`, `raise`, `lower`, `replan`, `takeback`, `undo` \| Act on a limb by `action` \| 532 \|
136	136	\| `limits` \| — \| Every limit the owner can hit, value and bounds \| 977 \|
137		\| `maintenance` \| `status`, `prepare`, `resume` \| Prepare the forge for a power-off, check the drain, or end maintenance (admin) \| 256 \|
	137	\| `maintenance` \| `status`, `prepare`, `resume`, `pause-duty`, `resume-duty` \| Prepare the forge for a power-off, check the drain, or end maintenance (admin) \| 308 \|
138	138	\| `make_it_yours` \| — \| Fork an app you can see: code and history, following the original \| 337 \|
139	139	\| `map` \| — \| An app's plan: ready or waiting, critical path, crew, parkedTwice; `done` adds closed; `first: N` ranks N first \| 375 \|
140	140	\| `merge` \| — \| Land a pull, as its page's Merge button does; a person's only \| 319 \|
		@@ -0,0 +1,14 @@
	1	# Pause and resume a maintenance duty (#11205)
	2
	3	Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once.
	4
	5	All doors are for a forge admin and call one function, `switchDuty` in `src/maintain/pause.ts`, so they say the same words.
	6
	7	- On the web. `/maintenance` lists each duty once; the ones the runner runs carry their state ("on", or "paused since … ; why") and one Pause / Resume button (`POST /maintenance/duties/:name`, same-origin only, form body bounded, `paused` is `yes` or `no`).
	8	- In the terminal. Ctrl-K, Pause or resume a maintenance duty (an admin who is a person): `j`/`k` choose, Enter or Space flips, Esc closes. It writes the same row, so a web viewer sees it on its next load.
	9	- Over the API. `GET /api/v1/maintenance/duties` lists the states; `POST /api/v1/maintenance/duties/:name` with `{"paused": true\|false}` needs an `admin`-scope token.
	10	- In the MCP. The `maintenance` tool: `pause-duty` and `resume-duty` with `duty`.
	11
	12	An unknown duty name is 404 (a flash on the page); a `paused` that is not a boolean is 422.
	13
	14	Tests: `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates).
		@@ -0,0 +1,11 @@
	1	import type { Route } from "../shared.ts";
	2
	3	export const route: Route = {
	4	method: "get",
	5	path: "/api/v1/maintenance/duties",
	6	summary:
	7	"Each maintenance duty with whether it is paused, and why and by whom (a forge admin's token); answers { duties }",
	8	auth: true,
	9	role: "admin",
	10	refusals: { forbidden: "the token is not a forge admin" },
	11	};
		@@ -102,6 +102,7 @@ export { route as getApiV1Front } from "./getApiV1Front.ts";
102	102	export { route as getApiV1Health } from "./getApiV1Health.ts";
103	103	export { route as getApiV1Here } from "./getApiV1Here.ts";
104	104	export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
	105	export { route as getApiV1MaintenanceDuties } from "./getApiV1MaintenanceDuties.ts";
105	106	export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
106	107	export { route as getApiV1Map } from "./getApiV1Map.ts";
107	108	export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
		@@ -286,6 +287,7 @@ export { route as postApiV1FleetOpsIdUndo } from "./postApiV1FleetOpsIdUndo.ts";
286	287	export { route as postApiV1Forges } from "./postApiV1Forges.ts";
287	288	export { route as postApiV1ForgesNameClaimLink } from "./postApiV1ForgesNameClaimLink.ts";
288	289	export { route as postApiV1Heartbeat } from "./postApiV1Heartbeat.ts";
	290	export { route as postApiV1MaintenanceDutiesName } from "./postApiV1MaintenanceDutiesName.ts";
289	291	export { route as postApiV1Migrate } from "./postApiV1Migrate.ts";
290	292	export { route as postApiV1Orgs } from "./postApiV1Orgs.ts";
291	293	export { route as postApiV1OrgsOrgTeams } from "./postApiV1OrgsOrgTeams.ts";
		@@ -0,0 +1,1 @@
	1	391de19c4b728caa065a215f40be9cf48eda5929d63d10876598d85b728932eb
		@@ -0,0 +1,12 @@
	1	// A maintenance duty's on/off switch (#11205): paused by a person or by three errors in a row,
	2	// with why and when. `from_run` is the newest run at the last resume: only later runs count toward three.
	3	export default `
	4	CREATE TABLE maintenance_duty_state (
	5	duty TEXT PRIMARY KEY,
	6	paused INTEGER NOT NULL DEFAULT 0,
	7	reason TEXT NOT NULL DEFAULT '',
	8	by TEXT NOT NULL DEFAULT '',
	9	since TEXT NOT NULL DEFAULT '',
	10	from_run INTEGER NOT NULL DEFAULT 0
	11	);
	12	`;
		@@ -1,9 +1,11 @@
1	1	/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
2	2
	3	import { switchDuty, dutyStates } from "../../maintain/pause.ts";
3	4	import { latest } from "../../maintain/runs.ts";
4	5	import type { Handler } from "../context.ts";
5	6	import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
6		import { forbidden, json, requireScope, unauthorized } from "../respond.ts";
	7	import { crossOrigin } from "../context.ts";
	8	import { apiError, boundedBody, forbidden, json, notFound, requireScope, unauthorized } from "../respond.ts";
7	9	import type { Router } from "../router.ts";
8	10
9	11	export function maintenanceRunsRoutes(router: Router<Handler>): void {
		@@ -17,4 +19,23 @@ export function maintenanceRunsRoutes(router: Router<Handler>): void {
17	19	const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
18	20	return json({ runs: latest(ctx.app.db, limit) });
19	21	});
	22	router.add("GET", "/api/v1/maintenance/duties", (ctx) => {
	23	notePhase(ctx.request, "session", takeSessionTime(ctx.request));
	24	if (!ctx.auth.user) return unauthorized();
	25	if (!ctx.auth.user.isAdmin) return forbidden();
	26	const denied = requireScope(ctx, "ro");
	27	if (denied) return denied;
	28	return json({ duties: dutyStates(ctx.app.db) });
	29	});
	30	router.add("POST", "/api/v1/maintenance/duties/:name", async (ctx) => {
	31	if (!ctx.auth.user) return unauthorized();
	32	if (!ctx.auth.user.isAdmin) return forbidden();
	33	const denied = requireScope(ctx, "admin");
	34	if (denied) return denied;
	35	if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
	36	const input = await boundedBody(ctx.request, 1024);
	37	if (input === "too_large") return apiError("too_large", 413, "The request is too large.");
	38	const r = switchDuty(ctx.app.db, ctx.params["name"], input["paused"], ctx.auth.user.username);
	39	return r.ok ? json({ says: r.says }) : r.code === "not_found" ? notFound() : apiError("invalid", 422, r.says);
	40	});
20	41	}
		@@ -2,22 +2,28 @@ import { prepare, resumeNow, status } from "../../../maintenance/service.ts";
2	2	import { duties } from "../../../maintenance/duties.ts";
3	3	import { statusWithStandby } from "../../../maintenance/standby.ts";
4	4	import { scopeAllows } from "../../../core/tokens.ts";
	5	import { dutyStates, switchDuty } from "../../../maintain/pause.ts";
5	6	import { good, bad, type Tool } from "../shared.ts";
6	7
7	8	export const tool: Tool = {
8	9	name: "maintenance",
9		doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says }, duties }`",
	10	doc: "`action` is `status` (default), `prepare`, `resume`, `pause-duty` or `resume-duty` (with `duty`, a name from `status`); `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says }, duties }`",
10	11	description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).",
11	12	inputSchema: {
12	13	type: "object",
13		properties: { action: { type: "string", enum: ["status", "prepare", "resume"] }, reason: { type: "string", maxLength: 200 } },
	14	properties: { action: { type: "string", enum: ["status", "prepare", "resume", "pause-duty", "resume-duty"] }, duty: { type: "string", maxLength: 64 }, reason: { type: "string", maxLength: 200 } },
14	15	},
15	16	writes: true,
16	17	async run(ctx, user, args) {
17	18	if (!user.isAdmin) return bad("not_found", "no such tool here");
18	19	const action = args["action"] ?? "status";
19		if (action === "status") return good({ ...(await statusWithStandby(status(ctx.app))), duties: duties(ctx.app.config.dataDir).map((d) => d.says) });
20		if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume");
	20	if (action === "status") return good({ ...(await statusWithStandby(status(ctx.app))), duties: [...duties(ctx.app.config.dataDir).map((d) => d.says), ...dutyStates(ctx.app.db).map((d) => d.says)] });
	21	if (action === "pause-duty" \|\| action === "resume-duty") {
	22	if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
	23	const r = switchDuty(ctx.app.db, args["duty"], action === "pause-duty", user.username);
	24	return r.ok ? good({ says: r.says }) : bad(r.code, r.says);
	25	}
	26	if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare, resume, pause-duty or resume-duty");
21	27	if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
22	28	if (action === "resume") {
23	29	const r = resumeNow(ctx.app, user);
		@@ -30,7 +30,7 @@ import { contributePage } from "../web/pages/contribute.ts";
30	30	import { pipelinePage } from "../web/pages/pipeline.ts";
31	31	import { adminDataPage, appDataPage, dataPage } from "../web/pages/data.ts";
32	32	import { fleetAct, fleetReceipt, fleetUndo } from "../web/pages/fleet-ops.ts";
33		import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
	33	import { maintenanceDuty, maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
34	34	import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts";
35	35	import { agentStartSet, billingLinksSet, limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts";
36	36	import { plansSet } from "../web/settings/plans.ts";
		@@ -986,6 +986,7 @@ export function buildRouter(): Router<Handler> {
986	986	router.add("GET", "/maintenance", maintenancePage);
987	987	router.add("POST", "/maintenance/prepare", maintenancePrepare);
988	988	router.add("POST", "/maintenance/resume", maintenanceResume);
	989	router.add("POST", "/maintenance/duties/:name", maintenanceDuty);
989	990	router.add("GET", "/resilience/nightly/:job", nightlyOutputPage);
990	991	// A selection's spend and runs over time; literal, above /:owner.
991	992	router.add("GET", "/fleet", fleetPage);
		@@ -0,0 +1,70 @@
	1	/** A maintenance duty's on/off switch (#11205): stored, set by a person or by three errors in a row; plain data in and out. */
	2
	3	import type { Database } from "bun:sqlite";
	4	import { clip } from "./duty.ts";
	5	import { duties as registered } from "./registry.ts";
	6
	7	/** Errored runs in a row that pause a duty. */
	8	export const ERRORS_TO_PAUSE = 3;
	9
	10	export interface DutyState {
	11	readonly name: string;
	12	readonly paused: boolean;
	13	/** The row in words: the page, the API and the MCP say this. */
	14	readonly says: string;
	15	/** `says` without the name: what a row already headed by the name adds. */
	16	readonly detail: string;
	17	}
	18
	19	interface Row { paused: number; reason: string; by: string; since: string; from_run: number }
	20
	21	const stateRow = (db: Database, name: string): Row \| null =>
	22	db.query("SELECT paused, reason, by, since, from_run FROM maintenance_duty_state WHERE duty = ?1").get(name) as Row \| null;
	23
	24	export const isPaused = (db: Database, name: string): boolean => stateRow(db, name)?.paused === 1;
	25
	26	/** A run that did not come out `ok`: it threw, timed out or had a finding refused. */
	27	export const errored = (outcome: string): boolean => outcome !== "ok";
	28
	29	function put(db: Database, name: string, paused: boolean, reason: string, by: string, now: Date): void {
	30	const from = (db.query("SELECT COALESCE(MAX(id), 0) AS id FROM maintenance_run WHERE duty = ?1").get(name) as { id: number }).id;
	31	db.query(
	32	`INSERT INTO maintenance_duty_state (duty, paused, reason, by, since, from_run) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
	33	ON CONFLICT(duty) DO UPDATE SET paused = ?2, reason = ?3, by = ?4, since = ?5, from_run = CASE WHEN ?2 = 0 THEN ?6 ELSE from_run END`,
	34	).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), from);
	35	}
	36
	37	export const setPaused = (db: Database, name: string, paused: boolean, by: string, now = new Date()): void =>
	38	put(db, name, paused, paused ? `paused by ${by}` : "", by, now);
	39
	40	/** After a run: three errored runs in a row (since the last resume) pause the duty, with the last error as the reason. Returns whether it paused. */
	41	export function pauseOnErrors(db: Database, name: string, now = new Date()): boolean {
	42	const from = stateRow(db, name)?.from_run ?? 0;
	43	const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[];
	44	if (rows.length < ERRORS_TO_PAUSE \|\| !rows.every((r) => errored(r.outcome))) return false;
	45	put(db, name, true, `paused after ${ERRORS_TO_PAUSE} errors in a row; last error: ${rows[0]!.outcome}`, "the runner", now);
	46	return true;
	47	}
	48
	49	export function dutyState(db: Database, name: string): DutyState {
	50	const s = stateRow(db, name);
	51	if (s?.paused === 1) {
	52	const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}`;
	53	return { name, paused: true, says: `${name}: ${detail}.`, detail };
	54	}
	55	const last = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(name) as { outcome: string } \| null;
	56	const detail = `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`;
	57	return { name, paused: false, says: `${name}: ${detail}.`, detail };
	58	}
	59
	60	export const dutyStates = (db: Database): DutyState[] => registered.map((d) => dutyState(db, d.name));
	61
	62	export const knownDuty = (name: string): boolean => registered.some((d) => d.name === name);
	63
	64	/** One switch for page, API and MCP: the same words. */
	65	export function switchDuty(db: Database, name: unknown, paused: unknown, by: string): { ok: true; says: string } \| { ok: false; code: "invalid" \| "not_found"; says: string } {
	66	if (typeof name !== "string" \|\| !knownDuty(name)) return { ok: false, code: "not_found", says: "No such maintenance duty." };
	67	if (typeof paused !== "boolean") return { ok: false, code: "invalid", says: "paused is true or false." };
	68	setPaused(db, name, paused, by);
	69	return { ok: true, says: paused ? `${name} is paused.` : `${name} is on again.` };
	70	}