b3b23a7 · 3869000 · f7ea4d8 · e45070f · 10b84d0
@@ -0,0 +1,13 @@
# Maintenance duties (#11204)
A duty is one file in `src/maintain/duties/`: a `name`, an `intervalMs` and a `run` that returns `findings` and `actions`. Register it with one line in `src/maintain/registry.ts`.
Every minute the runner (`src/maintain/runner.ts`, inside `during("maintain")`) runs each duty whose interval has passed since it last started, never two runs of one duty at once, and writes a `maintenance_run` row: duty, started, finished, findings, actions, probes, outcome.
A finding must carry a probe: `{ kind: "query" | "measurement", what, number, at }`. A finding without one is refused: the run keeps no findings or actions and its outcome reads `refused: finding "…" has no probe`. A duty that throws reads `failed: <message>`.
`GET /api/v1/maintenance/runs?limit=20` (forge admin, `ro` scope) lists the latest rows. The shipped duty, `pulls-waiting`, counts open pulls older than 24 hours.
The terminal door is the palette entry "Maintenance duty runs" (`src/tui/screens/maintainruns.ts`): the same rows through the same read, offered only to a forge admin; anyone else is offered nothing and sees no rows.
Each run is bounded: a duty that has not answered in 30 seconds reads `failed: timed out` and the next duty runs (the late duty keeps its slot until it settles, so it never overlaps itself). A duty name must be lowercase words and dashes and unique, or the runner refuses to start. Each duty keeps its newest 200 rows; older ones are deleted after each run. A finding that says nothing reads `refused: a finding says nothing`. A non-admin's token reads 403 `forbidden`.
@@ -0,0 +1,11 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "get",
  path: "/api/v1/maintenance/runs",
  summary:
    "The latest maintenance duty runs, newest first, each with its findings, actions and probes (a forge admin's token; ?limit= up to 100); answers { runs }",
  auth: true,
  role: "admin",
  refusals: { forbidden: "the token is not a forge admin" },
};
@@ -102,6 +102,7 @@
export { route as getApiV1Front } from "./getApiV1Front.ts";
export { route as getApiV1Health } from "./getApiV1Health.ts";
export { route as getApiV1Here } from "./getApiV1Here.ts";
export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
export { route as getApiV1Map } from "./getApiV1Map.ts";
export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
export { route as getApiV1MigrateJobStream } from "./getApiV1MigrateJobStream.ts";
@@ -52,6 +52,7 @@
const ALIAS: Readonly<Record<string, string>> = {
  heartbeat: "meta",
  telemetry: "meta",
  drills: "meta",
  maintenance: "meta",
  devices: "meta",
  thoughts: "feed",
  share: "feed",
@@ -0,0 +1,1 @@
c7266eda3263e87bc4570f7dd1403080b279857caf3f4fb5a21414509bf13fa5
@@ -0,0 +1,19 @@
// Maintenance runs (#11204): one row per run of a maintenance duty, with what
// it found, what it did and the probe behind each finding. Additive.
// The index `maintenance_run_duty` serves "when did this duty last start" and
// the pruning of a duty's old rows; the primary key serves the latest-first
// list. The example duty adds no index to `issue`: a new one there changes
// the planner's choice for the cold reads.
export default `
  CREATE TABLE maintenance_run (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    duty TEXT NOT NULL,
    started TEXT NOT NULL,
    finished TEXT NOT NULL,
    findings TEXT NOT NULL DEFAULT '[]',
    actions TEXT NOT NULL DEFAULT '[]',
    probes TEXT NOT NULL DEFAULT '[]',
    outcome TEXT NOT NULL
  );
  CREATE INDEX maintenance_run_duty ON maintenance_run (duty, id);
  `;
@@ -369,6 +369,7 @@
import { businessPlanRoutes } from "./api/business_plan.ts";
import { tokenRoutes } from "./api/tokens.ts";
import { accountAdmin, adminRoutes } from "./api/admin.ts";
import { maintenanceRoutes } from "./api/maintenance.ts";
import { maintenanceRunsRoutes } from "./api/maintenance_runs.ts";
import { diskReadingRoutes } from "./api/disk_readings.ts";
import { TEMPLATE_ERRORS } from "../apps/template_tip.ts";
import { flagsRoutes } from "./api/flags.ts";
@@ -1260,6 +1261,7 @@
export function apiRoutes(router: Router<Handler>): void {
  forgeOriginRoutes(router);
  adminRoutes(router);
  maintenanceRoutes(router);
  maintenanceRunsRoutes(router);
  diskReadingRoutes(router);
  flagsRoutes(router);
  appTemplateRoutes(router);
@@ -0,0 +1,20 @@
/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
import { latest } from "../../maintain/runs.ts";
import type { Handler } from "../context.ts";
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { forbidden, json, requireScope, unauthorized } from "../respond.ts";
import type { Router } from "../router.ts";
export function maintenanceRunsRoutes(router: Router<Handler>): void {
  router.add("GET", "/api/v1/maintenance/runs", (ctx) => {
    notePhase(ctx.request, "session", takeSessionTime(ctx.request));
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return forbidden();
      
        sentinelcrew
        22h ago
        outdated
      
      Non-admin gets 404 while the contract declares forbidden/member. Align them.
    
    const denied = requireScope(ctx, "ro");
    if (denied) return denied;
    const asked = Number(ctx.url.searchParams.get("limit") ?? 20);
    const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
    return json({ runs: latest(ctx.app.db, limit) });
  });
}
@@ -59,6 +59,7 @@
import { VERIFY_EVERY_MS } from "./checks/verify.ts";
import { RED_HOLD_MS } from "./checks/red_main.ts";
import { stopQueue } from "./checks/queue.ts";
import { startLooks } from "./looks/service.ts";
import { startMaintain } from "./maintain/runner.ts";
import { startMirrorOut } from "./origins/mirror-out.ts";
import { startMirrorPulls } from "./origins/mirror-pulls.ts";
import { startProbes } from "./contract/probes.ts";
@@ -501,6 +502,8 @@
const mirrorPulls = startMirrorPulls(app);
// The look: a picture of every environment that comes up green, taken
// through this forge's own listener.
const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined;
// Maintenance duties (src/maintain): each runs on its interval and leaves a row.
const maintain = startMaintain(app);
// Is it so: the forge knocks on every door of an app with a contract after
// each production deploy and once a night, and compares with the table.
const probes = apps === undefined ? undefined : startProbes(app, apps);
@@ -953,6 +956,7 @@
function drain(signal: NodeJS.Signals): Promise<unknown> {
    mirrorOut(),
    mirrorPulls(),
    looks?.stop(),
    maintain.stop(),
    probes?.stop(),
    nightly?.stop(),
    snapshots.stop(),
@@ -0,0 +1,25 @@
/** Example duty (#11204): how many open pulls have waited over 24 hours. Reads only. */
import type { Duty } from "../duty.ts";
const DAY_MS = 24 * 3_600_000;
const SQL = "SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1";
export const duty: Duty = {
  name: "pulls-waiting",
  intervalMs: 3_600_000,
  run({ db, now }) {
    const cutoff = new Date(now.getTime() - DAY_MS).toISOString();
    const n = (db.query(SQL).get(cutoff) as { n: number }).n;
    if (n === 0) return { findings: [], actions: [] };
    return {
      findings: [
        {
          summary: `${n} open pull${n === 1 ? "" : "s"} waiting over 24 hours`,
          probe: { kind: "query", what: SQL.replace("?1", `'${cutoff}'`), number: n, at: now.toISOString() },
        },
      ],
      actions: [],
    };
  },
};
@@ -0,0 +1,63 @@
/** What a maintenance duty is (#11204): a name, an interval and a handler; plain data in, plain data out. */
import type { Database } from "bun:sqlite";
/** The number a finding rests on and when it was read: a query's result or a measurement. */
export interface Probe {
  readonly kind: "query" | "measurement";
  /** The SQL, or what was measured. */
  readonly what: string;
  readonly number: number;
  /** ISO time the number was read. */
  readonly at: string;
}
export interface Finding {
  readonly summary: string;
  readonly probe?: Probe | undefined;
}
export interface DutyResult {
  readonly findings: readonly Finding[];
  /** What the duty did about it, one line each. */
  readonly actions: readonly string[];
}
export interface DutyContext {
  readonly db: Database;
  readonly now: Date;
}
export interface Duty {
  /** Lowercase words and dashes; the row's `duty`. */
  readonly name: string;
  readonly intervalMs: number;
  readonly run: (ctx: DutyContext) => DutyResult | Promise<DutyResult>;
}
const NAME = /^[a-z][a-z0-9-]{0,63}$/;
const MAX_TEXT = 500;
export const validName = (name: string): boolean => NAME.test(name);
      
        sentinelcrew
        22h ago
        
      
      validName is unused: dead code unless enforced.
    
/** Why a probe is no probe, or undefined when it is one. */
export function probeFault(p: Probe | undefined): string | undefined {
  if (p === undefined || p === null) return "no probe";
  if (p.kind !== "query" && p.kind !== "measurement") return "probe kind is neither query nor measurement";
  if (typeof p.what !== "string" || p.what.trim() === "") return "probe says nothing of what was read";
  if (typeof p.number !== "number" || !Number.isFinite(p.number)) return "probe has no number";
  if (typeof p.at !== "string" || Number.isNaN(Date.parse(p.at))) return "probe has no time";
  return undefined;
}
/** The first finding refused, as a sentence; undefined when every finding stands on a probe. */
export function refusal(findings: readonly Finding[]): string | undefined {
  for (const f of findings) {
    if (typeof f?.summary !== "string" || f.summary.trim() === "") return "refused: a finding says nothing";
      
        sentinelcrew
        22h ago
        outdated
      
      Return text should start with refused: like the probe case.
    
    const fault = probeFault(f.probe);
    if (fault !== undefined) return `refused: finding "${f.summary.slice(0, 80)}" has ${fault}`;
  }
  return undefined;
}
export const clip = (text: string): string => (text.length > MAX_TEXT ? `${text.slice(0, MAX_TEXT)}…` : text);
@@ -0,0 +1,6 @@
/** The duties the forge runs: one line per duty file in `duties/`. */
import type { Duty } from "./duty.ts";
import { duty as pullsWaiting } from "./duties/stale_pulls.ts";
export const duties: readonly Duty[] = [pullsWaiting];
@@ -0,0 +1,92 @@
/**
 * The duty runner (#11204): on the clock, each duty whose interval has passed since it last
 * started runs once, never twice at the same time, and leaves a `maintenance_run` row. A
 * finding without a probe is refused: the run's outcome says so and no finding is kept.
 */
import type { App } from "../app.ts";
import { during } from "../perf/jobs.ts";
import { clip, refusal, validName, type Duty } from "./duty.ts";
import { duties as registered } from "./registry.ts";
import { lastStarted, prune, record } from "./runs.ts";
export const TICK_MS = 60_000;
const FIRST_MS = 15_000;
/** A duty that has not answered in this long is recorded as timed out and the next duty runs. */
export const DUTY_TIMEOUT_MS = 30_000;
/** Rows kept per duty; older ones are deleted after each run. */
export const KEEP_PER_DUTY = 200;
export interface Maintain {
  /** One pass: every due duty runs; resolves when they have. */
  tick(now?: Date): Promise<void>;
  stop(): void;
}
export function startMaintain(app: App, options: { duties?: readonly Duty[]; everyMs?: number; timeoutMs?: number; keep?: number } = {}): Maintain {
  const list = options.duties ?? registered;
  const seen = new Set<string>();
  for (const d of list) {
    if (!validName(d.name)) throw new Error(`maintenance duty name "${d.name}" is not lowercase words and dashes`);
    if (seen.has(d.name)) throw new Error(`maintenance duty "${d.name}" is registered twice`);
    seen.add(d.name);
  }
  const timeoutMs = options.timeoutMs ?? DUTY_TIMEOUT_MS;
      
        sentinelcrew
        22h ago
        outdated
      
      No deadline on duty.run: a hung duty stalls every other duty. Add a timeout and record it as failed.
    
  const running = new Set<string>();
  async function runOne(duty: Duty, now: Date): Promise<void> {
    if (running.has(duty.name)) return;
    running.add(duty.name);
    const started = now;
    let findings: Awaited<ReturnType<Duty["run"]>>["findings"] = [];
    let actions: readonly string[] = [];
    let outcome = "ok";
    try {
      // The duty keeps its `running` slot until it really settles, so a timed-out duty never overlaps itself.
      const work = Promise.resolve().then(() => duty.run({ db: app.db, now }));
      work.then(() => running.delete(duty.name), () => running.delete(duty.name));
      let timer: ReturnType<typeof setTimeout> | undefined;
      const deadline = new Promise<never>((_, reject) => {
        timer = setTimeout(() => reject(new Error("timed out")), timeoutMs);
      });
      const result = await Promise.race([work, deadline]).finally(() => clearTimeout(timer));
      const refused = refusal(result.findings);
      if (refused !== undefined) outcome = refused;
      else {
        findings = result.findings;
        actions = result.actions.map(clip);
      }
    } catch (error) {
      outcome = clip(`failed: ${error instanceof Error ? error.message : String(error)}`);
    }
    try {
      record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome });
      prune(app.db, duty.name, options.keep ?? KEEP_PER_DUTY);
    } catch (error) {
      app.log.error("maintenance run not recorded", { duty: duty.name, error: String(error) });
    }
  }
  async function tick(now = new Date()): Promise<void> {
    for (const duty of list) {
      const last = lastStarted(app.db, duty.name);
      if (last !== undefined && now.getTime() - last < duty.intervalMs) continue;
      if (running.has(duty.name)) continue;
      await runOne(duty, now);
      // Duties take turns: the loop turns between one and the next.
      await new Promise((resolve) => setImmediate(resolve));
    }
  }
  const first = setTimeout(() => void during("maintain", () => tick()), FIRST_MS);
  const every = setInterval(() => void during("maintain", () => tick()), options.everyMs ?? TICK_MS);
  first.unref?.();
  every.unref?.();
  return {
    tick,
    stop() {
      clearTimeout(first);
      clearInterval(every);
    },
  };
}
@@ -0,0 +1,58 @@
/** `maintenance_run`: the record of each duty's runs. */
import type { Database } from "bun:sqlite";
import type { Finding } from "./duty.ts";
export interface RunRow {
  readonly id: number;
  readonly duty: string;
  readonly started: string;
  readonly finished: string;
  readonly findings: readonly unknown[];
  readonly actions: readonly string[];
  readonly probes: readonly unknown[];
  readonly outcome: string;
}
export function record(
  db: Database,
  r: { duty: string; started: Date; finished: Date; findings: readonly Finding[]; actions: readonly string[]; outcome: string },
): void {
  db.query(
    "INSERT INTO maintenance_run (duty, started, finished, findings, actions, probes, outcome) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
  ).run(
    r.duty,
    r.started.toISOString(),
    r.finished.toISOString(),
    JSON.stringify(r.findings.map((f) => f.summary)),
    JSON.stringify(r.actions),
    JSON.stringify(r.findings.flatMap((f) => (f.probe ? [f.probe] : []))),
    r.outcome,
  );
}
/** Keep a duty's newest `keep` rows; the rest go. */
export function prune(db: Database, duty: string, keep: number): void {
  db.query("DELETE FROM maintenance_run WHERE duty = ?1 AND id NOT IN (SELECT id FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT ?2)").run(duty, keep);
}
export function lastStarted(db: Database, duty: string): number | undefined {
  const row = db.query("SELECT started FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(duty) as
    | { started: string }
    | null;
  return row ? Date.parse(row.started) : undefined;
}
export const LIST_MAX = 100;
export function latest(db: Database, limit: number): RunRow[] {
  const rows = db.query("SELECT * FROM maintenance_run ORDER BY id DESC LIMIT ?1").all(Math.min(Math.max(1, limit), LIST_MAX)) as {
    id: number; duty: string; started: string; finished: string; findings: string; actions: string; probes: string; outcome: string;
  }[];
  return rows.map((r) => ({
    ...r,
    findings: JSON.parse(r.findings) as unknown[],
    actions: JSON.parse(r.actions) as string[],
    probes: JSON.parse(r.probes) as unknown[],
  }));
}
@@ -55,6 +55,7 @@
import { entry as money } from "./money.ts";
import { entry as loopStalls } from "./loopstalls.ts";
import { entry as slowPagesEntry } from "./slowpages.ts";
import { entry as maintenance } from "./maintenance.ts";
import { entry as maintainRuns } from "./maintainruns.ts";
import { entry as mirrorPacksEntry } from "./mirrorpacks.ts";
import { entry as packReposEntry } from "./packrepos.ts";
import { modelCeilingEntry, modelGrantEntry } from "./model.ts";
@@ -115,6 +116,7 @@
export const ENTRIES: readonly PaletteEntry[] = [
  telemetry,
  resilience,
  maintenance,
  maintainRuns,
  mirrorPacksEntry,
  packReposEntry,
  valves,
@@ -0,0 +1,60 @@
/**
 * The latest maintenance duty runs in the terminal (#11204): what
 * `GET /api/v1/maintenance/runs` lists, through the same read (`latest`) and
 * the same gate, a forge admin; anyone else is offered nothing and sees no
 * rows. Read-only; j/k scroll, Esc or q closes it.
 */
import { latest } from "../../maintain/runs.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { pad } from "../term/width.ts";
import { Scroll, drawLines, hitScroll } from "./common.ts";
import type { PaletteEntry } from "./entries.ts";
const mayRead = (session: Session): boolean => session.user?.isAdmin === true;
/** A header, then a run a row: duty, started, findings, probes, outcome. Empty for anyone the gate refuses. */
export function runLines(session: Session, limit = 50): string[] {
  if (!mayRead(session)) return [];
  const head = `${pad("Duty", 24)}${pad("Started (UTC)", 21)}${pad("Found", 7)}${pad("Probes", 8)}Outcome`;
  const rows = latest(session.db, limit).map(
    (r) =>
      `${pad(r.duty, 24)}${pad(r.started.slice(0, 19).replace("T", " "), 21)}${pad(String(r.findings.length), 7)}${pad(String(r.probes.length), 8)}${r.outcome}`,
  );
  return [head, ...rows];
}
export function maintainRunsScreen(session: Session, ctx: Ctx): Screen {
  const scroll = new Scroll();
  return {
    title: "Maintenance duty runs",
    keys: [binding(["j", "k"], "scroll"), binding(["esc", "q"], "back")],
    render(area, s) {
      const c = ctx.theme.c;
      const lines = runLines(session);
      const spans = lines.map((text, i) => [{ text, style: style(i === 0 ? c.fg : c.mutedFg, c.bg) }]);
      drawLines(s, area, c, scroll, spans);
      hitScroll(ctx, area, scroll, lines.length);
    },
    key(e) {
      if (scroll.key(e, runLines(session).length)) return true;
      if (e.key === "esc" || e.key === "q") {
        ctx.pop();
        return true;
      }
      return false;
    },
  };
}
export const entry: PaletteEntry = {
  id: "maintenance-runs",
  title: "Maintenance duty runs",
  hint: "what each duty found, did and probed",
  after: ["maintenance"],
  offered: mayRead,
  run: (session, ctx) => ctx.push(maintainRunsScreen(session, ctx)),
};
@@ -64,6 +64,7 @@
import { nightlyDue } from "../src/checks/scan_store.ts";
import { startNotifier } from "../src/crew/notify.ts";
import { fireDue } from "../src/apps/feature_schedule.ts";
import { pullDue } from "../src/apps/pull_schedule.ts";
import { startMaintain } from "../src/maintain/runner.ts";
/** Thousands of issues and pulls, as the issue asks; runs, comments and events thinned so the file runs in seconds (#1242). */
const BUSY = { issues: 2000, pulls: 2000, requested: 100, runs: 5000, comments: 5000, events: 10000, reviewComments: 500 };
@@ -134,6 +135,8 @@
function jobs(): Record<string, () => unknown> {
  const memory = startMemorySampler(app, 1e9);
  sampler.stop();
  memory.stop();
  const maintainer = startMaintain(app, { everyMs: 1e9 });
  maintainer.stop();
  return {
    sampler: () => sampler.tick(),
    "memory-sampler": () => memory.tick(),
@@ -221,6 +224,7 @@
function jobs(): Record<string, () => unknown> {
    "nightly-scan": () => nightlyDue(app.db, new Date(Date.now() - 24 * 3_600_000).toISOString(), 25),
    "feature-schedule": () => fireDue(app),
    "pull-schedule": () => pullDue(app),
    maintain: () => maintainer.tick(new Date(Date.now() + 1e9)),
  };
}
@@ -0,0 +1,196 @@
/** The duty runner (#11204): schedule, record, refusal of a finding without a probe, the example duty, the admin list. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { duties } from "../src/maintain/registry.ts";
import { startMaintain } from "../src/maintain/runner.ts";
import { latest } from "../src/maintain/runs.ts";
import type { Duty } from "../src/maintain/duty.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
let root = "";
let app: App;
let servers: Servers;
let admin = "";
let bob = "";
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-maintain-"));
  app = createApp({ dataDir: root, port: 0 });
  servers = await serve(app);
  const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
  if (!a.ok || !b.ok) throw new Error("user");
  admin = createToken(app.db, a.value.id, "ro", "ro").token;
  bob = createToken(app.db, b.value.id, "ro", "ro").token;
});
afterAll(() => {
  servers.stop?.();
  rmSync(root, { recursive: true, force: true });
});
const probe = { kind: "measurement", what: "queue depth", number: 3, at: "2026-11-11T00:00:00.000Z" } as const;
const clear = () => app.db.exec("DELETE FROM maintenance_run");
test("a duty runs when due, not before its interval, and leaves a row with findings and probes", async () => {
  clear();
  let calls = 0;
  const duty: Duty = { name: "t-sched", intervalMs: 1000, run: () => (calls++, { findings: [{ summary: "deep queue", probe }], actions: ["noted"] }) };
  const m = startMaintain(app, { duties: [duty], everyMs: 1e9 });
  m.stop();
  const t0 = new Date("2026-11-11T00:00:00Z");
  await m.tick(t0);
  await m.tick(new Date(t0.getTime() + 500));
  expect(calls).toBe(1);
  await m.tick(new Date(t0.getTime() + 1500));
  expect(calls).toBe(2);
  const [row] = latest(app.db, 1);
  expect(row).toMatchObject({ duty: "t-sched", outcome: "ok", findings: ["deep queue"], actions: ["noted"], probes: [probe] });
  expect(row!.started).toBe("2026-11-11T00:00:01.500Z");
  expect(Date.parse(row!.finished)).toBeGreaterThan(0);
});
test("a duty never runs twice at once", async () => {
  clear();
  let live = 0;
  let worst = 0;
  const duty: Duty = {
    name: "t-once", intervalMs: 0,
    run: async () => { live++; worst = Math.max(worst, live); await new Promise((r) => setTimeout(r, 20)); live--; return { findings: [], actions: [] }; },
  };
  const m = startMaintain(app, { duties: [duty], everyMs: 1e9 });
  m.stop();
  await Promise.all([m.tick(), m.tick()]);
  expect(worst).toBe(1);
});
test("a finding without a probe is refused and the outcome says so", async () => {
  clear();
  const bad: Duty = { name: "t-bad", intervalMs: 1, run: () => ({ findings: [{ summary: "it is slow" }], actions: ["did a thing"] }) };
  const nan: Duty = { name: "t-nan", intervalMs: 1, run: () => ({ findings: [{ summary: "x", probe: { ...probe, number: NaN } }], actions: [] }) };
  const m = startMaintain(app, { duties: [bad, nan], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const rows = latest(app.db, 5);
  const byName = Object.fromEntries(rows.map((r) => [r.duty, r]));
  expect(byName["t-bad"]!.outcome).toContain("refused");
  expect(byName["t-bad"]!.outcome).toContain("no probe");
  expect(byName["t-bad"]!.findings).toEqual([]);
  expect(byName["t-bad"]!.actions).toEqual([]);
  expect(byName["t-nan"]!.outcome).toContain("probe has no number");
});
test("a duty that throws is recorded as failed and the next still runs", async () => {
  clear();
  const boom: Duty = { name: "t-boom", intervalMs: 1, run: () => { throw new Error("no"); } };
  const fine: Duty = { name: "t-fine", intervalMs: 1, run: () => ({ findings: [], actions: [] }) };
  const m = startMaintain(app, { duties: [boom, fine], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const rows = latest(app.db, 5);
  expect(rows.find((r) => r.duty === "t-boom")!.outcome).toBe("failed: no");
  expect(rows.find((r) => r.duty === "t-fine")!.outcome).toBe("ok");
});
test("the example duty is registered and counts open pulls waiting over 24 hours, with its probe", async () => {
  clear();
  expect(duties.map((d) => d.name)).toContain("pulls-waiting");
  const now = new Date("2026-11-11T12:00:00Z");
  app.db.exec("PRAGMA foreign_keys = OFF");
  const ins = app.db.prepare("INSERT INTO issue (id, repo_id, number, kind, title, author_id, state, created_at) VALUES (?1, 'r', ?2, 'pull', 't', 'u', ?3, ?4)");
  ins.run("p1", 1, "open", "2026-11-09T00:00:00.000Z");
  ins.run("p2", 2, "open", "2026-11-11T11:00:00.000Z");
  ins.run("p3", 3, "closed", "2026-11-01T00:00:00.000Z");
  const m = startMaintain(app, { duties: duties.filter((d) => d.name === "pulls-waiting"), everyMs: 1e9 });
  m.stop();
  await m.tick(now);
  const [row] = latest(app.db, 1);
  expect(row!.outcome).toBe("ok");
  expect(row!.findings).toEqual(["1 open pull waiting over 24 hours"]);
  expect(row!.probes[0]).toMatchObject({ kind: "query", number: 1, at: now.toISOString() });
  app.db.exec("DELETE FROM issue WHERE id IN ('p1','p2','p3')");
});
test("the example duty's count searches an existing index, not the table", () => {
  const plan = (app.db.query("EXPLAIN QUERY PLAN SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1").all("x") as { detail: string }[]).map((r) => r.detail).join("\n");
  expect(plan).toMatch(/USING (COVERING )?INDEX/);
  expect(plan).not.toMatch(/SCAN|TEMP B-TREE/);
});
test("GET /api/v1/maintenance/runs lists the latest rows for an admin only", async () => {
  clear();
  const m = startMaintain(app, { duties: [{ name: "t-list", intervalMs: 1, run: () => ({ findings: [], actions: [] }) }], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const get = (token?: string, q = "") => fetch(`${servers.url}/api/v1/maintenance/runs${q}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
  expect((await get()).status).toBe(401);
  expect((await get(bob)).status).toBe(403);
  const res = await get(admin, "?limit=5");
  expect(res.status).toBe(200);
  const body = (await res.json()) as { runs: { duty: string; outcome: string }[] };
  expect(body.runs[0]).toMatchObject({ duty: "t-list", outcome: "ok" });
  expect((await get(admin, "?limit=abc")).status).toBe(200);
});
test("a hung duty is recorded as timed out and the next duty still runs, without overlapping itself", async () => {
  clear();
  let release = () => {};
  let starts = 0;
  const hung: Duty = { name: "t-hung", intervalMs: 1, run: () => { starts++; return new Promise((r) => { release = () => r({ findings: [], actions: [] }); }); } };
  const fine: Duty = { name: "t-after", intervalMs: 1, run: () => ({ findings: [], actions: [] }) };
  const m = startMaintain(app, { duties: [hung, fine], everyMs: 1e9, timeoutMs: 20 });
  m.stop();
  await m.tick();
  const rows = latest(app.db, 5);
  expect(rows.find((r) => r.duty === "t-hung")!.outcome).toBe("failed: timed out");
  expect(rows.find((r) => r.duty === "t-after")!.outcome).toBe("ok");
  await m.tick(new Date(Date.now() + 10));
  expect(starts).toBe(1);
  release();
});
test("an empty finding reads refused:", async () => {
  clear();
  const d: Duty = { name: "t-empty", intervalMs: 1, run: () => ({ findings: [{ summary: " ", probe }], actions: [] }) };
  const m = startMaintain(app, { duties: [d], everyMs: 1e9 });
  m.stop();
  await m.tick();
  expect(latest(app.db, 1)[0]!.outcome).toBe("refused: a finding says nothing");
});
test("old rows are pruned to a cap per duty, and other duties keep theirs", async () => {
  clear();
  const a: Duty = { name: "t-a", intervalMs: 1, run: () => ({ findings: [], actions: [] }) };
  const b: Duty = { name: "t-b", intervalMs: 1, run: () => ({ findings: [], actions: [] }) };
  const m = startMaintain(app, { duties: [a, b], everyMs: 1e9, keep: 3 });
  m.stop();
  for (let i = 0; i < 6; i++) await m.tick(new Date(Date.now() + i * 10));
  const rows = latest(app.db, 100);
  expect(rows.filter((r) => r.duty === "t-a")).toHaveLength(3);
  expect(rows.filter((r) => r.duty === "t-b")).toHaveLength(3);
});
test("a duty name that is not lowercase words and dashes, or is registered twice, is refused at start", () => {
  const run = () => ({ findings: [], actions: [] });
  expect(() => startMaintain(app, { duties: [{ name: "Bad Name", intervalMs: 1, run }], everyMs: 1e9 })).toThrow("not lowercase");
  expect(() => startMaintain(app, { duties: [{ name: "dup", intervalMs: 1, run }, { name: "dup", intervalMs: 1, run }], everyMs: 1e9 })).toThrow("twice");
});
test("the terminal screen lists runs for an admin and nothing for anyone else", async () => {
  clear();
  const m = startMaintain(app, { duties: [{ name: "t-tui", intervalMs: 1, run: () => ({ findings: [], actions: [] }) }], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const { runLines } = await import("../src/tui/screens/maintainruns.ts");
  const as = (isAdmin: boolean) => ({ user: { isAdmin }, db: app.db }) as never;
  expect(runLines(as(true)).some((l) => l.startsWith("t-tui"))).toBe(true);
  expect(runLines(as(false))).toEqual([]);
  expect(runLines({ user: undefined, db: app.db } as never)).toEqual([]);
});
@@ -117,6 +117,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["capacity", "Capacity", "concurrency and guards, live; e changes a settable one"],
    ["resilience", "Resilience", "where snapshots go and the mirror; each protection green, red or untested"],
    ["maintenance", "Maintenance", "prepare the forge for a power-off, check the drain, or end maintenance"],
    ["maintenance-runs", "Maintenance duty runs", "what each duty found, did and probed"],
    ["crawlers", "Crawlers", "their share of answering time, and who was refused"],
    ["scrapes", "Scrapes", "what was read by repository, and groups in a running capture"],
    ["traffic", "Traffic", "who is using the forge's answering time, and the top readers"],
@@ -250,6 +251,7 @@
test("a site-admin agent is offered the roster without the person-only entries",
    ["proposal-tokens", "Proposal tokens", "mint, list and revoke what a child proposes with"],
    ["can", "What I can do", "your role and what it allows here"],
    ["bugs", "Bugs", "open bugs and the bug clock"],
    ["maintenance-runs", "Maintenance duty runs", "what each duty found, did and probed"],
    ["traffic-sources", "Traffic sources", "where people come from and where they go, for the forge"],
    ["audit", "Who can do what", "each app's doors from its contract.json: yes, own or no"],
    ["archived", "Archived repositories", "put away, not gone: open one or restore it"],
		@@ -0,0 +1,13 @@
	1	# Maintenance duties (#11204)
	2
	3	A duty is one file in `src/maintain/duties/`: a `name`, an `intervalMs` and a `run` that returns `findings` and `actions`. Register it with one line in `src/maintain/registry.ts`.
	4
	5	Every minute the runner (`src/maintain/runner.ts`, inside `during("maintain")`) runs each duty whose interval has passed since it last started, never two runs of one duty at once, and writes a `maintenance_run` row: duty, started, finished, findings, actions, probes, outcome.
	6
	7	A finding must carry a probe: `{ kind: "query" \| "measurement", what, number, at }`. A finding without one is refused: the run keeps no findings or actions and its outcome reads `refused: finding "…" has no probe`. A duty that throws reads `failed: <message>`.
	8
	9	`GET /api/v1/maintenance/runs?limit=20` (forge admin, `ro` scope) lists the latest rows. The shipped duty, `pulls-waiting`, counts open pulls older than 24 hours.
	10
	11	The terminal door is the palette entry "Maintenance duty runs" (`src/tui/screens/maintainruns.ts`): the same rows through the same read, offered only to a forge admin; anyone else is offered nothing and sees no rows.
	12
	13	Each run is bounded: a duty that has not answered in 30 seconds reads `failed: timed out` and the next duty runs (the late duty keeps its slot until it settles, so it never overlaps itself). A duty name must be lowercase words and dashes and unique, or the runner refuses to start. Each duty keeps its newest 200 rows; older ones are deleted after each run. A finding that says nothing reads `refused: a finding says nothing`. A non-admin's token reads 403 `forbidden`.
		@@ -0,0 +1,11 @@
	1	import type { Route } from "../shared.ts";
	2
	3	export const route: Route = {
	4	method: "get",
	5	path: "/api/v1/maintenance/runs",
	6	summary:
	7	"The latest maintenance duty runs, newest first, each with its findings, actions and probes (a forge admin's token; ?limit= up to 100); answers { runs }",
	8	auth: true,
	9	role: "admin",
	10	refusals: { forbidden: "the token is not a forge admin" },
	11	};
		@@ -102,6 +102,7 @@ export { route as getApiV1Front } from "./getApiV1Front.ts";
102	102	export { route as getApiV1Health } from "./getApiV1Health.ts";
103	103	export { route as getApiV1Here } from "./getApiV1Here.ts";
104	104	export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
	105	export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
105	106	export { route as getApiV1Map } from "./getApiV1Map.ts";
106	107	export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
107	108	export { route as getApiV1MigrateJobStream } from "./getApiV1MigrateJobStream.ts";
		@@ -52,6 +52,7 @@ const ALIAS: Readonly<Record<string, string>> = {
52	52	heartbeat: "meta",
53	53	telemetry: "meta",
54	54	drills: "meta",
	55	maintenance: "meta",
55	56	devices: "meta",
56	57	thoughts: "feed",
57	58	share: "feed",
		@@ -0,0 +1,1 @@
	1	c7266eda3263e87bc4570f7dd1403080b279857caf3f4fb5a21414509bf13fa5
		@@ -0,0 +1,19 @@
	1	// Maintenance runs (#11204): one row per run of a maintenance duty, with what
	2	// it found, what it did and the probe behind each finding. Additive.
	3	// The index `maintenance_run_duty` serves "when did this duty last start" and
	4	// the pruning of a duty's old rows; the primary key serves the latest-first
	5	// list. The example duty adds no index to `issue`: a new one there changes
	6	// the planner's choice for the cold reads.
	7	export default `
	8	CREATE TABLE maintenance_run (
	9	id INTEGER PRIMARY KEY AUTOINCREMENT,
	10	duty TEXT NOT NULL,
	11	started TEXT NOT NULL,
	12	finished TEXT NOT NULL,
	13	findings TEXT NOT NULL DEFAULT '[]',
	14	actions TEXT NOT NULL DEFAULT '[]',
	15	probes TEXT NOT NULL DEFAULT '[]',
	16	outcome TEXT NOT NULL
	17	);
	18	CREATE INDEX maintenance_run_duty ON maintenance_run (duty, id);
	19	`;
		@@ -369,6 +369,7 @@ import { businessPlanRoutes } from "./api/business_plan.ts";
369	369	import { tokenRoutes } from "./api/tokens.ts";
370	370	import { accountAdmin, adminRoutes } from "./api/admin.ts";
371	371	import { maintenanceRoutes } from "./api/maintenance.ts";
	372	import { maintenanceRunsRoutes } from "./api/maintenance_runs.ts";
372	373	import { diskReadingRoutes } from "./api/disk_readings.ts";
373	374	import { TEMPLATE_ERRORS } from "../apps/template_tip.ts";
374	375	import { flagsRoutes } from "./api/flags.ts";
		@@ -1260,6 +1261,7 @@ export function apiRoutes(router: Router<Handler>): void {
1260	1261	forgeOriginRoutes(router);
1261	1262	adminRoutes(router);
1262	1263	maintenanceRoutes(router);
	1264	maintenanceRunsRoutes(router);
1263	1265	diskReadingRoutes(router);
1264	1266	flagsRoutes(router);
1265	1267	appTemplateRoutes(router);
		@@ -0,0 +1,20 @@
	1	/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
	2
	3	import { latest } from "../../maintain/runs.ts";
	4	import type { Handler } from "../context.ts";
	5	import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
	6	import { forbidden, json, requireScope, unauthorized } from "../respond.ts";
	7	import type { Router } from "../router.ts";
	8
	9	export function maintenanceRunsRoutes(router: Router<Handler>): void {
	10	router.add("GET", "/api/v1/maintenance/runs", (ctx) => {
	11	notePhase(ctx.request, "session", takeSessionTime(ctx.request));
	12	if (!ctx.auth.user) return unauthorized();
	13	if (!ctx.auth.user.isAdmin) return forbidden();
		sentinelcrew 22h ago outdated Non-admin gets 404 while the contract declares `forbidden`/`member`. Align them.
	14	const denied = requireScope(ctx, "ro");
	15	if (denied) return denied;
	16	const asked = Number(ctx.url.searchParams.get("limit") ?? 20);
	17	const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
	18	return json({ runs: latest(ctx.app.db, limit) });
	19	});
	20	}
		@@ -59,6 +59,7 @@ import { VERIFY_EVERY_MS } from "./checks/verify.ts";
59	59	import { RED_HOLD_MS } from "./checks/red_main.ts";
60	60	import { stopQueue } from "./checks/queue.ts";
61	61	import { startLooks } from "./looks/service.ts";
	62	import { startMaintain } from "./maintain/runner.ts";
62	63	import { startMirrorOut } from "./origins/mirror-out.ts";
63	64	import { startMirrorPulls } from "./origins/mirror-pulls.ts";
64	65	import { startProbes } from "./contract/probes.ts";
		@@ -501,6 +502,8 @@ const mirrorPulls = startMirrorPulls(app);
501	502	// The look: a picture of every environment that comes up green, taken
502	503	// through this forge's own listener.
503	504	const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined;
	505	// Maintenance duties (src/maintain): each runs on its interval and leaves a row.
	506	const maintain = startMaintain(app);
504	507	// Is it so: the forge knocks on every door of an app with a contract after
505	508	// each production deploy and once a night, and compares with the table.
506	509	const probes = apps === undefined ? undefined : startProbes(app, apps);
		@@ -953,6 +956,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
953	956	mirrorOut(),
954	957	mirrorPulls(),
955	958	looks?.stop(),
	959	maintain.stop(),
956	960	probes?.stop(),
957	961	nightly?.stop(),
958	962	snapshots.stop(),
		@@ -0,0 +1,25 @@
	1	/** Example duty (#11204): how many open pulls have waited over 24 hours. Reads only. */
	2
	3	import type { Duty } from "../duty.ts";
	4
	5	const DAY_MS = 24 * 3_600_000;
	6	const SQL = "SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1";
	7
	8	export const duty: Duty = {
	9	name: "pulls-waiting",
	10	intervalMs: 3_600_000,
	11	run({ db, now }) {
	12	const cutoff = new Date(now.getTime() - DAY_MS).toISOString();
	13	const n = (db.query(SQL).get(cutoff) as { n: number }).n;
	14	if (n === 0) return { findings: [], actions: [] };
	15	return {
	16	findings: [
	17	{
	18	summary: `${n} open pull${n === 1 ? "" : "s"} waiting over 24 hours`,
	19	probe: { kind: "query", what: SQL.replace("?1", `'${cutoff}'`), number: n, at: now.toISOString() },
	20	},
	21	],
	22	actions: [],
	23	};
	24	},
	25	};