f7ea4d8 · e45070f · 10b84d0
@@ -0,0 +1,9 @@
# Maintenance duties (#11204)
A duty is one file in `src/maintain/duties/`: a `name`, an `intervalMs` and a `run` that returns `findings` and `actions`. Register it with one line in `src/maintain/registry.ts`.
Every minute the runner (`src/maintain/runner.ts`, inside `during("maintain")`) runs each duty whose interval has passed since it last started, never two runs of one duty at once, and writes a `maintenance_run` row: duty, started, finished, findings, actions, probes, outcome.
A finding must carry a probe: `{ kind: "query" | "measurement", what, number, at }`. A finding without one is refused: the run keeps no findings or actions and its outcome reads `refused: finding "…" has no probe`. A duty that throws reads `failed: <message>`.
`GET /api/v1/maintenance/runs?limit=20` (forge admin, `ro` scope) lists the latest rows. The shipped duty, `pulls-waiting`, counts open pulls older than 24 hours.
@@ -0,0 +1,11 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "get",
  path: "/api/v1/maintenance/runs",
  summary:
    "The latest maintenance duty runs, newest first, each with its findings, actions and probes (a forge admin's token; ?limit= up to 100); answers { runs }",
  auth: true,
  role: "member",
  refusals: { forbidden: "the token is not a person who administers the forge" },
};
@@ -102,6 +102,7 @@
export { route as getApiV1Front } from "./getApiV1Front.ts";
export { route as getApiV1Health } from "./getApiV1Health.ts";
export { route as getApiV1Here } from "./getApiV1Here.ts";
export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
export { route as getApiV1Map } from "./getApiV1Map.ts";
export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
export { route as getApiV1MigrateJobStream } from "./getApiV1MigrateJobStream.ts";
@@ -0,0 +1,1 @@
c7266eda3263e87bc4570f7dd1403080b279857caf3f4fb5a21414509bf13fa5
@@ -0,0 +1,18 @@
// Maintenance runs (#11204): one row per run of a maintenance duty, with what
// it found, what it did and the probe behind each finding. Additive.
// `maintenance_run_duty` serves "when did this duty last start" and
// the primary key the latest-first list. The example duty adds no index to
// `issue`: a new one there changes the planner's choice for the cold reads.
export default `
  CREATE TABLE maintenance_run (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    duty TEXT NOT NULL,
    started TEXT NOT NULL,
    finished TEXT NOT NULL,
    findings TEXT NOT NULL DEFAULT '[]',
    actions TEXT NOT NULL DEFAULT '[]',
    probes TEXT NOT NULL DEFAULT '[]',
    outcome TEXT NOT NULL
  );
  CREATE INDEX maintenance_run_duty ON maintenance_run (duty, id);
  `;
@@ -369,6 +369,7 @@
import { businessPlanRoutes } from "./api/business_plan.ts";
import { tokenRoutes } from "./api/tokens.ts";
import { accountAdmin, adminRoutes } from "./api/admin.ts";
import { maintenanceRoutes } from "./api/maintenance.ts";
import { maintenanceRunsRoutes } from "./api/maintenance_runs.ts";
import { diskReadingRoutes } from "./api/disk_readings.ts";
import { TEMPLATE_ERRORS } from "../apps/template_tip.ts";
import { flagsRoutes } from "./api/flags.ts";
@@ -1260,6 +1261,7 @@
export function apiRoutes(router: Router<Handler>): void {
  forgeOriginRoutes(router);
  adminRoutes(router);
  maintenanceRoutes(router);
  maintenanceRunsRoutes(router);
  diskReadingRoutes(router);
  flagsRoutes(router);
  appTemplateRoutes(router);
@@ -0,0 +1,20 @@
/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
import { latest } from "../../maintain/runs.ts";
import type { Handler } from "../context.ts";
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { json, notFound, requireScope, unauthorized } from "../respond.ts";
import type { Router } from "../router.ts";
export function maintenanceRunsRoutes(router: Router<Handler>): void {
  router.add("GET", "/api/v1/maintenance/runs", (ctx) => {
    notePhase(ctx.request, "session", takeSessionTime(ctx.request));
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return notFound();
    const denied = requireScope(ctx, "ro");
    if (denied) return denied;
    const asked = Number(ctx.url.searchParams.get("limit") ?? 20);
    const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
    return json({ runs: latest(ctx.app.db, limit) });
  });
}
@@ -59,6 +59,7 @@
import { VERIFY_EVERY_MS } from "./checks/verify.ts";
import { RED_HOLD_MS } from "./checks/red_main.ts";
import { stopQueue } from "./checks/queue.ts";
import { startLooks } from "./looks/service.ts";
import { startMaintain } from "./maintain/runner.ts";
import { startMirrorOut } from "./origins/mirror-out.ts";
import { startMirrorPulls } from "./origins/mirror-pulls.ts";
import { startProbes } from "./contract/probes.ts";
@@ -501,6 +502,8 @@
const mirrorPulls = startMirrorPulls(app);
// The look: a picture of every environment that comes up green, taken
// through this forge's own listener.
const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined;
// Maintenance duties (src/maintain): each runs on its interval and leaves a row.
const maintain = startMaintain(app);
// Is it so: the forge knocks on every door of an app with a contract after
// each production deploy and once a night, and compares with the table.
const probes = apps === undefined ? undefined : startProbes(app, apps);
@@ -953,6 +956,7 @@
function drain(signal: NodeJS.Signals): Promise<unknown> {
    mirrorOut(),
    mirrorPulls(),
    looks?.stop(),
    maintain.stop(),
    probes?.stop(),
    nightly?.stop(),
    snapshots.stop(),
@@ -0,0 +1,25 @@
/** Example duty (#11204): how many open pulls have waited over 24 hours. Reads only. */
import type { Duty } from "../duty.ts";
const DAY_MS = 24 * 3_600_000;
const SQL = "SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1";
export const duty: Duty = {
  name: "pulls-waiting",
  intervalMs: 3_600_000,
  run({ db, now }) {
    const cutoff = new Date(now.getTime() - DAY_MS).toISOString();
    const n = (db.query(SQL).get(cutoff) as { n: number }).n;
    if (n === 0) return { findings: [], actions: [] };
    return {
      findings: [
        {
          summary: `${n} open pull${n === 1 ? "" : "s"} waiting over 24 hours`,
          probe: { kind: "query", what: SQL.replace("?1", `'${cutoff}'`), number: n, at: now.toISOString() },
        },
      ],
      actions: [],
    };
  },
};
@@ -0,0 +1,63 @@
/** What a maintenance duty is (#11204): a name, an interval and a handler; plain data in, plain data out. */
import type { Database } from "bun:sqlite";
/** The number a finding rests on and when it was read: a query's result or a measurement. */
export interface Probe {
  readonly kind: "query" | "measurement";
  /** The SQL, or what was measured. */
  readonly what: string;
  readonly number: number;
  /** ISO time the number was read. */
  readonly at: string;
}
export interface Finding {
  readonly summary: string;
  readonly probe?: Probe | undefined;
}
export interface DutyResult {
  readonly findings: readonly Finding[];
  /** What the duty did about it, one line each. */
  readonly actions: readonly string[];
}
export interface DutyContext {
  readonly db: Database;
  readonly now: Date;
}
export interface Duty {
  /** Lowercase words and dashes; the row's `duty`. */
  readonly name: string;
  readonly intervalMs: number;
  readonly run: (ctx: DutyContext) => DutyResult | Promise<DutyResult>;
}
const NAME = /^[a-z][a-z0-9-]{0,63}$/;
const MAX_TEXT = 500;
export const validName = (name: string): boolean => NAME.test(name);
/** Why a probe is no probe, or undefined when it is one. */
export function probeFault(p: Probe | undefined): string | undefined {
  if (p === undefined || p === null) return "no probe";
  if (p.kind !== "query" && p.kind !== "measurement") return "probe kind is neither query nor measurement";
  if (typeof p.what !== "string" || p.what.trim() === "") return "probe says nothing of what was read";
  if (typeof p.number !== "number" || !Number.isFinite(p.number)) return "probe has no number";
  if (typeof p.at !== "string" || Number.isNaN(Date.parse(p.at))) return "probe has no time";
  return undefined;
}
/** The first finding refused, as a sentence; undefined when every finding stands on a probe. */
export function refusal(findings: readonly Finding[]): string | undefined {
  for (const f of findings) {
    if (typeof f?.summary !== "string" || f.summary.trim() === "") return "a finding says nothing";
    const fault = probeFault(f.probe);
    if (fault !== undefined) return `refused: finding "${f.summary.slice(0, 80)}" has ${fault}`;
  }
  return undefined;
}
export const clip = (text: string): string => (text.length > MAX_TEXT ? `${text.slice(0, MAX_TEXT)}…` : text);
@@ -0,0 +1,6 @@
/** The duties the forge runs: one line per duty file in `duties/`. */
import type { Duty } from "./duty.ts";
import { duty as pullsWaiting } from "./duties/stale_pulls.ts";
export const duties: readonly Duty[] = [pullsWaiting];
@@ -0,0 +1,75 @@
/**
 * The duty runner (#11204): on the clock, each duty whose interval has passed since it last
 * started runs once, never twice at the same time, and leaves a `maintenance_run` row. A
 * finding without a probe is refused: the run's outcome says so and no finding is kept.
 */
import type { App } from "../app.ts";
import { during } from "../perf/jobs.ts";
import { clip, refusal, type Duty } from "./duty.ts";
import { duties as registered } from "./registry.ts";
import { lastStarted, record } from "./runs.ts";
export const TICK_MS = 60_000;
const FIRST_MS = 15_000;
export interface Maintain {
  /** One pass: every due duty runs; resolves when they have. */
  tick(now?: Date): Promise<void>;
  stop(): void;
}
export function startMaintain(app: App, options: { duties?: readonly Duty[]; everyMs?: number } = {}): Maintain {
  const list = options.duties ?? registered;
  const running = new Set<string>();
  async function runOne(duty: Duty, now: Date): Promise<void> {
    if (running.has(duty.name)) return;
    running.add(duty.name);
    const started = now;
    let findings: Awaited<ReturnType<Duty["run"]>>["findings"] = [];
    let actions: readonly string[] = [];
    let outcome = "ok";
    try {
      const result = await duty.run({ db: app.db, now });
      const refused = refusal(result.findings);
      if (refused !== undefined) outcome = refused;
      else {
        findings = result.findings;
        actions = result.actions.map(clip);
      }
    } catch (error) {
      outcome = clip(`failed: ${error instanceof Error ? error.message : String(error)}`);
    } finally {
      running.delete(duty.name);
    }
    try {
      record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome });
    } catch (error) {
      app.log.error("maintenance run not recorded", { duty: duty.name, error: String(error) });
    }
  }
  async function tick(now = new Date()): Promise<void> {
    for (const duty of list) {
      const last = lastStarted(app.db, duty.name);
      if (last !== undefined && now.getTime() - last < duty.intervalMs) continue;
      if (running.has(duty.name)) continue;
      await runOne(duty, now);
      // Duties take turns: the loop turns between one and the next.
      await new Promise((resolve) => setImmediate(resolve));
    }
  }
  const first = setTimeout(() => void during("maintain", () => tick()), FIRST_MS);
  const every = setInterval(() => void during("maintain", () => tick()), options.everyMs ?? TICK_MS);
  first.unref?.();
  every.unref?.();
  return {
    tick,
    stop() {
      clearTimeout(first);
      clearInterval(every);
    },
  };
}
@@ -0,0 +1,53 @@
/** `maintenance_run`: the record of each duty's runs. */
import type { Database } from "bun:sqlite";
import type { Finding } from "./duty.ts";
export interface RunRow {
  readonly id: number;
  readonly duty: string;
  readonly started: string;
  readonly finished: string;
  readonly findings: readonly unknown[];
  readonly actions: readonly string[];
  readonly probes: readonly unknown[];
  readonly outcome: string;
}
export function record(
  db: Database,
  r: { duty: string; started: Date; finished: Date; findings: readonly Finding[]; actions: readonly string[]; outcome: string },
): void {
  db.query(
    "INSERT INTO maintenance_run (duty, started, finished, findings, actions, probes, outcome) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
  ).run(
    r.duty,
    r.started.toISOString(),
    r.finished.toISOString(),
    JSON.stringify(r.findings.map((f) => f.summary)),
    JSON.stringify(r.actions),
    JSON.stringify(r.findings.flatMap((f) => (f.probe ? [f.probe] : []))),
    r.outcome,
  );
}
export function lastStarted(db: Database, duty: string): number | undefined {
  const row = db.query("SELECT started FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(duty) as
    | { started: string }
    | null;
  return row ? Date.parse(row.started) : undefined;
}
export const LIST_MAX = 100;
export function latest(db: Database, limit: number): RunRow[] {
  const rows = db.query("SELECT * FROM maintenance_run ORDER BY id DESC LIMIT ?1").all(Math.min(Math.max(1, limit), LIST_MAX)) as {
    id: number; duty: string; started: string; finished: string; findings: string; actions: string; probes: string; outcome: string;
  }[];
  return rows.map((r) => ({
    ...r,
    findings: JSON.parse(r.findings) as unknown[],
    actions: JSON.parse(r.actions) as string[],
    probes: JSON.parse(r.probes) as unknown[],
  }));
}
@@ -64,6 +64,7 @@
import { nightlyDue } from "../src/checks/scan_store.ts";
import { startNotifier } from "../src/crew/notify.ts";
import { fireDue } from "../src/apps/feature_schedule.ts";
import { pullDue } from "../src/apps/pull_schedule.ts";
import { startMaintain } from "../src/maintain/runner.ts";
/** Thousands of issues and pulls, as the issue asks; runs, comments and events thinned so the file runs in seconds (#1242). */
const BUSY = { issues: 2000, pulls: 2000, requested: 100, runs: 5000, comments: 5000, events: 10000, reviewComments: 500 };
@@ -134,6 +135,8 @@
function jobs(): Record<string, () => unknown> {
  const memory = startMemorySampler(app, 1e9);
  sampler.stop();
  memory.stop();
  const maintainer = startMaintain(app, { everyMs: 1e9 });
  maintainer.stop();
  return {
    sampler: () => sampler.tick(),
    "memory-sampler": () => memory.tick(),
@@ -221,6 +224,7 @@
function jobs(): Record<string, () => unknown> {
    "nightly-scan": () => nightlyDue(app.db, new Date(Date.now() - 24 * 3_600_000).toISOString(), 25),
    "feature-schedule": () => fireDue(app),
    "pull-schedule": () => pullDue(app),
    maintain: () => maintainer.tick(new Date(Date.now() + 1e9)),
  };
}
@@ -0,0 +1,140 @@
/** The duty runner (#11204): schedule, record, refusal of a finding without a probe, the example duty, the admin list. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { duties } from "../src/maintain/registry.ts";
import { startMaintain } from "../src/maintain/runner.ts";
import { latest } from "../src/maintain/runs.ts";
import type { Duty } from "../src/maintain/duty.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
let root = "";
let app: App;
let servers: Servers;
let admin = "";
let bob = "";
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-maintain-"));
  app = createApp({ dataDir: root, port: 0 });
  servers = await serve(app);
  const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
  if (!a.ok || !b.ok) throw new Error("user");
  admin = createToken(app.db, a.value.id, "ro", "ro").token;
  bob = createToken(app.db, b.value.id, "ro", "ro").token;
});
afterAll(() => {
  servers.stop?.();
  rmSync(root, { recursive: true, force: true });
});
const probe = { kind: "measurement", what: "queue depth", number: 3, at: "2026-11-11T00:00:00.000Z" } as const;
const clear = () => app.db.exec("DELETE FROM maintenance_run");
test("a duty runs when due, not before its interval, and leaves a row with findings and probes", async () => {
  clear();
  let calls = 0;
  const duty: Duty = { name: "t-sched", intervalMs: 1000, run: () => (calls++, { findings: [{ summary: "deep queue", probe }], actions: ["noted"] }) };
  const m = startMaintain(app, { duties: [duty], everyMs: 1e9 });
  m.stop();
  const t0 = new Date("2026-11-11T00:00:00Z");
  await m.tick(t0);
  await m.tick(new Date(t0.getTime() + 500));
  expect(calls).toBe(1);
  await m.tick(new Date(t0.getTime() + 1500));
  expect(calls).toBe(2);
  const [row] = latest(app.db, 1);
  expect(row).toMatchObject({ duty: "t-sched", outcome: "ok", findings: ["deep queue"], actions: ["noted"], probes: [probe] });
  expect(row!.started).toBe("2026-11-11T00:00:01.500Z");
  expect(Date.parse(row!.finished)).toBeGreaterThan(0);
});
test("a duty never runs twice at once", async () => {
  clear();
  let live = 0;
  let worst = 0;
  const duty: Duty = {
    name: "t-once", intervalMs: 0,
    run: async () => { live++; worst = Math.max(worst, live); await new Promise((r) => setTimeout(r, 20)); live--; return { findings: [], actions: [] }; },
  };
  const m = startMaintain(app, { duties: [duty], everyMs: 1e9 });
  m.stop();
  await Promise.all([m.tick(), m.tick()]);
  expect(worst).toBe(1);
});
test("a finding without a probe is refused and the outcome says so", async () => {
  clear();
  const bad: Duty = { name: "t-bad", intervalMs: 1, run: () => ({ findings: [{ summary: "it is slow" }], actions: ["did a thing"] }) };
  const nan: Duty = { name: "t-nan", intervalMs: 1, run: () => ({ findings: [{ summary: "x", probe: { ...probe, number: NaN } }], actions: [] }) };
  const m = startMaintain(app, { duties: [bad, nan], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const rows = latest(app.db, 5);
  const byName = Object.fromEntries(rows.map((r) => [r.duty, r]));
  expect(byName["t-bad"]!.outcome).toContain("refused");
  expect(byName["t-bad"]!.outcome).toContain("no probe");
  expect(byName["t-bad"]!.findings).toEqual([]);
  expect(byName["t-bad"]!.actions).toEqual([]);
  expect(byName["t-nan"]!.outcome).toContain("probe has no number");
});
test("a duty that throws is recorded as failed and the next still runs", async () => {
  clear();
  const boom: Duty = { name: "t-boom", intervalMs: 1, run: () => { throw new Error("no"); } };
  const fine: Duty = { name: "t-fine", intervalMs: 1, run: () => ({ findings: [], actions: [] }) };
  const m = startMaintain(app, { duties: [boom, fine], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const rows = latest(app.db, 5);
  expect(rows.find((r) => r.duty === "t-boom")!.outcome).toBe("failed: no");
  expect(rows.find((r) => r.duty === "t-fine")!.outcome).toBe("ok");
});
test("the example duty is registered and counts open pulls waiting over 24 hours, with its probe", async () => {
  clear();
  expect(duties.map((d) => d.name)).toContain("pulls-waiting");
  const now = new Date("2026-11-11T12:00:00Z");
  app.db.exec("PRAGMA foreign_keys = OFF");
  const ins = app.db.prepare("INSERT INTO issue (id, repo_id, number, kind, title, author_id, state, created_at) VALUES (?1, 'r', ?2, 'pull', 't', 'u', ?3, ?4)");
  ins.run("p1", 1, "open", "2026-11-09T00:00:00.000Z");
  ins.run("p2", 2, "open", "2026-11-11T11:00:00.000Z");
  ins.run("p3", 3, "closed", "2026-11-01T00:00:00.000Z");
  const m = startMaintain(app, { duties: duties.filter((d) => d.name === "pulls-waiting"), everyMs: 1e9 });
  m.stop();
  await m.tick(now);
  const [row] = latest(app.db, 1);
  expect(row!.outcome).toBe("ok");
  expect(row!.findings).toEqual(["1 open pull waiting over 24 hours"]);
  expect(row!.probes[0]).toMatchObject({ kind: "query", number: 1, at: now.toISOString() });
  app.db.exec("DELETE FROM issue WHERE id IN ('p1','p2','p3')");
});
test("the example duty's count searches an existing index, not the table", () => {
  const plan = (app.db.query("EXPLAIN QUERY PLAN SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1").all("x") as { detail: string }[]).map((r) => r.detail).join("\n");
  expect(plan).toMatch(/USING (COVERING )?INDEX/);
  expect(plan).not.toMatch(/SCAN|TEMP B-TREE/);
});
test("GET /api/v1/maintenance/runs lists the latest rows for an admin only", async () => {
  clear();
  const m = startMaintain(app, { duties: [{ name: "t-list", intervalMs: 1, run: () => ({ findings: [], actions: [] }) }], everyMs: 1e9 });
  m.stop();
  await m.tick();
  const get = (token?: string, q = "") => fetch(`${servers.url}/api/v1/maintenance/runs${q}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
  expect((await get()).status).toBe(401);
  expect((await get(bob)).status).toBe(404);
  const res = await get(admin, "?limit=5");
  expect(res.status).toBe(200);
  const body = (await res.json()) as { runs: { duty: string; outcome: string }[] };
  expect(body.runs[0]).toMatchObject({ duty: "t-list", outcome: "ok" });
  expect((await get(admin, "?limit=abc")).status).toBe(200);
});
		@@ -0,0 +1,9 @@
	1	# Maintenance duties (#11204)
	2
	3	A duty is one file in `src/maintain/duties/`: a `name`, an `intervalMs` and a `run` that returns `findings` and `actions`. Register it with one line in `src/maintain/registry.ts`.
	4
	5	Every minute the runner (`src/maintain/runner.ts`, inside `during("maintain")`) runs each duty whose interval has passed since it last started, never two runs of one duty at once, and writes a `maintenance_run` row: duty, started, finished, findings, actions, probes, outcome.
	6
	7	A finding must carry a probe: `{ kind: "query" \| "measurement", what, number, at }`. A finding without one is refused: the run keeps no findings or actions and its outcome reads `refused: finding "…" has no probe`. A duty that throws reads `failed: <message>`.
	8
	9	`GET /api/v1/maintenance/runs?limit=20` (forge admin, `ro` scope) lists the latest rows. The shipped duty, `pulls-waiting`, counts open pulls older than 24 hours.
		@@ -0,0 +1,11 @@
	1	import type { Route } from "../shared.ts";
	2
	3	export const route: Route = {
	4	method: "get",
	5	path: "/api/v1/maintenance/runs",
	6	summary:
	7	"The latest maintenance duty runs, newest first, each with its findings, actions and probes (a forge admin's token; ?limit= up to 100); answers { runs }",
	8	auth: true,
	9	role: "member",
	10	refusals: { forbidden: "the token is not a person who administers the forge" },
	11	};
		@@ -102,6 +102,7 @@ export { route as getApiV1Front } from "./getApiV1Front.ts";
102	102	export { route as getApiV1Health } from "./getApiV1Health.ts";
103	103	export { route as getApiV1Here } from "./getApiV1Here.ts";
104	104	export { route as getApiV1Keeper } from "./getApiV1Keeper.ts";
	105	export { route as getApiV1MaintenanceRuns } from "./getApiV1MaintenanceRuns.ts";
105	106	export { route as getApiV1Map } from "./getApiV1Map.ts";
106	107	export { route as getApiV1MigrateJob } from "./getApiV1MigrateJob.ts";
107	108	export { route as getApiV1MigrateJobStream } from "./getApiV1MigrateJobStream.ts";
		@@ -0,0 +1,1 @@
	1	c7266eda3263e87bc4570f7dd1403080b279857caf3f4fb5a21414509bf13fa5
		@@ -0,0 +1,18 @@
	1	// Maintenance runs (#11204): one row per run of a maintenance duty, with what
	2	// it found, what it did and the probe behind each finding. Additive.
	3	// `maintenance_run_duty` serves "when did this duty last start" and
	4	// the primary key the latest-first list. The example duty adds no index to
	5	// `issue`: a new one there changes the planner's choice for the cold reads.
	6	export default `
	7	CREATE TABLE maintenance_run (
	8	id INTEGER PRIMARY KEY AUTOINCREMENT,
	9	duty TEXT NOT NULL,
	10	started TEXT NOT NULL,
	11	finished TEXT NOT NULL,
	12	findings TEXT NOT NULL DEFAULT '[]',
	13	actions TEXT NOT NULL DEFAULT '[]',
	14	probes TEXT NOT NULL DEFAULT '[]',
	15	outcome TEXT NOT NULL
	16	);
	17	CREATE INDEX maintenance_run_duty ON maintenance_run (duty, id);
	18	`;
		@@ -369,6 +369,7 @@ import { businessPlanRoutes } from "./api/business_plan.ts";
369	369	import { tokenRoutes } from "./api/tokens.ts";
370	370	import { accountAdmin, adminRoutes } from "./api/admin.ts";
371	371	import { maintenanceRoutes } from "./api/maintenance.ts";
	372	import { maintenanceRunsRoutes } from "./api/maintenance_runs.ts";
372	373	import { diskReadingRoutes } from "./api/disk_readings.ts";
373	374	import { TEMPLATE_ERRORS } from "../apps/template_tip.ts";
374	375	import { flagsRoutes } from "./api/flags.ts";
		@@ -1260,6 +1261,7 @@ export function apiRoutes(router: Router<Handler>): void {
1260	1261	forgeOriginRoutes(router);
1261	1262	adminRoutes(router);
1262	1263	maintenanceRoutes(router);
	1264	maintenanceRunsRoutes(router);
1263	1265	diskReadingRoutes(router);
1264	1266	flagsRoutes(router);
1265	1267	appTemplateRoutes(router);
		@@ -0,0 +1,20 @@
	1	/** `GET /api/v1/maintenance/runs`: the latest maintenance duty runs, for a forge admin. */
	2
	3	import { latest } from "../../maintain/runs.ts";
	4	import type { Handler } from "../context.ts";
	5	import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
	6	import { json, notFound, requireScope, unauthorized } from "../respond.ts";
	7	import type { Router } from "../router.ts";
	8
	9	export function maintenanceRunsRoutes(router: Router<Handler>): void {
	10	router.add("GET", "/api/v1/maintenance/runs", (ctx) => {
	11	notePhase(ctx.request, "session", takeSessionTime(ctx.request));
	12	if (!ctx.auth.user) return unauthorized();
	13	if (!ctx.auth.user.isAdmin) return notFound();
	14	const denied = requireScope(ctx, "ro");
	15	if (denied) return denied;
	16	const asked = Number(ctx.url.searchParams.get("limit") ?? 20);
	17	const limit = Number.isInteger(asked) && asked > 0 ? asked : 20;
	18	return json({ runs: latest(ctx.app.db, limit) });
	19	});
	20	}
		@@ -59,6 +59,7 @@ import { VERIFY_EVERY_MS } from "./checks/verify.ts";
59	59	import { RED_HOLD_MS } from "./checks/red_main.ts";
60	60	import { stopQueue } from "./checks/queue.ts";
61	61	import { startLooks } from "./looks/service.ts";
	62	import { startMaintain } from "./maintain/runner.ts";
62	63	import { startMirrorOut } from "./origins/mirror-out.ts";
63	64	import { startMirrorPulls } from "./origins/mirror-pulls.ts";
64	65	import { startProbes } from "./contract/probes.ts";
		@@ -501,6 +502,8 @@ const mirrorPulls = startMirrorPulls(app);
501	502	// The look: a picture of every environment that comes up green, taken
502	503	// through this forge's own listener.
503	504	const looks = appsOn ? startLooks(app, { localUrl: servers.url }) : undefined;
	505	// Maintenance duties (src/maintain): each runs on its interval and leaves a row.
	506	const maintain = startMaintain(app);
504	507	// Is it so: the forge knocks on every door of an app with a contract after
505	508	// each production deploy and once a night, and compares with the table.
506	509	const probes = apps === undefined ? undefined : startProbes(app, apps);
		@@ -953,6 +956,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
953	956	mirrorOut(),
954	957	mirrorPulls(),
955	958	looks?.stop(),
	959	maintain.stop(),
956	960	probes?.stop(),
957	961	nightly?.stop(),
958	962	snapshots.stop(),
		@@ -0,0 +1,25 @@
	1	/** Example duty (#11204): how many open pulls have waited over 24 hours. Reads only. */
	2
	3	import type { Duty } from "../duty.ts";
	4
	5	const DAY_MS = 24 * 3_600_000;
	6	const SQL = "SELECT COUNT(*) AS n FROM issue WHERE kind = 'pull' AND state = 'open' AND parked = 0 AND created_at < ?1";
	7
	8	export const duty: Duty = {
	9	name: "pulls-waiting",
	10	intervalMs: 3_600_000,
	11	run({ db, now }) {
	12	const cutoff = new Date(now.getTime() - DAY_MS).toISOString();
	13	const n = (db.query(SQL).get(cutoff) as { n: number }).n;
	14	if (n === 0) return { findings: [], actions: [] };
	15	return {
	16	findings: [
	17	{
	18	summary: `${n} open pull${n === 1 ? "" : "s"} waiting over 24 hours`,
	19	probe: { kind: "query", what: SQL.replace("?1", `'${cutoff}'`), number: n, at: now.toISOString() },
	20	},
	21	],
	22	actions: [],
	23	};
	24	},
	25	};
		@@ -0,0 +1,63 @@
	1	/** What a maintenance duty is (#11204): a name, an interval and a handler; plain data in, plain data out. */
	2
	3	import type { Database } from "bun:sqlite";
	4
	5	/** The number a finding rests on and when it was read: a query's result or a measurement. */
	6	export interface Probe {
	7	readonly kind: "query" \| "measurement";
	8	/** The SQL, or what was measured. */
	9	readonly what: string;
	10	readonly number: number;
	11	/** ISO time the number was read. */
	12	readonly at: string;
	13	}
	14
	15	export interface Finding {
	16	readonly summary: string;
	17	readonly probe?: Probe \| undefined;
	18	}
	19
	20	export interface DutyResult {
	21	readonly findings: readonly Finding[];
	22	/** What the duty did about it, one line each. */
	23	readonly actions: readonly string[];
	24	}
	25
	26	export interface DutyContext {
	27	readonly db: Database;
	28	readonly now: Date;
	29	}
	30
	31	export interface Duty {
	32	/** Lowercase words and dashes; the row's `duty`. */
	33	readonly name: string;
	34	readonly intervalMs: number;
	35	readonly run: (ctx: DutyContext) => DutyResult \| Promise<DutyResult>;
	36	}
	37
	38	const NAME = /^[a-z][a-z0-9-]{0,63}$/;
	39	const MAX_TEXT = 500;
	40
	41	export const validName = (name: string): boolean => NAME.test(name);
	42
	43	/** Why a probe is no probe, or undefined when it is one. */
	44	export function probeFault(p: Probe \| undefined): string \| undefined {
	45	if (p === undefined \|\| p === null) return "no probe";
	46	if (p.kind !== "query" && p.kind !== "measurement") return "probe kind is neither query nor measurement";
	47	if (typeof p.what !== "string" \|\| p.what.trim() === "") return "probe says nothing of what was read";
	48	if (typeof p.number !== "number" \|\| !Number.isFinite(p.number)) return "probe has no number";
	49	if (typeof p.at !== "string" \|\| Number.isNaN(Date.parse(p.at))) return "probe has no time";
	50	return undefined;
	51	}
	52
	53	/** The first finding refused, as a sentence; undefined when every finding stands on a probe. */
	54	export function refusal(findings: readonly Finding[]): string \| undefined {
	55	for (const f of findings) {
	56	if (typeof f?.summary !== "string" \|\| f.summary.trim() === "") return "a finding says nothing";
	57	const fault = probeFault(f.probe);
	58	if (fault !== undefined) return `refused: finding "${f.summary.slice(0, 80)}" has ${fault}`;
	59	}
	60	return undefined;
	61	}
	62
	63	export const clip = (text: string): string => (text.length > MAX_TEXT ? `${text.slice(0, MAX_TEXT)}…` : text);